From 1535548cd6eccce36fba6a93aa9bac4c4486dba9 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 4 Nov 2024 17:24:36 +0000 Subject: [PATCH 001/169] [patch] add kmodel api call to playbook --- .../playbooks/oneclick_add_aibroker.yml | 9 +++++---- .../roles/aibroker/tasks/tenant/install/main.yml | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 4b06b45776..a163e775f6 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -18,7 +18,7 @@ # DB2 # ----------------------------------------------------------------------------- - # db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" + db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" pre_tasks: # For the full set of supported environment variables refer to the playbook documentation @@ -44,12 +44,13 @@ - lookup('env', 'MAS_AIBROKER_DB_DATABASE') != "" - lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') != "" - lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') != "" - # - lookup('env', 'MAS_CONFIG_DIR') != "" + - lookup('env', 'MAS_CONFIG_DIR') != "" # db2 support + - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" # db2 support fail_msg: "One or more required environment variables are not defined" roles: - # - ibm.mas_devops.db2 - # - ibm.mas_devops.suite_config + - ibm.mas_devops.db2 # db2 support + - ibm.mas_devops.suite_config # db2 support - ibm.mas_devops.odh - ibm.mas_devops.kmodels - ibm.mas_devops.suite_app_install diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 3c59b8d993..99b4b9b523 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -58,3 +58,18 @@ kubernetes.core.k8s: apply: yes definition: "{{ lookup('template', 'templates/tenant/rbac.yml.j2') }}" + +- name: debug domain + debug: + msg: "Domain name ......... {{ domain }}" + +- name: Remove prefix from domain + set_fact: + hostname: "{{ domain | regex_replace('^[^.]+\\.', '') }}" + +- name: Apply Kmodel api + shell: | + curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ hostname }}/api/v1/tenant?id={{ tenantNamespace }}" \ + --header 'accept: */*' \ + --header 'Content-Type: application/json' \ + --data '{"name": "string","public": true,"max_models": 10}' From 512cd4b0841df1c52ed941be51da3835ad4c11df Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 09:29:16 +0000 Subject: [PATCH 002/169] [patch] fix linter errors --- .../roles/aibroker/tasks/tenant/install/main.yml | 8 ++++---- .../roles/cp4d/templates/catalog_sources/5.0.0.yml | 1 - ibm/mas_devops/roles/suite_app_config/vars/health.yml | 3 +-- ibm/mas_devops/roles/suite_app_config/vars/manage.yml | 3 +-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 99b4b9b523..8c0ffa6f51 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -59,13 +59,13 @@ apply: yes definition: "{{ lookup('template', 'templates/tenant/rbac.yml.j2') }}" -- name: debug domain +- name: "debug domain" debug: msg: "Domain name ......... {{ domain }}" -- name: Remove prefix from domain - set_fact: - hostname: "{{ domain | regex_replace('^[^.]+\\.', '') }}" +- name: "Remove prefix from domain" + set_fact: + hostname: "{{ domain | regex_replace('^[^.]+\\.', '') }}" - name: Apply Kmodel api shell: | diff --git a/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.0.0.yml b/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.0.0.yml index fda8c80507..100b5f77c3 100644 --- a/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.0.0.yml +++ b/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.0.0.yml @@ -20,4 +20,3 @@ catsrc_info: registry: icr.io # tag: 4.8.0 digest: sha256:6b43f09c1335a28e71669a0ecbd6fdee0ec2afe74d62c014b0d1d2d5a04b4ee3 - diff --git a/ibm/mas_devops/roles/suite_app_config/vars/health.yml b/ibm/mas_devops/roles/suite_app_config/vars/health.yml index 2041305ec8..12a6c9176d 100644 --- a/ibm/mas_devops/roles/suite_app_config/vars/health.yml +++ b/ibm/mas_devops/roles/suite_app_config/vars/health.yml @@ -10,8 +10,7 @@ mas_app_settings_aio_flag: "{{ lookup('env', 'MAS_APP_SETTINGS_AIO_FLAG') | defa mas_app_settings_db2_schema: "{{ lookup('env', 'MAS_APP_SETTINGS_DB2_SCHEMA') | default('maximo', true)}}" mas_app_settings_db_schema: "{{ lookup('env', 'MAS_APP_SETTINGS_DB_SCHEMA') | default(mas_app_settings_db2_schema, true)}}" mas_app_settings_demodata: "{{ lookup('env', 'MAS_APP_SETTINGS_DEMODATA') | default('false', true) }}" -mas_app_settings_db2vargraphic : "{{ lookup('env', 'MAS_APP_SETTINGS_DB2VARGRAPHIC') | default('true', true) | bool }}" - +mas_app_settings_db2vargraphic: "{{ lookup('env', 'MAS_APP_SETTINGS_DB2VARGRAPHIC') | default('true', true) | bool }}" mas_app_settings_tablespace: "{{ lookup('env', 'MAS_APP_SETTINGS_TABLESPACE') | default('MAXDATA', true)}}" mas_app_settings_indexspace: "{{ lookup('env', 'MAS_APP_SETTINGS_INDEXSPACE') | default('MAXINDEX', true)}}" diff --git a/ibm/mas_devops/roles/suite_app_config/vars/manage.yml b/ibm/mas_devops/roles/suite_app_config/vars/manage.yml index 68d3e72604..b78c953edd 100644 --- a/ibm/mas_devops/roles/suite_app_config/vars/manage.yml +++ b/ibm/mas_devops/roles/suite_app_config/vars/manage.yml @@ -11,8 +11,7 @@ mas_app_settings_aio_flag: "{{ lookup('env', 'MAS_APP_SETTINGS_AIO_FLAG') | defa mas_app_settings_db2_schema: "{{ lookup('env', 'MAS_APP_SETTINGS_DB2_SCHEMA') | default('maximo', true)}}" mas_app_settings_db_schema: "{{ lookup('env', 'MAS_APP_SETTINGS_DB_SCHEMA') | default(mas_app_settings_db2_schema, true)}}" mas_app_settings_demodata: "{{ lookup('env', 'MAS_APP_SETTINGS_DEMODATA') | default('false', true) | bool }}" -mas_app_settings_db2vargraphic : "{{ lookup('env', 'MAS_APP_SETTINGS_DB2VARGRAPHIC') | default('true', true) | bool }}" - +mas_app_settings_db2vargraphic: "{{ lookup('env', 'MAS_APP_SETTINGS_DB2VARGRAPHIC') | default('true', true) | bool }}" mas_app_settings_tablespace: "{{ lookup('env', 'MAS_APP_SETTINGS_TABLESPACE') | default('MAXDATA', true)}}" mas_app_settings_indexspace: "{{ lookup('env', 'MAS_APP_SETTINGS_INDEXSPACE') | default('MAXINDEX', true)}}" From e58cb958e1e04a12c6ead38fd1680b15edf3d14c Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 11:25:35 +0000 Subject: [PATCH 003/169] [patch] add domain for KMODEL API --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 1 + .../roles/aibroker/tasks/tenant/install/main.yml | 14 +++++++------- ibm/mas_devops/roles/aibroker/vars/main.yml | 1 + 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index a163e775f6..cbb5f3f41f 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -44,6 +44,7 @@ - lookup('env', 'MAS_AIBROKER_DB_DATABASE') != "" - lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') != "" - lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') != "" + - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" - lookup('env', 'MAS_CONFIG_DIR') != "" # db2 support - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" # db2 support fail_msg: "One or more required environment variables are not defined" diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 8c0ffa6f51..2e5eb19cfb 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -59,17 +59,17 @@ apply: yes definition: "{{ lookup('template', 'templates/tenant/rbac.yml.j2') }}" -- name: "debug domain" - debug: - msg: "Domain name ......... {{ domain }}" +# - name: "debug domain" +# debug: +# msg: "Domain name ......... {{ domain }}" -- name: "Remove prefix from domain" - set_fact: - hostname: "{{ domain | regex_replace('^[^.]+\\.', '') }}" +# - name: "Remove prefix from domain" +# set_fact: +# hostname: "{{ domain | regex_replace('^[^.]+\\.', '') }}" - name: Apply Kmodel api shell: | - curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ hostname }}/api/v1/tenant?id={{ tenantNamespace }}" \ + curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id={{ tenantNamespace }}" \ --header 'accept: */*' \ --header 'Content-Type: application/json' \ --data '{"name": "string","public": true,"max_models": 10}' diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 183daafa0e..63e611396b 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -5,6 +5,7 @@ mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-opera aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" pullSecretName: "ibm-entitlement" +mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" # Whether to perform an install or remove tenant tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" From 8175177ccbb8a9b13f93129ce59524bee0e397a1 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 16:22:48 +0000 Subject: [PATCH 004/169] [patch] add python scripts --- .../aibroker/files/create_entitlement.py | 107 +++++++++++++++ .../roles/aibroker/files/create_tenant.py | 77 +++++++++++ .../files/create_tenant_entitlement.py | 123 ++++++++++++++++++ ibm/mas_devops/roles/aibroker/tasks/main.yml | 9 ++ .../aibroker/tasks/tenant/install/main.yml | 3 + 5 files changed, 319 insertions(+) create mode 100644 ibm/mas_devops/roles/aibroker/files/create_entitlement.py create mode 100644 ibm/mas_devops/roles/aibroker/files/create_tenant.py create mode 100644 ibm/mas_devops/roles/aibroker/files/create_tenant_entitlement.py diff --git a/ibm/mas_devops/roles/aibroker/files/create_entitlement.py b/ibm/mas_devops/roles/aibroker/files/create_entitlement.py new file mode 100644 index 0000000000..b8fde3f917 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/files/create_entitlement.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + +import datetime +import ibm_db +import sys +import os + +if len(sys.argv) < 8: + print("Usage: python3 create_entitlement.py entitlement_type model_type allowed_number_of_model_training_per_month allowed_number_of_deployed_model_instance allowed_number_of_concurrent_training allowed_number_of_input allowed_watsonx_call allowed_file_size") + print("Sample usage") + print("Usage: python3 create_entitlement.py standard pcc 10 1 1 10000 100 1MB") + exit() + +entitlement_type = sys.argv[1] +model_type = sys.argv[2] +allowed_number_of_model_training_per_month = sys.argv[3] +allowed_number_of_deployed_model_instance = sys.argv[4] +allowed_number_of_concurrent_training = sys.argv[5] +allowed_number_of_input = sys.argv[6] +allowed_watsonx_call = sys.argv[7] +allowed_file_size = sys.argv[8] + +if entitlement_type is None: + entitlement_type = 'standard' + +if model_type is None: + model_type = 'pcc' + +if allowed_number_of_model_training_per_month is None: + allowed_number_of_model_training_per_month = 10 + +if allowed_number_of_deployed_model_instance is None: + allowed_number_of_deployed_model_instance = 1 + +if allowed_number_of_concurrent_training is None: + allowed_number_of_concurrent_training = 1 + +if allowed_number_of_input is None: + allowed_number_of_input = 10000 + +if allowed_watsonx_call is None: + allowed_watsonx_call = 10000 + +if allowed_file_size is None: + allowed_file_size = '1MB' + + +# Retrieving schema +schema = os.getenv("NAMESPACE") +schema_uppercase = schema.upper() +print(schema_uppercase) +assert schema != None, "namespace env variable not found." + +# Retrieving DB Details from env vars +db_host = os.getenv("db_host") +db_port = os.getenv("db_port") +db_name = os.getenv("db2_db") +db_username = os.getenv("db_username") +db_password = os.getenv("db_password") + +assert db_host != None, "db_host env variable not found." +assert db_port != None, "db_port env variable not found." +assert db_name != None, "db_name env variable not found." +assert db_username != None, "db_username env variable not found." +assert db_password != None, "db_password env variable not found." + + + +# ct stores current time +ct = datetime.datetime.now() + +array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} +#conn = ibm_db.pconnect("DATABASE=BLUDB;HOSTNAME=api.ivt811.cp.fyre.ibm.com;PORT=8890;PROTOCOL=TCPIP;UID=db2inst1;PWD=IQg1fAtVOyOQZVA;", "", "", array) + +# use certificate to connect to DB2 +conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_ +password};", "", "", array) + +#statement_entitlement_string = "INSERT INTO AIBROKER.AIBROKER_ENTITLEMENT_TYPE (ENTITLEMENT_TYPE, MODEL_TYPE, ALLOWED_NUMBER_OF_MODEL_TRAINING_PER_MONTH, ALLOWED_NUMBER_OF_DEPLOYED_MODEL_INSTANCE, ALLOWED_NUMBER_OF_CONCURRENT_TRAINING, ALLOWED_NUMBER_OF_INPUT,ALLOWED_NUMBER_OF_WATSONX_CALL, ALLOWED_FILE_SIZE, CREATION_TIMESTAMP, UPDATED_TIMESTAMP) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?,?);" + +statement_entitlement_string = f"INSERT INTO \"{schema_uppercase}\".AIBROKER_ENTITLEMENT_TYPE (ENTITLEMENT_TYPE, MODEL_TYPE, ALLOWED_NUMBER_OF_MODEL_TRAINING_PER_MONTH, ALLOWED_N +UMBER_OF_DEPLOYED_MODEL_INSTANCE, ALLOWED_NUMBER_OF_CONCURRENT_TRAINING, ALLOWED_NUMBER_OF_INPUT,ALLOWED_NUMBER_OF_WATSONX_CALL, ALLOWED_FILE_SIZE, CREATION_TIMESTAMP, UPDATED_TI +MESTAMP) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?,?);" +stmt = ibm_db.prepare(conn, statement_entitlement_string) +ibm_db.bind_param(stmt, 1, entitlement_type) +ibm_db.bind_param(stmt, 2, model_type) +ibm_db.bind_param(stmt, 3, allowed_number_of_model_training_per_month) +ibm_db.bind_param(stmt, 4, allowed_number_of_deployed_model_instance) +ibm_db.bind_param(stmt, 5, allowed_number_of_concurrent_training) +ibm_db.bind_param(stmt, 6, allowed_number_of_input) +ibm_db.bind_param(stmt, 7, allowed_watsonx_call) +ibm_db.bind_param(stmt, 8, allowed_file_size) +ibm_db.bind_param(stmt, 9, ct) +ibm_db.bind_param(stmt, 10, ct) +try: + stmt = ibm_db.execute(stmt) +except Exception as e: + print("Transaction couldn't be completed.") + ibm_db.rollback(conn) +else: + ibm_db.commit(conn) + print("Transaction complete. AIBROKER_ENTITLEMENT_TYPE table has been populated.") diff --git a/ibm/mas_devops/roles/aibroker/files/create_tenant.py b/ibm/mas_devops/roles/aibroker/files/create_tenant.py new file mode 100644 index 0000000000..36d2cddc1c --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/files/create_tenant.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + +import datetime +import ibm_db +import sys +import os + +if len(sys.argv) < 3: + print("Usage: python3 create_tenant.py tenant_name sls_url dro_url ") + print("Sample usage") + print("Usage: python3 create_tenant.py kmodels-user3 sls_url dro_url") + exit() + +tenant_name = sys.argv[1] +sls_url = sys.argv[2] +dro_url = sys.argv[3] + + + +# Retrieving schema +schema = os.getenv("NAMESPACE") +schema_uppercase = schema.upper() +print(schema_uppercase) +assert schema != None, "namespace env variable not found." + +# Retrieving DB Details from env vars +db_host = os.getenv("db_host") +db_port = os.getenv("db_port") +db_name = os.getenv("db2_db") +db_username = os.getenv("db_username") +db_password = os.getenv("db_password") + +assert db_host != None, "db_host env variable not found." +assert db_port != None, "db_port env variable not found." +assert db_name != None, "db_name env variable not found." +assert db_username != None, "db_username env variable not found." +assert db_password != None, "db_password env variable not found." + +array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} +conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) +#conn = ibm_db.pconnect(f"DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) + + + + + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Adding entry in AIBROKER_TENANT table. +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +#statement_entitlement_string = f"INSERT INTO {schema}.AIBROKER_TENANT(NAME, STATUS, SLS_URL, DRO_URL) VALUES(?,?,?,?);" +statement_entitlement_string = f"INSERT INTO \"{schema_uppercase}\".AIBROKER_TENANT(NAME, STATUS, SLS_URL, DRO_URL) VALUES(?,?,?,?);" + + +stmt = ibm_db.prepare(conn, statement_entitlement_string) +ibm_db.bind_param(stmt, 1, tenant_name) +ibm_db.bind_param(stmt, 2, 'active') +ibm_db.bind_param(stmt, 3, sls_url) +ibm_db.bind_param(stmt, 4, dro_url) + +try: + stmt = ibm_db.execute(stmt) +except Exception as e: + print("Transaction couldn't be completed. The Tenant has not been created") + ibm_db.rollback(conn) +else: + ibm_db.commit(conn) + print("Transaction complete. The Tenant has been created.") + + diff --git a/ibm/mas_devops/roles/aibroker/files/create_tenant_entitlement.py b/ibm/mas_devops/roles/aibroker/files/create_tenant_entitlement.py new file mode 100644 index 0000000000..08f25899c2 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/files/create_tenant_entitlement.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + +import datetime +import ibm_db +import sys +import os + +if len(sys.argv) < 5: + print("Usage: python3 create_tenant_entitlement.py tenant_name entitlement_type model_type entilement_start_date entilement_end_date ") + print("Sample usage") + print("Usage: python3 create_tenant_entitlement.py kmodels-user3 standard pcc 2024-09-12 2025-09-12") + exit() + +tenant_name = sys.argv[1] + +entitlement_type = sys.argv[2] +model_type = sys.argv[3] +entilement_start_date=sys.argv[4] +entilement_end_date=sys.argv[5] + + +# Retrieving schema +schema = os.getenv("NAMESPACE") +schema_uppercase = schema.upper() +print(schema_uppercase) +assert schema != None, "namespace env variable not found." + +# Retrieving DB Details from env vars +db_host = os.getenv("db_host") +db_port = os.getenv("db_port") +db_name = os.getenv("db2_db") +db_username = os.getenv("db_username") +db_password = os.getenv("db_password") + +assert db_host != None, "db_host env variable not found." +assert db_port != None, "db_port env variable not found." +assert db_name != None, "db_name env variable not found." +assert db_username != None, "db_username env variable not found." +assert db_password != None, "db_password env variable not found." + +array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} +conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) +#conn = ibm_db.pconnect(f"DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) + + +# Retrirving entitlement_type id using entitlement_type and model_type + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Retrirving entitlement_type id using entitlement_type and model_type +# - +# -------------------------------------------------------------------------------------------------------------------------------- +#entitlement_type_string = f"SELECT ID FROM {schema}.AIBROKER_ENTITLEMENT_TYPE WHERE ENTITLEMENT_TYPE = ? AND MODEL_TYPE = ?;" +entitlement_type_string = f"SELECT ID FROM \"{schema_uppercase}\".AIBROKER_ENTITLEMENT_TYPE WHERE ENTITLEMENT_TYPE = ? AND MODEL_TYPE = ?;" +print(entitlement_type_string) +stmt = ibm_db.prepare(conn, entitlement_type_string) +ibm_db.bind_param(stmt, 1, entitlement_type) +ibm_db.bind_param(stmt, 2, model_type) + +try: + ibm_db.execute(stmt) + assert ibm_db.fetch_row(stmt) != False, "entitlement_type and model_type combination does not exist in database." + entitlement_id = int(ibm_db.result(stmt, "ID")) + print(f"Fetched entitlement id : {entitlement_id}") +except Exception as e: + print(ibm_db.stmt_errormsg(stmt)) + print(f"Couldn't retrieve entitlement type from database. Exception : {e}") + + + + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Retrirving tenant id using tenant_name +# - +# -------------------------------------------------------------------------------------------------------------------------------- + + +#tenant_string = f"SELECT ID FROM {schema}.AIBROKER_TENANT WHERE NAME = ?;" +tenant_string = f"SELECT ID FROM \"{schema_uppercase}\".AIBROKER_TENANT WHERE NAME = ?;" +stmt = ibm_db.prepare(conn, tenant_string) +ibm_db.bind_param(stmt, 1, tenant_name) + + +try: + ibm_db.execute(stmt) + assert ibm_db.fetch_row(stmt) != False, "tenant_name does not exist in database."+tenant_name + tenant_id = int(ibm_db.result(stmt, "ID")) + print(f"Fetched tenant id : {tenant_id}") +except Exception as e: + print(ibm_db.stmt_errormsg(stmt)) + print(f"Couldn't retrieve tenant id from database. Exception : {e}") + + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - insert into AIBROKER_TENANT_ENTITLEMENT +# - +# -------------------------------------------------------------------------------------------------------------------------------- + + +#statement_tenant_entitlement = f"INSERT INTO {schema}.AIBROKER_TENANT_ENTITLEMENT(TENANT_ID, ENTITLEMENT_ID, START_DATE, EXPIRE_DATE) VALUES(?,?,?,?);" +statement_tenant_entitlement = f"INSERT INTO \"{schema_uppercase}\".AIBROKER_TENANT_ENTITLEMENT(TENANT_ID, ENTITLEMENT_ID, START_DATE, EXPIRE_DATE) VALUES(?,?,?,?);" + +stmt = ibm_db.prepare(conn, statement_tenant_entitlement) +ibm_db.bind_param(stmt, 1, tenant_id) +ibm_db.bind_param(stmt, 2, entitlement_id) +ibm_db.bind_param(stmt, 3, entilement_start_date ) +ibm_db.bind_param(stmt, 4, entilement_end_date) + +try: + stmt = ibm_db.execute(stmt) +except Exception as e: + print("Transaction couldn't be completed. The relationship of tenant and entitlement has not been created") + ibm_db.rollback(conn) +else: + ibm_db.commit(conn) + print("Transaction complete. The relationship of tenant and entitlement has been created.") diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 612a1b4f66..3c35c7d20b 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -5,6 +5,15 @@ # install AI Broker api # - include_tasks: tasks/aibroker/main.yml +- name: Create entitlement + command: python3 {{ role_path }}/files/create_entitlement.py standard pcc 10 1 1 10000 100 1MB + register: create_entitlement + ignore_errors: yes + +- name: Debug create entitlement + debug: + msg: "{{ create_entitlement }}" + # create AI Broker tenant - include_tasks: tasks/tenant/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 2e5eb19cfb..c44bb8aea3 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -73,3 +73,6 @@ --header 'accept: */*' \ --header 'Content-Type: application/json' \ --data '{"name": "string","public": true,"max_models": 10}' + +# python3 create_tenant.py kmodels-user3 sls_url dro_url +# python3 create_tenant_entitlement.py kmodels-user3 standard pcc 2024-01-01 2027-01-01 From 10059d67076db1c3aa411bfc83b7d6c4581ad8f4 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 16:26:27 +0000 Subject: [PATCH 005/169] [patch] remove whitespace --- ibm/mas_devops/roles/aibroker/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 3c35c7d20b..bc587aa409 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -5,7 +5,7 @@ # install AI Broker api # - include_tasks: tasks/aibroker/main.yml -- name: Create entitlement +- name: Create entitlement command: python3 {{ role_path }}/files/create_entitlement.py standard pcc 10 1 1 10000 100 1MB register: create_entitlement ignore_errors: yes From 93dc3a8d53ec0ba1386847474b6b070fd4d58c38 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 16:35:03 +0000 Subject: [PATCH 006/169] [patch] add python script for create tenant --- .../aibroker/tasks/tenant/install/main.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index c44bb8aea3..3a86c65d8e 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -74,5 +74,20 @@ --header 'Content-Type: application/json' \ --data '{"name": "string","public": true,"max_models": 10}' -# python3 create_tenant.py kmodels-user3 sls_url dro_url -# python3 create_tenant_entitlement.py kmodels-user3 standard pcc 2024-01-01 2027-01-01 +- name: Create tenant + command: python3 {{ role_path }}/files/create_tenant.py {{ tenantNamespace }} sls_url dro_url + register: create_tenant + ignore_errors: yes + +- name: Debug Create tenant + debug: + msg: "{{ create_tenant }}" + +- name: Create tenant entitlement + command: python3 {{ role_path }}/files/create_tenant_entitlement.py {{ tenantNamespace }} standard pcc 2024-01-01 2027-01-01 + register: create_tenant_entitlement + ignore_errors: yes + +- name: Debug Create tenant entitlement + debug: + msg: "{{ create_tenant_entitlement }}" From 6a90b900b4bb1c8e9f3301f12cea5ab49ffb1673 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 16:38:40 +0000 Subject: [PATCH 007/169] [patch] remove not used string --- ibm/mas_devops/roles/aibroker/files/create_entitlement.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_entitlement.py b/ibm/mas_devops/roles/aibroker/files/create_entitlement.py index b8fde3f917..25e8d18956 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_entitlement.py +++ b/ibm/mas_devops/roles/aibroker/files/create_entitlement.py @@ -75,7 +75,6 @@ ct = datetime.datetime.now() array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} -#conn = ibm_db.pconnect("DATABASE=BLUDB;HOSTNAME=api.ivt811.cp.fyre.ibm.com;PORT=8890;PROTOCOL=TCPIP;UID=db2inst1;PWD=IQg1fAtVOyOQZVA;", "", "", array) # use certificate to connect to DB2 conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_ From e56a35615e650d8ea62182ccd13834f4a7888dda Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 16:40:42 +0000 Subject: [PATCH 008/169] [patch] remove commented out connection --- ibm/mas_devops/roles/aibroker/files/create_tenant.py | 5 ----- .../roles/aibroker/files/create_tenant_entitlement.py | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_tenant.py b/ibm/mas_devops/roles/aibroker/files/create_tenant.py index 36d2cddc1c..d36891e97d 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_tenant.py +++ b/ibm/mas_devops/roles/aibroker/files/create_tenant.py @@ -43,11 +43,6 @@ array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) -#conn = ibm_db.pconnect(f"DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) - - - - # -------------------------------------------------------------------------------------------------------------------------------- # - diff --git a/ibm/mas_devops/roles/aibroker/files/create_tenant_entitlement.py b/ibm/mas_devops/roles/aibroker/files/create_tenant_entitlement.py index 08f25899c2..e71486a5dc 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_tenant_entitlement.py +++ b/ibm/mas_devops/roles/aibroker/files/create_tenant_entitlement.py @@ -45,17 +45,7 @@ array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) -#conn = ibm_db.pconnect(f"DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) - -# Retrirving entitlement_type id using entitlement_type and model_type - -# -------------------------------------------------------------------------------------------------------------------------------- -# - -# - Retrirving entitlement_type id using entitlement_type and model_type -# - -# -------------------------------------------------------------------------------------------------------------------------------- -#entitlement_type_string = f"SELECT ID FROM {schema}.AIBROKER_ENTITLEMENT_TYPE WHERE ENTITLEMENT_TYPE = ? AND MODEL_TYPE = ?;" entitlement_type_string = f"SELECT ID FROM \"{schema_uppercase}\".AIBROKER_ENTITLEMENT_TYPE WHERE ENTITLEMENT_TYPE = ? AND MODEL_TYPE = ?;" print(entitlement_type_string) stmt = ibm_db.prepare(conn, entitlement_type_string) From 2ac09a6bca050e24f090d731914ad4b4c9c4029d Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 16:55:52 +0000 Subject: [PATCH 009/169] [patch] fix conn string --- ibm/mas_devops/roles/aibroker/files/create_entitlement.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_entitlement.py b/ibm/mas_devops/roles/aibroker/files/create_entitlement.py index 25e8d18956..73e1a53490 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_entitlement.py +++ b/ibm/mas_devops/roles/aibroker/files/create_entitlement.py @@ -77,8 +77,7 @@ array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} # use certificate to connect to DB2 -conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_ -password};", "", "", array) +conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) #statement_entitlement_string = "INSERT INTO AIBROKER.AIBROKER_ENTITLEMENT_TYPE (ENTITLEMENT_TYPE, MODEL_TYPE, ALLOWED_NUMBER_OF_MODEL_TRAINING_PER_MONTH, ALLOWED_NUMBER_OF_DEPLOYED_MODEL_INSTANCE, ALLOWED_NUMBER_OF_CONCURRENT_TRAINING, ALLOWED_NUMBER_OF_INPUT,ALLOWED_NUMBER_OF_WATSONX_CALL, ALLOWED_FILE_SIZE, CREATION_TIMESTAMP, UPDATED_TIMESTAMP) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?,?);" From a2982274a6f565daea3e3b0cef4b9a9511505862 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 5 Nov 2024 17:01:12 +0000 Subject: [PATCH 010/169] [patch] fix python scripts --- ibm/mas_devops/roles/aibroker/files/create_entitlement.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_entitlement.py b/ibm/mas_devops/roles/aibroker/files/create_entitlement.py index 73e1a53490..82688e3884 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_entitlement.py +++ b/ibm/mas_devops/roles/aibroker/files/create_entitlement.py @@ -81,9 +81,7 @@ #statement_entitlement_string = "INSERT INTO AIBROKER.AIBROKER_ENTITLEMENT_TYPE (ENTITLEMENT_TYPE, MODEL_TYPE, ALLOWED_NUMBER_OF_MODEL_TRAINING_PER_MONTH, ALLOWED_NUMBER_OF_DEPLOYED_MODEL_INSTANCE, ALLOWED_NUMBER_OF_CONCURRENT_TRAINING, ALLOWED_NUMBER_OF_INPUT,ALLOWED_NUMBER_OF_WATSONX_CALL, ALLOWED_FILE_SIZE, CREATION_TIMESTAMP, UPDATED_TIMESTAMP) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?,?);" -statement_entitlement_string = f"INSERT INTO \"{schema_uppercase}\".AIBROKER_ENTITLEMENT_TYPE (ENTITLEMENT_TYPE, MODEL_TYPE, ALLOWED_NUMBER_OF_MODEL_TRAINING_PER_MONTH, ALLOWED_N -UMBER_OF_DEPLOYED_MODEL_INSTANCE, ALLOWED_NUMBER_OF_CONCURRENT_TRAINING, ALLOWED_NUMBER_OF_INPUT,ALLOWED_NUMBER_OF_WATSONX_CALL, ALLOWED_FILE_SIZE, CREATION_TIMESTAMP, UPDATED_TI -MESTAMP) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?,?);" +statement_entitlement_string = f"INSERT INTO \"{schema_uppercase}\".AIBROKER_ENTITLEMENT_TYPE (ENTITLEMENT_TYPE, MODEL_TYPE, ALLOWED_NUMBER_OF_MODEL_TRAINING_PER_MONTH, ALLOWED_NUMBER_OF_DEPLOYED_MODEL_INSTANCE, ALLOWED_NUMBER_OF_CONCURRENT_TRAINING, ALLOWED_NUMBER_OF_INPUT,ALLOWED_NUMBER_OF_WATSONX_CALL, ALLOWED_FILE_SIZE, CREATION_TIMESTAMP, UPDATED_TIMESTAMP) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?,?);" stmt = ibm_db.prepare(conn, statement_entitlement_string) ibm_db.bind_param(stmt, 1, entitlement_type) ibm_db.bind_param(stmt, 2, model_type) From 4e7ec2ec222b5cc98a05b4eae7cc3289a113d202 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 7 Nov 2024 17:02:48 +0000 Subject: [PATCH 011/169] [patch] use api instead pyrhon scripts --- ibm/mas_devops/roles/aibroker/tasks/main.yml | 37 ++++++++++++++----- .../aibroker/tasks/tenant/install/main.yml | 34 ++++++++--------- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index bc587aa409..7de22ab596 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -5,21 +5,38 @@ # install AI Broker api # - include_tasks: tasks/aibroker/main.yml -- name: Create entitlement - command: python3 {{ role_path }}/files/create_entitlement.py standard pcc 10 1 1 10000 100 1MB - register: create_entitlement - ignore_errors: yes +# - name: Create entitlement +# command: python3 {{ role_path }}/files/create_entitlement.py standard pcc 10 1 1 10000 100 1MB +# register: create_entitlement +# ignore_errors: yes -- name: Debug create entitlement - debug: - msg: "{{ create_entitlement }}" - -# create AI Broker tenant -- include_tasks: tasks/tenant/main.yml +# curl -X 'POST' \ +# 'http://localhost:8080/ibm/aibroker/service/rest/api/v1/tenant' \ +# -H 'accept: */*' \ +# -H 'Content-Type: application/json' \ +# -d '{ +# "tenant_name": "karol_tenant", +# "sls_url": "karol_tenant_sls", +# "dro_url": "karol_tenant_dro_url" +# }' # create AI Broker api key - include_tasks: tasks/apikey/main.yml +# read api and setfacts + +- name: Create tenant and add to db2 for saas + shell: | + curl -k -X POST "https://aibroker.aibdev.apps.aibrokerdev.cp.fyre.ibm.com/ibm/aibroker/service/rest/api/v1/tenant" \ + --header 'apikey: OzKQck2Bh1OaxqizJ1d1SXufMXNfophE' \ + --header 'accept: */*' \ + --header 'Content-Type: application/json' \ + --header 'tenantid: aibroker-user' \ + --data ' {"tenant_name": "aibroker-user", "sls_url": "karol_tenant_sls", "dro_url": "karol_tenant_dro_url"}' + +# create AI Broker tenant +- include_tasks: tasks/tenant/main.yml + # create s3 api key - include_tasks: tasks/s3/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 3a86c65d8e..df2520b70d 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -74,20 +74,20 @@ --header 'Content-Type: application/json' \ --data '{"name": "string","public": true,"max_models": 10}' -- name: Create tenant - command: python3 {{ role_path }}/files/create_tenant.py {{ tenantNamespace }} sls_url dro_url - register: create_tenant - ignore_errors: yes - -- name: Debug Create tenant - debug: - msg: "{{ create_tenant }}" - -- name: Create tenant entitlement - command: python3 {{ role_path }}/files/create_tenant_entitlement.py {{ tenantNamespace }} standard pcc 2024-01-01 2027-01-01 - register: create_tenant_entitlement - ignore_errors: yes - -- name: Debug Create tenant entitlement - debug: - msg: "{{ create_tenant_entitlement }}" +# - name: Create tenant +# command: python3 {{ role_path }}/files/create_tenant.py {{ tenantNamespace }} sls_url dro_url +# register: create_tenant +# ignore_errors: yes + +# - name: Debug Create tenant +# debug: +# msg: "{{ create_tenant }}" + +# - name: Create tenant entitlement +# command: python3 {{ role_path }}/files/create_tenant_entitlement.py {{ tenantNamespace }} standard pcc 2024-01-01 2027-01-01 +# register: create_tenant_entitlement +# ignore_errors: yes + +# - name: Debug Create tenant entitlement +# debug: +# msg: "{{ create_tenant_entitlement }}" From 1625213be8ec86b2189d7f61952e3f0807b6a17f Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 8 Nov 2024 16:00:49 +0000 Subject: [PATCH 012/169] [patch] update code logic --- ibm/mas_devops/roles/aibroker/tasks/main.yml | 79 ++++++++++++++------ ibm/mas_devops/roles/aibroker/vars/main.yml | 7 +- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 7de22ab596..342df007e6 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -5,38 +5,73 @@ # install AI Broker api # - include_tasks: tasks/aibroker/main.yml -# - name: Create entitlement -# command: python3 {{ role_path }}/files/create_entitlement.py standard pcc 10 1 1 10000 100 1MB -# register: create_entitlement -# ignore_errors: yes - -# curl -X 'POST' \ -# 'http://localhost:8080/ibm/aibroker/service/rest/api/v1/tenant' \ -# -H 'accept: */*' \ -# -H 'Content-Type: application/json' \ -# -d '{ -# "tenant_name": "karol_tenant", -# "sls_url": "karol_tenant_sls", -# "dro_url": "karol_tenant_dro_url" -# }' +# run api calls for aibroker saas +- name: Apply Kmodel api + shell: | + curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id=provision-tenant" \ + --header 'accept: */*' \ + --header 'Content-Type: application/json' \ + --data '{"name": "string","public": true,"max_models": 10}' -# create AI Broker api key -- include_tasks: tasks/apikey/main.yml +- name: "Check if API key secret exists in namespace: provision-tenant" + k8s_info: + api_version: v1 + kind: Secret + name: "provision-tenant----apikey-secret" + namespace: "{{ aibroker_namespace }}" + register: apikey_secret_info -# read api and setfacts +- name: Create API Key + script: "{{ role_path }}/files/create_apikey.sh provision-tenant {{ aibroker_namespace }}" + when: + - apikey_secret_info.resources | length == 0 -- name: Create tenant and add to db2 for saas +- name: "Lookup tenant provision-tenant api key" + shell: oc get secret provision-tenant----apikey-secret -n {{ aibroker_namespace }} -o json | jq -r '.data.AIBROKER_APIKEY' | base64 -d + register: provision_tenant_api_key_output + +- name: "Set aibroker api key variable" + set_fact: + provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" + +- name: "apikey" + debug: + msg: "{{ provision_tenant_api_key }}" + +- name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" shell: | - curl -k -X POST "https://aibroker.aibdev.apps.aibrokerdev.cp.fyre.ibm.com/ibm/aibroker/service/rest/api/v1/tenant" \ - --header 'apikey: OzKQck2Bh1OaxqizJ1d1SXufMXNfophE' \ + curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ + --header 'apikey: {{ provision_tenant_api_key }}' \ --header 'accept: */*' \ --header 'Content-Type: application/json' \ - --header 'tenantid: aibroker-user' \ - --data ' {"tenant_name": "aibroker-user", "sls_url": "karol_tenant_sls", "dro_url": "karol_tenant_dro_url"}' + --header 'tenantid: provision-tenant' \ + --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' + register: add_tenant_to_db2 + +- name: Debug tenant + debug: + msg: "{{ add_tenant_to_db2 }}" + +- name: "Create tenantEntitlements {{ tenantNamespace }} and add to db2 for saas" + shell: | + curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" \ + --header 'apikey: {{ provision_tenant_api_key }}' \ + --header 'accept: */*' \ + --header 'Content-Type: application/json' \ + --header 'tenantid: provision-tenant' \ + --data '{"tenant_name": "{{ tenantNamespace }}", "entitlement_type": "standard", "model_type": "similarity", "entitlement_start_date": "2024-01-01", "entitlement_end_date": "2025-01-01"}' + register: add_tenant_entitlements_to_db2 + +- name: Debug tenant entitlement + debug: + msg: "{{ add_tenant_entitlements_to_db2 }}" # create AI Broker tenant - include_tasks: tasks/tenant/main.yml +# create AI Broker api key +- include_tasks: tasks/apikey/main.yml + # create s3 api key - include_tasks: tasks/s3/main.yml diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 63e611396b..ad755dc3ca 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -5,7 +5,7 @@ mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-opera aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" pullSecretName: "ibm-entitlement" -mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" + # Whether to perform an install or remove tenant tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" @@ -39,3 +39,8 @@ mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | mas_aibroker_s3_action: "{{ lookup('env', 'MAS_AIBROKER_S3_ACTION') | default('install', true) }}" mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" + +# SAAS +mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" +mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') }}" +mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') }}" From 32b224be0ab2857663fe6e11a78af8cfd04bb3b7 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 8 Nov 2024 16:14:54 +0000 Subject: [PATCH 013/169] [patch] add is saas --- ibm/mas_devops/roles/aibroker/vars/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index ad755dc3ca..bb8ae41626 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -42,5 +42,6 @@ mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | defa # SAAS mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" -mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') }}" -mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') }}" +mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" +mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" +is_saas: "{{ lookup('env', 'IS_SAAS') | default('', false) }}" From 5d710244702212734c6a7ab372169b01154a45b6 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 8 Nov 2024 16:22:33 +0000 Subject: [PATCH 014/169] [patch] add condition to run when saas --- ibm/mas_devops/roles/aibroker/tasks/main.yml | 60 +------------------ .../roles/aibroker/tasks/saas/main.yml | 60 +++++++++++++++++++ 2 files changed, 62 insertions(+), 58 deletions(-) create mode 100644 ibm/mas_devops/roles/aibroker/tasks/saas/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 342df007e6..701e4a0ff2 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -6,65 +6,9 @@ # - include_tasks: tasks/aibroker/main.yml # run api calls for aibroker saas -- name: Apply Kmodel api - shell: | - curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id=provision-tenant" \ - --header 'accept: */*' \ - --header 'Content-Type: application/json' \ - --data '{"name": "string","public": true,"max_models": 10}' - -- name: "Check if API key secret exists in namespace: provision-tenant" - k8s_info: - api_version: v1 - kind: Secret - name: "provision-tenant----apikey-secret" - namespace: "{{ aibroker_namespace }}" - register: apikey_secret_info - -- name: Create API Key - script: "{{ role_path }}/files/create_apikey.sh provision-tenant {{ aibroker_namespace }}" +- include_tasks: tasks/saas/main.yml when: - - apikey_secret_info.resources | length == 0 - -- name: "Lookup tenant provision-tenant api key" - shell: oc get secret provision-tenant----apikey-secret -n {{ aibroker_namespace }} -o json | jq -r '.data.AIBROKER_APIKEY' | base64 -d - register: provision_tenant_api_key_output - -- name: "Set aibroker api key variable" - set_fact: - provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" - -- name: "apikey" - debug: - msg: "{{ provision_tenant_api_key }}" - -- name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" - shell: | - curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ - --header 'apikey: {{ provision_tenant_api_key }}' \ - --header 'accept: */*' \ - --header 'Content-Type: application/json' \ - --header 'tenantid: provision-tenant' \ - --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' - register: add_tenant_to_db2 - -- name: Debug tenant - debug: - msg: "{{ add_tenant_to_db2 }}" - -- name: "Create tenantEntitlements {{ tenantNamespace }} and add to db2 for saas" - shell: | - curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" \ - --header 'apikey: {{ provision_tenant_api_key }}' \ - --header 'accept: */*' \ - --header 'Content-Type: application/json' \ - --header 'tenantid: provision-tenant' \ - --data '{"tenant_name": "{{ tenantNamespace }}", "entitlement_type": "standard", "model_type": "similarity", "entitlement_start_date": "2024-01-01", "entitlement_end_date": "2025-01-01"}' - register: add_tenant_entitlements_to_db2 - -- name: Debug tenant entitlement - debug: - msg: "{{ add_tenant_entitlements_to_db2 }}" + - is_saas == "true" # create AI Broker tenant - include_tasks: tasks/tenant/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml new file mode 100644 index 0000000000..a5be7a786a --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml @@ -0,0 +1,60 @@ +--- +- name: Apply Kmodel api + shell: | + curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id=provision-tenant" \ + --header 'accept: */*' \ + --header 'Content-Type: application/json' \ + --data '{"name": "string","public": true,"max_models": 10}' + +- name: "Check if API key secret exists in namespace: provision-tenant" + k8s_info: + api_version: v1 + kind: Secret + name: "provision-tenant----apikey-secret" + namespace: "{{ aibroker_namespace }}" + register: apikey_secret_info + +- name: Create API Key + script: "{{ role_path }}/files/create_apikey.sh provision-tenant {{ aibroker_namespace }}" + when: + - apikey_secret_info.resources | length == 0 + +- name: "Lookup tenant provision-tenant api key" + shell: oc get secret provision-tenant----apikey-secret -n {{ aibroker_namespace }} -o json | jq -r '.data.AIBROKER_APIKEY' | base64 -d + register: provision_tenant_api_key_output + +- name: "Set aibroker api key variable" + set_fact: + provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" + +- name: "apikey" + debug: + msg: "{{ provision_tenant_api_key }}" + +- name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" + shell: | + curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ + --header 'apikey: {{ provision_tenant_api_key }}' \ + --header 'accept: */*' \ + --header 'Content-Type: application/json' \ + --header 'tenantid: provision-tenant' \ + --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' + register: add_tenant_to_db2 + +- name: Debug tenant + debug: + msg: "{{ add_tenant_to_db2 }}" + +- name: "Create tenantEntitlements {{ tenantNamespace }} and add to db2 for saas" + shell: | + curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" \ + --header 'apikey: {{ provision_tenant_api_key }}' \ + --header 'accept: */*' \ + --header 'Content-Type: application/json' \ + --header 'tenantid: provision-tenant' \ + --data '{"tenant_name": "{{ tenantNamespace }}", "entitlement_type": "standard", "model_type": "similarity", "entitlement_start_date": "2024-01-01", "entitlement_end_date": "2025-01-01"}' + register: add_tenant_entitlements_to_db2 + +- name: Debug tenant entitlement + debug: + msg: "{{ add_tenant_entitlements_to_db2 }}" From 24ce196250bdb9ad76a5926b32c0f1d2d2aaad60 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 8 Nov 2024 16:46:58 +0000 Subject: [PATCH 015/169] [patch] rename saas variable --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 2 ++ ibm/mas_devops/roles/aibroker/tasks/main.yml | 2 +- ibm/mas_devops/roles/aibroker/vars/main.yml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index cbb5f3f41f..8b57c9e5ea 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -45,6 +45,8 @@ - lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') != "" - lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') != "" - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" + - lookup('env', 'MAS_AIBROKER_SLS_URL') != "" + - lookup('env', 'MAS_AIBROKER_DRO_URL') != "" - lookup('env', 'MAS_CONFIG_DIR') != "" # db2 support - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" # db2 support fail_msg: "One or more required environment variables are not defined" diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 701e4a0ff2..769ef9a71f 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -8,7 +8,7 @@ # run api calls for aibroker saas - include_tasks: tasks/saas/main.yml when: - - is_saas == "true" + - mas_aibroker_saas == "true" # create AI Broker tenant - include_tasks: tasks/tenant/main.yml diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index bb8ae41626..532f4e84e9 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -44,4 +44,4 @@ mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | defa mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" -is_saas: "{{ lookup('env', 'IS_SAAS') | default('', false) }}" +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('', false) }}" From 83bc521fadc7a794e64e2cba4c888e9bd480e1d9 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 8 Nov 2024 16:52:02 +0000 Subject: [PATCH 016/169] [patch] remove whitespaces --- ibm/mas_devops/roles/aibroker/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 769ef9a71f..22a6f8c349 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -5,7 +5,7 @@ # install AI Broker api # - include_tasks: tasks/aibroker/main.yml -# run api calls for aibroker saas +# run api calls for aibroker saas - include_tasks: tasks/saas/main.yml when: - mas_aibroker_saas == "true" From 82733a4b533a1dad45b9531b8de93f772fc3c5b6 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 11 Nov 2024 16:44:40 +0000 Subject: [PATCH 017/169] [patch] add script to create sls and dro secret --- .../aibroker/files/alm_sample_sls_use.py | 259 ++++++++++++++++++ .../roles/aibroker/files/create_dro_token.sh | 49 ++++ .../roles/aibroker/files/create_sls_secret.sh | 46 ++++ .../roles/aibroker/tasks/saas/main.yml | 8 +- .../aibroker/tasks/tenant/install/main.yml | 34 ++- .../aibroker/templates/tenant/rbac.yml.j2 | 1 + ibm/mas_devops/roles/aibroker/vars/main.yml | 2 + 7 files changed, 384 insertions(+), 15 deletions(-) create mode 100644 ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py create mode 100644 ibm/mas_devops/roles/aibroker/files/create_dro_token.sh create mode 100644 ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh diff --git a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py new file mode 100644 index 0000000000..ad3d42b8e2 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py @@ -0,0 +1,259 @@ +# ----------------------------------------------------------- +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +# ----------------------------------------------------------- + +# This python script shows a simple model through which Sustainability offerings can interact in the SaaS pattern with a common ALM AppPoint pool. We'll +# see registration (provisioning), usage and deprovisioning. For clarity and simplicity, we do not perform validation of the certificate presented by the SLS service. Of course, you will want to +# do this in your offering. + +import os, base64, time, urllib3, requests, json, sys , secrets , string , random +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +slsUrl = sys.argv[1] +slsRegistrationKey = sys.argv[2] +#print(slsUrl) +#print(slsRegistrationKey) +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Configuration section - you need to complete steps 1-5 +# - +# -------------------------------------------------------------------------------------------------------------------------------- + + +def generate_api_key(): + alphabet = string.ascii_letters + string.digits + temp= ''.join(secrets.choice(alphabet) for i in range(12)) + return "88"+ temp.lower() + + +# 1: Provide a name for your offering + +offeringName = "aibroker" + +# 2: Provide am identifier for your instance. This should be pseudounique, so chances of collisions are low. For example, based on the cluster +# identity, or a pseudo unique UUID created at initial installation time + +instanceIdentifier = generate_api_key() +#print("id") +print(instanceIdentifier) + +# 3: Provide the SLS instance URL (SRE will give you this) + + + +# 4: Provide the SLS registration key (SRE will give you this) + + + +# 5: Decide if you want the demonstration to tear down your client afterwards (as you would do in a deprovision). If not, the client +# will remain registered for you to experiment with. + +# set it to be False Kewei +#deprovision=True +deprovision=False + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: preparation +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# Prepare the SLS clientId to use, and prepare for storing client certificates that are generated at registration time. +slsClientId=offeringName+"-"+instanceIdentifier +certs_dir=f"{os.getcwd()}/certs" +clientCaCrtPath=certs_dir+"/"+slsClientId+"-ca.crt" +clientTlsCrtPath=certs_dir+"/"+slsClientId+"-tls.crt" +clientTlsKeyPath=certs_dir+"/"+slsClientId+"-tls.key" + +#print ("Using clientId %s"%slsClientId) +#print ("Client certificates for interacting with SLS will be stored in "+certs_dir+"/"+slsClientId) + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: registration (provision) or reuse +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# Let's check to see if client certificates already exist - if so, we can immediately move on to working with AppPoints. If not, +# we'll need to register ourselves with SLS to obtain the client certificates. + +if os.path.isfile(clientCaCrtPath): + # A client certificate appears to exist already. So let's use it. + #print ("Discovered client certificates for SLS client "+slsClientId+". Let's use them.") + pass +else: + #print ("This appears to be a new SLS client. Let's attempt to register and obtain some client certificates.") + pass + + # Make initial registration request + data = {'clientId': slsClientId, 'description': 'ALM sample client registration', 'role': 'manage'} # TODO: Rawa - Swagger docs incorrectly state 'id' rather than 'clientId' + headers = {'X-Registration-Key': slsRegistrationKey} # TODO: Rawa - Swagger docs don't document the need for this critical header + + #print ("Making initial registration request for client "+slsClientId) + response = requests.post(slsUrl+"/api/registrations",verify=False,json=data, headers=headers ) + response.raise_for_status() + + registrationId = response.json()['registrationId'] + #print("Completing registration for registration request "+registrationId) + + # + # Poll registration status until it's complete + # + provisioningWaiting = True + while provisioningWaiting: + headers = {'X-Registration-Key': slsRegistrationKey} + response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=False,headers=headers ) + response.raise_for_status() + status = response.json()['state'] + #print("Waiting for client provisioning to be completed. Current status: %s" %status) + if status=="AWAITING_CONFIRMATION": + # We are now ready to proceed; client certs are available + #print ("Extracting client certificates") + clientTlskey= base64.b64decode(response.json()['certs']['tls.key']) + clientTlscrt= base64.b64decode(response.json()['certs']['tls.crt']) + clientCacrt= base64.b64decode(response.json()['certs']['ca.crt']) + with open(f'{clientTlsCrtPath}', 'wb') as f: + f.write(clientTlscrt) + with open(f'{clientTlsKeyPath}', 'wb') as f: + f.write(clientTlskey) + with open(f'{clientCaCrtPath}', 'wb') as f: + f.write(clientCacrt) + provisioningWaiting = False + time.sleep(1) + + # The next step is to confirm the client. Poll status until complete. + confirmationWaiting = True + while confirmationWaiting: + response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=False, cert=(clientTlsCrtPath,clientTlsKeyPath)) + #print (response) + status = response.json()['state'] + #print("Waiting for client registration to be confirmed. Current status: %s" %status) + if status=="REGISTERED": + #print ("Client registration confirmed.") + confirmationWaiting = False + time.sleep(1) + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: Define helper functions to perform SLS actions: mirror AppPoint usage to SLS and obtain AppPoint usage from SLS +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# Update 'quantity' AppPoints used in SLS. After 24 hours, the AppPoints will be automatically returned (ensuring no orphaned AppPoints if your service instance disappears) +def spendAppPoints(quantity): + #print ("Let's spend "+str(quantity)+" AppPoints") + data = {'validity': 86400, 'quantity': quantity} + response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) + response.raise_for_status() + +# String representation of AppPoint usage by your client. TODO: SLS will shortly be providing an update to correctly report usage for your client. At present, it returns the overall usage across all clients. +def getUsage(): + response = requests.get(slsUrl+"/api/tokens/",verify=False,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner + response.raise_for_status() + j = json.loads(response.content.decode('utf-8')) + return j[0]['used'] + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: Interact with AppPoints +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# The ALM SaaS metering model is to perform the following actions every hour: +# - Locally calculate your AppPoint usage (based on your individual pricing model) +# - Check out the ValueMetric-Usage feature with (i) a licenseeId that is pseudo unique, (ii) an expiry of 86400 and (iii) a quantity matching the number of AppPoints +# +# Note that at each hour, you simply update your AppPoint usage: you do not need to explicitly return your previous hour's AppPoints +# +# As an example, let's suppose your AppPoints are charged on (i) users and (ii) IOPoints. +# 25 Users = 25 AppPoints +# 100 IOPoints = 10 AppPoints +# +# You can use one of two approaches to check out AppPoints. +# +# SIMPLE "single shot" +# -------------------- +# - Report AppPoint usage in one go. +# - This means a single, consistent licenseeId at the instance level - for example, licenseeId="", i.e. reusing the SLS clientId is convenient. +# - Set the quantity to be 25+10 = 35 +# +# +# COMPLEX "breakdown by capability" +# --------------------------------- +# - Report AppPoint usage as a sequence of updates, per capability +# - Enables you to use SLS as a way to understand unequivocally breakdown of AppPoint usage by capability +# - Each capability has its own licenseeId - for example: +# - example: licenseeId="",quantity=25 +# licenseeId="",quantity=10 +# +# In this code example, we'll do a single shot for simplicity and clarity + +# As described above, for simplicity in the single shot model, we set the licenseeId to the SLS clientId +licenseeId = slsClientId + +# Now let's define the license feature to check out. In SaaS ALM, this will be "MAS-ValueMetric-Usage" +productId = "MAS-ValueMetric-Usage" + +# +# Now let's start the day-to-day operation of the software - i.e., hourly updates of SLS of AppPoint usage, and hourly reporting of data (ready to send to DRO) +# + +#print ("Simulating a few hours' usage. Note that you will require an ALM-ready version of SLS, and an ALM-ready license, to report AppPoint usage correctly in shared environments.") + +# Hour 0: Check out 25+10=35 AppPoints +#print ("Hour 0") +#spendAppPoints(35) +# Get AppPoint usage from SLS (e.g., for reporting via DRO). +#print("Usage: "+str(getUsage())) + +# Hour 1: It's now 143 AppPoints overall +#print ("Hour 1") +#spendAppPoints(143) +#print("Usage: "+str(getUsage())) + +# Hour 2: It's now 133 AppPoints overall +#print ("Hour 2") +#spendAppPoints(133) +#print("Usage: "+str(getUsage())) + +# Hour 3: you get the picture :) +# +# ... + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: optional deprovision +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# Now suppose a request has been made to deprovision your service. How do we gracefully decouple from SLS? +if deprovision: + +# At deprovision, return your licenses and unregister the client + print ("Preparing for deprovision: returning any AppPoints to the pool") + response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response.raise_for_status() + print("Usage after deprovisioning: "+str(getUsage())) + + print ("Preparing for deprovision: unregistering the SLS client") + response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response.raise_for_status() + #print ("Deleting redundant client certificates") + os.remove(clientTlsCrtPath) + os.remove(clientTlsKeyPath) + os.remove(clientCaCrtPath) + +else: + #print ("No deprovisioning will occur. Your client is still registered with SLS.") + pass + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: end +# - +# -------------------------------------------------------------------------------------------------------------------------------- +#print ("Demo over.") diff --git a/ibm/mas_devops/roles/aibroker/files/create_dro_token.sh b/ibm/mas_devops/roles/aibroker/files/create_dro_token.sh new file mode 100644 index 0000000000..77c493e798 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/files/create_dro_token.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +###### +# +# create DRO token in k8s secret +# +####### + +echo "Usage ./create_dro_token.sh tenant_name_in_lower_case instance_id token" + + +NAMESPACE='aibroker' + + +TENANT=$1 +instance_id=$2 +token=$3 + +if [ -z ${instance_id}] +then + #echo "❌ Missing instance_id " + #exit 1 + echo "using default namespace aibroker" + NAMESPACE='aibroker' + +else + NAMESPACE=mas-${instance_id}-aibroker +fi + +echo $NAMESPACE + +echo $token + + + +if [ -z ${TENANT} ]; then + echo "using default tenant name=aibroker-user" + TENANT='aibroker-user' + #echo "Usage ./create_apikey.sh tenant_name_in_lower_case" + #exit 1 +fi + + + + + +echo "Creating DRO token in k8s secret" +oc create secret generic ${TENANT}----dro-secret -n ${NAMESPACE} \ + --from-literal=DRO_TOKEN=${token} diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh new file mode 100644 index 0000000000..61185d8d55 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +TENANT=$1 +instance_id=$2 + +if [ -z ${TENANT} ]; then + #echo "❌ Missing tenant name" + #exit 1 + echo "Usage ./create_sls_secret.sh tenant-name aibroker-instance-id-from-CustomResourceDefinitions sls-url-from-sls-routes slsRegistrationKey-from-sls-api-pod-Environment" + echo "for example" + echo "./create_sls_secret.sh aibroker-user aibdev https://sls.ibm-sls.xxx.ibm.com xxxx-xxxx-xxxx-xxx" + exit 1 +fi + +cwd=$(pwd) +echo $cwd + +mkdir -p certs + +echo "creating SLS registration, please wait....." +instanceIdentifier=`python3 alm_sample_sls_use.py $3 $4` +echo "SLS registration is created successfully." + +registrationKey=$4 + + + +echo $instanceIdentifier +# Set variables +# Need to change +NAMESPACE=mas-${instance_id}-aibroker +slsClientId="aibroker"-$instanceIdentifier +echo $slsClientId + +SECRET_NAME=${TENANT}----sls-secret +CA_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-ca.crt +TLS_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.crt +KEY_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.key + +# Create the secret +oc create secret generic $SECRET_NAME -n $NAMESPACE \ + --from-file=$CA_FILE_PATH \ + --from-file=$TLS_FILE_PATH \ + --from-file=$KEY_FILE_PATH \ + --from-literal=SLS_CLIENT_ID=${slsClientId} \ + --from-literal=SLS_REGISTRATION_KEY=${registrationKey} diff --git a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml index a5be7a786a..46712fa469 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml @@ -27,10 +27,6 @@ set_fact: provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" -- name: "apikey" - debug: - msg: "{{ provision_tenant_api_key }}" - - name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" shell: | curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ @@ -41,7 +37,7 @@ --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' register: add_tenant_to_db2 -- name: Debug tenant +- name: Debug tenant #TODO: remove on finall commit debug: msg: "{{ add_tenant_to_db2 }}" @@ -55,6 +51,6 @@ --data '{"tenant_name": "{{ tenantNamespace }}", "entitlement_type": "standard", "model_type": "similarity", "entitlement_start_date": "2024-01-01", "entitlement_end_date": "2025-01-01"}' register: add_tenant_entitlements_to_db2 -- name: Debug tenant entitlement +- name: Debug tenant entitlement #TODO: remove on finall commit debug: msg: "{{ add_tenant_entitlements_to_db2 }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index df2520b70d..10a6814d1f 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -74,15 +74,31 @@ --header 'Content-Type: application/json' \ --data '{"name": "string","public": true,"max_models": 10}' -# - name: Create tenant -# command: python3 {{ role_path }}/files/create_tenant.py {{ tenantNamespace }} sls_url dro_url -# register: create_tenant -# ignore_errors: yes - -# - name: Debug Create tenant -# debug: -# msg: "{{ create_tenant }}" - +- name: Create sls secret sample for AI Broker + command: python3 {{ role_path }}/files/alm_sample_sls_use.py {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} + register: create_sls_secret_output + ignore_errors: yes + +- name: Debug Create sls secret for AI Broker + debug: + msg: "{{ create_sls_secret_output }}" + +- name: Create SLS secret + script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }}" + register: sls_secret_output + +- name: Debug Create sls secret for AI Broker + debug: + msg: "{{ sls_secret_output }}" + +- name: Create DRO token + script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_aibroker_dro_token }}" + register: dro_token_output + +- name: Debug Create DRO token + debug: + msg: "{{ dro_token_output }}" + # - name: Create tenant entitlement # command: python3 {{ role_path }}/files/create_tenant_entitlement.py {{ tenantNamespace }} standard pcc 2024-01-01 2027-01-01 # register: create_tenant_entitlement diff --git a/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 index 67825d90cc..030092c087 100644 --- a/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 @@ -39,6 +39,7 @@ rules: - list - delete - get + - patch apiGroups: - serving.kserve.io resources: diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 532f4e84e9..2853f09d18 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -45,3 +45,5 @@ mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('', false) }}" +mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" +mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" From 28ce4d8bba3cef47b3fdcdb5c30cbd57560bdd35 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 11 Nov 2024 16:57:34 +0000 Subject: [PATCH 018/169] [patch] fix whitespace --- .../roles/aibroker/files/create_sls_secret.sh | 2 +- .../roles/aibroker/tasks/tenant/install/main.yml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index 61185d8d55..cde9f14a89 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -18,7 +18,7 @@ echo $cwd mkdir -p certs echo "creating SLS registration, please wait....." -instanceIdentifier=`python3 alm_sample_sls_use.py $3 $4` +instanceIdentifier=`python3 {{ role_path }}/files/alm_sample_sls_use.py $3 $4` echo "SLS registration is created successfully." registrationKey=$4 diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 10a6814d1f..b6ccfdbe31 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -74,14 +74,14 @@ --header 'Content-Type: application/json' \ --data '{"name": "string","public": true,"max_models": 10}' -- name: Create sls secret sample for AI Broker - command: python3 {{ role_path }}/files/alm_sample_sls_use.py {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} - register: create_sls_secret_output - ignore_errors: yes +# - name: Create sls secret sample for AI Broker +# command: python3 {{ role_path }}/files/alm_sample_sls_use.py {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} +# register: create_sls_secret_output +# ignore_errors: yes -- name: Debug Create sls secret for AI Broker - debug: - msg: "{{ create_sls_secret_output }}" +# - name: Debug Create sls secret for AI Broker +# debug: +# msg: "{{ create_sls_secret_output }}" - name: Create SLS secret script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }}" @@ -98,7 +98,7 @@ - name: Debug Create DRO token debug: msg: "{{ dro_token_output }}" - + # - name: Create tenant entitlement # command: python3 {{ role_path }}/files/create_tenant_entitlement.py {{ tenantNamespace }} standard pcc 2024-01-01 2027-01-01 # register: create_tenant_entitlement From 064b255f0b25fc8ab9c1eafaf13e8186ac000b4e Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 11 Nov 2024 17:01:51 +0000 Subject: [PATCH 019/169] [patch] update script for create SLS secret --- ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index cde9f14a89..ebaad94acc 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -18,7 +18,8 @@ echo $cwd mkdir -p certs echo "creating SLS registration, please wait....." -instanceIdentifier=`python3 {{ role_path }}/files/alm_sample_sls_use.py $3 $4` +# instanceIdentifier=`python3 {{ role_path }}/files/alm_sample_sls_use.py $3 $4` +instanceIdentifier=$(python3 ../roles/aibroker/files/alm_sample_sls_use.py $3 $4) echo "SLS registration is created successfully." registrationKey=$4 From 2e47edecbe353d2617c9fb277d455d5060067415 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 11 Nov 2024 17:07:25 +0000 Subject: [PATCH 020/169] [patch] added missing env to script for create DRO secret --- ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index b6ccfdbe31..8a91327b5c 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -92,7 +92,7 @@ msg: "{{ sls_secret_output }}" - name: Create DRO token - script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_aibroker_dro_token }}" + script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_dro_token }}" register: dro_token_output - name: Debug Create DRO token From 3c414fd4f98c93f310f4630e5068bc50a1748e08 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 11 Nov 2024 17:12:48 +0000 Subject: [PATCH 021/169] [patch] add condition to check if secret exists --- .../roles/aibroker/tasks/tenant/install/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 8a91327b5c..d69d913d8f 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -83,8 +83,18 @@ # debug: # msg: "{{ create_sls_secret_output }}" +- name: "Check if SLS key secret exists in namespace: {{ aibroker_namespace }}" + k8s_info: + api_version: v1 + kind: Secret + name: "{{ tenantNamespace }}----sls-secret" + namespace: "{{ aibroker_namespace }}" + register: sls_secret_info + - name: Create SLS secret script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }}" + when: + - sls_secret_info.resources | length == 0 register: sls_secret_output - name: Debug Create sls secret for AI Broker From 6c647c52db8f081df64c653b06e167227889ff5c Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 12 Nov 2024 12:25:03 +0000 Subject: [PATCH 022/169] [patch] add RBAC for patch jobs/cronjobs --- ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 index 030092c087..471d384780 100644 --- a/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 @@ -14,7 +14,13 @@ rules: - configmaps - secrets - verbs: + - create + - delete + - get - list + - patch + - update + - watch apiGroups: - metrics.k8s.io resources: From d2d8f5b73723d01b29d0cace077b2cda86d79f9a Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 12 Nov 2024 13:25:17 +0000 Subject: [PATCH 023/169] [patch] add patch to jobs/cronjobs in RBAC --- ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 index 471d384780..3ab9eddd0b 100644 --- a/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 @@ -35,6 +35,7 @@ rules: - verbs: - list - create + - patch apiGroups: - batch resources: From 940e5eb790800a0d4c88f914c6e143c5eb876004 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 12 Nov 2024 13:40:33 +0000 Subject: [PATCH 024/169] [patch] roll back old RBAC for km-controller --- .../roles/aibroker/templates/tenant/rbac.yml.j2 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 index 3ab9eddd0b..fa28085ab0 100644 --- a/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/tenant/rbac.yml.j2 @@ -14,13 +14,13 @@ rules: - configmaps - secrets - verbs: - - create - delete - - get + apiGroups: + - '' + resources: + - pods + - verbs: - list - - patch - - update - - watch apiGroups: - metrics.k8s.io resources: @@ -35,7 +35,7 @@ rules: - verbs: - list - create - - patch + - delete apiGroups: - batch resources: From fa87b2df1dfed6fba987f8c7d7d8b3a116bc668c Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 12 Nov 2024 13:46:41 +0000 Subject: [PATCH 025/169] [patch] add copy secret WX to tenant namespace --- .../roles/aibroker/tasks/watsonx/install/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/watsonx/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/watsonx/install/main.yml index 3963bb1bac..466818b286 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/watsonx/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/watsonx/install/main.yml @@ -11,3 +11,9 @@ script: "{{ role_path }}/files/create_wx.sh {{ tenantNamespace }} {{ aibroker_namespace }} {{ mas_aibroker_watsonxai_apikey }} {{ mas_aibroker_watsonxai_url }} {{ mas_aibroker_watsonxai_project_id }}" when: - wx_apikey_secret_info.resources | length == 0 + +# Copy secrets to tenant namespace +- name: 'Copy secret {{ tenantNamespace }}----wx-secret to namespace: {{ tenantNamespace }}' + shell: 'oc get secret {{ item }} -n mas-{{ mas_instance_id }}-aibroker -o yaml | sed "s/namespace: .*/namespace: {{ tenantNamespace }}/" | oc apply --force -f -' + with_items: + - '{{ tenantNamespace }}----wx-secret' From 6ba3783ab824749ec67c4370e7877387c37df2e5 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 12 Nov 2024 14:00:56 +0000 Subject: [PATCH 026/169] [patch] add delete dro secret if exists --- ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index d69d913d8f..e02588da45 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -101,6 +101,12 @@ debug: msg: "{{ sls_secret_output }}" +- name: Delete API key secret + command: oc delete secret "{{ tenantNamespace }}----dro-secret" -n {{ aibroker_namespace }} + register: delete_result + ignore_errors: yes + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + - name: Create DRO token script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_dro_token }}" register: dro_token_output From 373b6c1161bc459fc22bca0f52ab67da4d2559af Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 27 Nov 2024 13:19:09 +0000 Subject: [PATCH 027/169] [patch] set var MAS_AIBROKER_SAAS as false for default --- ibm/mas_devops/roles/aibroker/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 2853f09d18..1dd016cbc1 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -44,6 +44,6 @@ mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | defa mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" -mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('', false) }}" +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" From 876eae151f85a6bdbc802dbfa7cb74ea41805da9 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 27 Nov 2024 15:14:04 +0000 Subject: [PATCH 028/169] [patch] rename task --- ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index e02588da45..bf0fda0cab 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -101,7 +101,7 @@ debug: msg: "{{ sls_secret_output }}" -- name: Delete API key secret +- name: Delete DRO secret command: oc delete secret "{{ tenantNamespace }}----dro-secret" -n {{ aibroker_namespace }} register: delete_result ignore_errors: yes From 2112581e4571f830ebd3e6e38f3f309eca3151cd Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 3 Dec 2024 11:36:24 +0000 Subject: [PATCH 029/169] [patch] add condition for controller and watcher if saas --- .../roles/aibroker/tasks/saas/main.yml | 96 ++++++++++++------- .../aibroker/tasks/tenant/install/main.yml | 74 ++++++++++++-- .../kmodels-controller/configmap.yml.j2 | 4 + .../kmodels-watcher/configmap.yml.j2 | 4 + ibm/mas_devops/roles/kmodels/vars/main.yml | 5 +- 5 files changed, 143 insertions(+), 40 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml index 46712fa469..ea1d7ae732 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml @@ -1,10 +1,37 @@ --- -- name: Apply Kmodel api - shell: | - curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id=provision-tenant" \ - --header 'accept: */*' \ - --header 'Content-Type: application/json' \ - --data '{"name": "string","public": true,"max_models": 10}' +# TODO: create route + +# Adding inference server runtimes +# - name: 'create controller route' +# kubernetes.core.k8s: +# apply: yes +# definition: "{{ lookup('template', 'templates/saas/create-controller-route.yml.j2') }}" + +# - name: Apply Kmodel api +# shell: | +# curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id=provision-tenant" \ +# --header 'accept: */*' \ +# --header 'Content-Type: application/json' \ +# --data '{"name": "string","public": true,"max_models": 10}' +# register: kmodel_api_output + +# - name: Apply Kmodel api - using uri +# uri: +# url: https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id=provision-tenant +# validate_certs: no +# method: POST +# headers: +# accept: "*/*" +# Content-Type: application/json +# body_format: "json" +# body: "{\"name\": \"string\",\"public\": true,\"max_models\": 10}" +# status_code: [200,409] +# timeout: 30 +# register: kmodel_api_output + +# - name: Kmodel debug #TODO: remove before commit +# debug: +# msg: "POST output ...... {{ kmodel_api_output }}" - name: "Check if API key secret exists in namespace: provision-tenant" k8s_info: @@ -27,30 +54,33 @@ set_fact: provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" -- name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" - shell: | - curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ - --header 'apikey: {{ provision_tenant_api_key }}' \ - --header 'accept: */*' \ - --header 'Content-Type: application/json' \ - --header 'tenantid: provision-tenant' \ - --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' - register: add_tenant_to_db2 - -- name: Debug tenant #TODO: remove on finall commit - debug: - msg: "{{ add_tenant_to_db2 }}" - -- name: "Create tenantEntitlements {{ tenantNamespace }} and add to db2 for saas" - shell: | - curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" \ - --header 'apikey: {{ provision_tenant_api_key }}' \ - --header 'accept: */*' \ - --header 'Content-Type: application/json' \ - --header 'tenantid: provision-tenant' \ - --data '{"tenant_name": "{{ tenantNamespace }}", "entitlement_type": "standard", "model_type": "similarity", "entitlement_start_date": "2024-01-01", "entitlement_end_date": "2025-01-01"}' - register: add_tenant_entitlements_to_db2 - -- name: Debug tenant entitlement #TODO: remove on finall commit - debug: - msg: "{{ add_tenant_entitlements_to_db2 }}" +# - name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" +# shell: | +# curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ +# --header 'apikey: {{ provision_tenant_api_key }}' \ +# --header 'accept: */*' \ +# --header 'Content-Type: application/json' \ +# --header 'tenantid: provision-tenant' \ +# --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' +# register: add_tenant_to_db2 + +# # - name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri +# # uri: +# # url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" +# # validate_certs: no +# # method: POST +# # headers: +# # apikey: "{{ provision_tenant_api_key }}" +# # accept: "*/*" +# # Content-Type: "application/json" +# # tenantid: "provision-tenant" +# # body_format: "json" +# # body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"sls_url\": \"{{ mas_aibroker_sls_url }}\", \"dro_url\": \"{{ mas_aibroker_dro_url }}\"}" +# # status_code: [200] +# # timeout: 30 +# # register: add_tenant_to_db2 + +# - name: Debug tenant #TODO: remove on finall commit +# debug: +# msg: "{{ add_tenant_to_db2 }}" + diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index bf0fda0cab..49698e931e 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -67,12 +67,12 @@ # set_fact: # hostname: "{{ domain | regex_replace('^[^.]+\\.', '') }}" -- name: Apply Kmodel api - shell: | - curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id={{ tenantNamespace }}" \ - --header 'accept: */*' \ - --header 'Content-Type: application/json' \ - --data '{"name": "string","public": true,"max_models": 10}' +# - name: Apply Kmodel api +# shell: | +# curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id={{ tenantNamespace }}" \ +# --header 'accept: */*' \ +# --header 'Content-Type: application/json' \ +# --data '{"name": "string","public": true,"max_models": 10}' # - name: Create sls secret sample for AI Broker # command: python3 {{ role_path }}/files/alm_sample_sls_use.py {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} @@ -115,6 +115,38 @@ debug: msg: "{{ dro_token_output }}" + +# - name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" +# shell: | +# curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ +# --header 'apikey: {{ provision_tenant_api_key }}' \ +# --header 'accept: */*' \ +# --header 'Content-Type: application/json' \ +# --header 'tenantid: provision-tenant' \ +# --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' +# register: add_tenant_to_db2 + +- name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri + uri: + url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" + validate_certs: no + method: POST + headers: + apikey: "{{ provision_tenant_api_key }}" + accept: "*/*" + Content-Type: "application/json" + tenantid: "provision-tenant" + body_format: "json" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"sls_url\": \"{{ mas_aibroker_sls_url }}\", \"dro_url\": \"{{ mas_aibroker_dro_url }}\"}" + status_code: [200] + timeout: 30 + register: add_tenant_to_db2 + +- name: Debug tenant #TODO: remove on finall commit + debug: + msg: "{{ add_tenant_to_db2 }}" + + # - name: Create tenant entitlement # command: python3 {{ role_path }}/files/create_tenant_entitlement.py {{ tenantNamespace }} standard pcc 2024-01-01 2027-01-01 # register: create_tenant_entitlement @@ -123,3 +155,33 @@ # - name: Debug Create tenant entitlement # debug: # msg: "{{ create_tenant_entitlement }}" + +# - name: "Create tenantEntitlements {{ tenantNamespace }} and add to db2 for saas" +# shell: | +# curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" \ +# --header 'apikey: {{ provision_tenant_api_key }}' \ +# --header 'accept: */*' \ +# --header 'Content-Type: application/json' \ +# --header 'tenantid: provision-tenant' \ +# --data '{"tenant_name": "{{ tenantNamespace }}", "entitlement_type": "standard", "model_type": "similarity", "entitlement_start_date": "2024-01-01", "entitlement_end_date": "2025-01-01"}' +# register: add_tenant_entitlements_to_db2 + +- name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri + uri: + url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + validate_certs: no + method: POST + headers: + apikey: "{{ provision_tenant_api_key }}" + accept: "*/*" + Content-Type: "application/json" + tenantid: "provision-tenant" + body_format: "json" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"similarity\", \"entitlement_start_date\": \"2024-01-01\", \"entitlement_end_date\": \"2025-01-01\"}" + status_code: [200] + timeout: 30 + register: add_tenant_entitlements_to_db2 + +- name: Debug tenant entitlement #TODO: remove on finall commit + debug: + msg: "{{ add_tenant_entitlements_to_db2 }}" diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 index d83c7e2899..338786395c 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 @@ -15,5 +15,9 @@ data: PIPELINE_STEP_IMAGE_PREFIX: '' PIPELINE_DEFAULT_STEP_TAG: {{ mas_aibroker_pipeline_steps_tag }} CONNECTOR_DEFAULT_TAG: {{ mas_aibroker_connector_tag }} +{% if mas_aibroker_saas | lower == 'true' %} + DEFAULT_TENANTS: provision-tenant +{% else %} DEFAULT_TENANTS: {{ tenantNamespace }} +{% endif %} PLATFORM: openshift diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 index 18349314d3..4c45d0ae4a 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 @@ -11,4 +11,8 @@ data: WATCHER_SENDER_DELAY: "5" EXCLUDE_JOB_NAMESPACES: "ibm-cpd" PIPELINES: mas-{{ mas_instance_id }}-aibroker +{% if mas_aibroker_saas | lower == 'true' %} + TENANTS: provision-tenant +{% else %} TENANTS: {{ tenantNamespace }} +{% endif %} diff --git a/ibm/mas_devops/roles/kmodels/vars/main.yml b/ibm/mas_devops/roles/kmodels/vars/main.yml index 3e879806f7..35998a021b 100644 --- a/ibm/mas_devops/roles/kmodels/vars/main.yml +++ b/ibm/mas_devops/roles/kmodels/vars/main.yml @@ -2,7 +2,7 @@ # vars file for kmodels pullSecretName: "ibm-entitlement" # controller -mas_aibroker_controller_tag: 1.0.1 +mas_aibroker_controller_tag: 1.0.2 controller: "aibroker/controller:{{ mas_aibroker_controller_tag }}" # store mas_aibroker_store_tag: 1.0.1 @@ -27,3 +27,6 @@ mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', t # Kmodels tenant mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" tenantNamespace: "aibroker-{{ mas_aibroker_tenant_name }}" + +# Saas +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" \ No newline at end of file From 86c234e7c784c8cfb8943e88a79c0d0095508da2 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 3 Dec 2024 11:41:41 +0000 Subject: [PATCH 030/169] [patch] fix linter errors --- ibm/mas_devops/roles/aibroker/tasks/saas/main.yml | 3 --- ibm/mas_devops/roles/kmodels/vars/main.yml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml index ea1d7ae732..7c59804bb5 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml @@ -28,11 +28,9 @@ # status_code: [200,409] # timeout: 30 # register: kmodel_api_output - # - name: Kmodel debug #TODO: remove before commit # debug: # msg: "POST output ...... {{ kmodel_api_output }}" - - name: "Check if API key secret exists in namespace: provision-tenant" k8s_info: api_version: v1 @@ -83,4 +81,3 @@ # - name: Debug tenant #TODO: remove on finall commit # debug: # msg: "{{ add_tenant_to_db2 }}" - diff --git a/ibm/mas_devops/roles/kmodels/vars/main.yml b/ibm/mas_devops/roles/kmodels/vars/main.yml index 35998a021b..74ce967770 100644 --- a/ibm/mas_devops/roles/kmodels/vars/main.yml +++ b/ibm/mas_devops/roles/kmodels/vars/main.yml @@ -29,4 +29,4 @@ mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | defaul tenantNamespace: "aibroker-{{ mas_aibroker_tenant_name }}" # Saas -mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" \ No newline at end of file +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" From 54e897f49e2a66017bf23c2e57338682555a4b53 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 3 Dec 2024 12:19:30 +0000 Subject: [PATCH 031/169] [patch] fix trailing spaces --- ibm/mas_devops/roles/aibroker/tasks/saas/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml index 7c59804bb5..c6e12dc92c 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml @@ -29,7 +29,7 @@ # timeout: 30 # register: kmodel_api_output # - name: Kmodel debug #TODO: remove before commit -# debug: +# debug: # msg: "POST output ...... {{ kmodel_api_output }}" - name: "Check if API key secret exists in namespace: provision-tenant" k8s_info: From 416643faeb7b7b51b8076f69ddfeb615e58eb1bd Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 3 Dec 2024 14:04:52 +0000 Subject: [PATCH 032/169] [patch] update aibroker playbook to run tasks related to saas --- .../playbooks/oneclick_add_aibroker.yml | 31 ++++++++++++++++--- .../roles/aibroker/files/create_dro_token.sh | 13 ++------ ibm/mas_devops/roles/aibroker/vars/main.yml | 3 +- .../kmodels-controller/configmap.yml.j2 | 2 +- .../kmodels-watcher/configmap.yml.j2 | 2 +- ibm/mas_devops/roles/kmodels/vars/main.yml | 5 +-- 6 files changed, 35 insertions(+), 21 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 8b57c9e5ea..4d5e649090 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -16,13 +16,16 @@ ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" + # Control is saas release + mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" + # DB2 # ----------------------------------------------------------------------------- db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" pre_tasks: # For the full set of supported environment variables refer to the playbook documentation - - name: Check for required environment variables + - name: Check for required common environment variables assert: that: - lookup('env', 'MAS_ENTITLEMENT_KEY') != "" @@ -44,16 +47,34 @@ - lookup('env', 'MAS_AIBROKER_DB_DATABASE') != "" - lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') != "" - lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') != "" + # - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" + # - lookup('env', 'MAS_AIBROKER_SLS_URL') != "" + # - lookup('env', 'MAS_AIBROKER_DRO_URL') != "" + # - lookup('env', 'MAS_CONFIG_DIR') != "" # db2 support + # - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" # db2 support + fail_msg: "One or more required environment variables are not defined" + + - name: Check for required saas environment variables + assert: + that: - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" - lookup('env', 'MAS_AIBROKER_SLS_URL') != "" - lookup('env', 'MAS_AIBROKER_DRO_URL') != "" - - lookup('env', 'MAS_CONFIG_DIR') != "" # db2 support - - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" # db2 support + - lookup('env', 'MAS_CONFIG_DIR') != "" + - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" fail_msg: "One or more required environment variables are not defined" + when: mas_aibroker_saas == true + + - include_role: + # install db2 + name: ibm.mas_devops.db2 + when: mas_aibroker_saas == true + - include_role: + # config db2 + name: ibm.mas_devops.suite_config + when: mas_aibroker_saas == true roles: - - ibm.mas_devops.db2 # db2 support - - ibm.mas_devops.suite_config # db2 support - ibm.mas_devops.odh - ibm.mas_devops.kmodels - ibm.mas_devops.suite_app_install diff --git a/ibm/mas_devops/roles/aibroker/files/create_dro_token.sh b/ibm/mas_devops/roles/aibroker/files/create_dro_token.sh index 77c493e798..4850440c7c 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_dro_token.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_dro_token.sh @@ -8,16 +8,13 @@ echo "Usage ./create_dro_token.sh tenant_name_in_lower_case instance_id token" - NAMESPACE='aibroker' - TENANT=$1 instance_id=$2 token=$3 -if [ -z ${instance_id}] -then +if [[ -z ${instance_id} ]]; then #echo "❌ Missing instance_id " #exit 1 echo "using default namespace aibroker" @@ -31,8 +28,6 @@ echo $NAMESPACE echo $token - - if [ -z ${TENANT} ]; then echo "using default tenant name=aibroker-user" TENANT='aibroker-user' @@ -40,10 +35,6 @@ if [ -z ${TENANT} ]; then #exit 1 fi - - - - echo "Creating DRO token in k8s secret" oc create secret generic ${TENANT}----dro-secret -n ${NAMESPACE} \ - --from-literal=DRO_TOKEN=${token} + --from-literal=DRO_TOKEN=${token} diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 1dd016cbc1..f344f5c625 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -44,6 +44,7 @@ mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | defa mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" -mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" +# mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 index 338786395c..d03dfbce27 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 @@ -15,7 +15,7 @@ data: PIPELINE_STEP_IMAGE_PREFIX: '' PIPELINE_DEFAULT_STEP_TAG: {{ mas_aibroker_pipeline_steps_tag }} CONNECTOR_DEFAULT_TAG: {{ mas_aibroker_connector_tag }} -{% if mas_aibroker_saas | lower == 'true' %} +{% if mas_aibroker_saas %} DEFAULT_TENANTS: provision-tenant {% else %} DEFAULT_TENANTS: {{ tenantNamespace }} diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 index 4c45d0ae4a..5d8cc93e59 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 @@ -11,7 +11,7 @@ data: WATCHER_SENDER_DELAY: "5" EXCLUDE_JOB_NAMESPACES: "ibm-cpd" PIPELINES: mas-{{ mas_instance_id }}-aibroker -{% if mas_aibroker_saas | lower == 'true' %} +{% if mas_aibroker_saas %} TENANTS: provision-tenant {% else %} TENANTS: {{ tenantNamespace }} diff --git a/ibm/mas_devops/roles/kmodels/vars/main.yml b/ibm/mas_devops/roles/kmodels/vars/main.yml index 74ce967770..64d85b6b5d 100644 --- a/ibm/mas_devops/roles/kmodels/vars/main.yml +++ b/ibm/mas_devops/roles/kmodels/vars/main.yml @@ -2,7 +2,7 @@ # vars file for kmodels pullSecretName: "ibm-entitlement" # controller -mas_aibroker_controller_tag: 1.0.2 +mas_aibroker_controller_tag: 1.0.2-pre.dev10x controller: "aibroker/controller:{{ mas_aibroker_controller_tag }}" # store mas_aibroker_store_tag: 1.0.1 @@ -29,4 +29,5 @@ mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | defaul tenantNamespace: "aibroker-{{ mas_aibroker_tenant_name }}" # Saas -mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" +# mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" From 3e74aadce3319b3d48ca47ef42dd42d95e19a0b3 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 3 Dec 2024 15:06:28 +0000 Subject: [PATCH 033/169] [patch] fix linter issues --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 4d5e649090..bf5633563e 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -16,7 +16,7 @@ ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" - # Control is saas release + # Control is saas release mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" # DB2 @@ -53,7 +53,7 @@ # - lookup('env', 'MAS_CONFIG_DIR') != "" # db2 support # - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" # db2 support fail_msg: "One or more required environment variables are not defined" - + - name: Check for required saas environment variables assert: that: From 1a57c18b6c8566f8c6524a9a7c3837e5e9bbf67a Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 3 Dec 2024 15:46:35 +0000 Subject: [PATCH 034/169] Squashed commit of the following: commit 9206656d7384f91469b03841606d73c06682bd18 Author: David Parker Date: Tue Dec 3 14:36:31 2024 +0000 [major] Use new OLM functions in mas-devops (#1573) commit 3ccdbef982c83989942445ef077d4117d81fec48 Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Mon Dec 2 18:30:04 2024 +0530 [patch] Fixes ibmcloud_api_key issue in COS deprovision for ibmcloud (#1581) Co-authored-by: Harsh Tamakuwala commit 1283c03b65c60cfc0095969abc878264db98724f Author: Terence Quinn Date: Fri Nov 29 03:52:51 2024 -0600 [patch] create unique certificate for each db2 instance (#1579) commit e04138afb8c7ffdfe4be159323d98d404426bb88 Author: David Parker Date: Tue Nov 26 21:40:39 2024 +0000 [patch] get_default_storage_classes should always set failed=False (#1578) commit 7014f49fc63410d44808a7082e8a1ce922e93879 Author: Sanjay Prabhakar Date: Tue Nov 26 16:32:42 2024 +0000 [patch] Add estimates for cpd in readme (#1577) commit 9933bc23fb272ba7fe169ded7a54de71e952b55b Author: David Parker Date: Tue Nov 26 13:46:05 2024 +0000 [patch] Use correct variable for mongo extras (#1576) commit bb720a394502505f5c91849992fc276e0a4cc1c4 Author: David Parker Date: Tue Nov 26 13:32:30 2024 +0000 [patch] Set cpd_product_version from catalog metadata (#1575) commit ba23e83ae87fe889f6bf2fd6bd26da16d1034881 Author: Terence Quinn Date: Mon Nov 25 15:03:03 2024 -0600 [patch] Fix missing catalog_digest (#1574) Co-authored-by: David Parker commit 0f60c6150f2f4541673c22b8312113fe24ce67ac Author: Sanjay Prabhakar Date: Fri Nov 22 17:33:20 2024 +0000 [patch] Add back restore.yml.j2 (#1572) Co-authored-by: Sanjay Prabhakar commit 739c4d03781bca7c983d92e103e62e6542b72fa7 Author: David Parker Date: Fri Nov 22 16:02:24 2024 +0000 [major] Overhaul backup/restore capability (#1557) Co-authored-by: Sanjay Prabhakar commit d5ef4102841b56a9fa7045be01989a3056fed5ef Author: David Parker Date: Fri Nov 22 12:39:48 2024 +0000 [doc] Fix formatting commit 61272c972bc692df31bdd64104234257b1e41ef2 Author: Sanjay Prabhakar Date: Fri Nov 22 12:27:00 2024 +0000 [patch] fix restore for manage role (#1571) Co-authored-by: Sanjay Prabhakar commit 0a13abdc64bf2e458dcb42c9560965a5c107da9f Author: Sanjay Prabhakar Date: Thu Nov 21 10:40:05 2024 +0000 [patch] Db2 backup/restore: fix copy local to pod script (#1569) Co-authored-by: Sanjay Prabhakar commit f4745094e7a119e8fbf5eb2a341e856baaf757e3 Author: David Parker Date: Wed Nov 20 23:43:25 2024 +0000 [major] Move catalog & storage class metadata to python package (#1561) commit df4fe4c55d7a1f1d78723f1452b8a6c3f4dda3a9 Author: Lokesh <110647904+lokesh-sreedhara@users.noreply.github.com> Date: Wed Nov 20 08:49:28 2024 -0500 [patch] remove MIRRORLOGPATH folder creation (#1568) commit 5668f331e38307250c4bdb22e5e5dd2e1f687607 Author: Lokesh <110647904+lokesh-sreedhara@users.noreply.github.com> Date: Wed Nov 20 07:49:43 2024 -0500 [patch] fix conflicting DB config (#1567) commit 5c595c11aacfdb9a3044c257ba9042a08005f311 Author: Sanjay Prabhakar Date: Tue Nov 19 17:45:03 2024 +0000 [patch] can skip db2 in br_manage playbook (#1566) Co-authored-by: Sanjay Prabhakar commit 8c8f9933ce35e16862e3f328ee5fe79349a634ea Author: stonepd Date: Tue Nov 19 16:39:18 2024 +0000 [patch] update CP4D README, remove outdated fix code (#1565) Co-authored-by: Sanjay Prabhakar commit 5eb2cd4aa2ef8f9fcc12626e1f85b577b9219ef3 Author: Sanjay Prabhakar Date: Tue Nov 19 14:42:36 2024 +0000 [patch] fix restore for core and fix tar command in restore tasks (#1564) Co-authored-by: Sanjay Prabhakar commit e81d3a3e0ddd89ef0f7ed1289ff8a7f311599b33 Author: dclain Date: Sat Nov 16 15:39:54 2024 -0600 [patch] Address several issues w/ Nvidia playbook (#1558) commit 1792e98b77b32185c486da66a31c4c89fb11bfcf Author: Tremaine Hart Date: Thu Nov 14 13:19:26 2024 -0600 [patch] Fixes and updates for IBMCloud COS support (#1555) Co-authored-by: David Parker commit 495e26d52efbda11ccb94bbab990a00652e9cc0e Author: Daniel Istrate <16302386+istrate@users.noreply.github.com> Date: Thu Nov 14 13:02:17 2024 +0200 [doc] Document storage class access mode (#1554) Co-authored-by: Daniel Istrate commit 08c22d74515c90a46f19894a630cd2c8155bccef Author: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Date: Tue Nov 12 02:51:17 2024 +0530 [minor] Support November catalog (#1508) Co-authored-by: Parveen Kumar Co-authored-by: Jon Levell Co-authored-by: Jon Levell Co-authored-by: Ian Boden <82514609+IanBoden@users.noreply.github.com> Co-authored-by: Ian Boden commit 35017863e41ff139d567671a507800c49b4f1bc5 Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Thu Nov 7 09:53:24 2024 -0800 [patch] MAXMIS-897: check if attribute 'deployimagestitching' is defined (#1551) Co-authored-by: Brendan More O'Ferrall commit 00c50f15b3ba266f4ec42d0fd44bd81f95f9543c Author: caemar <44871769+caemar@users.noreply.github.com> Date: Thu Nov 7 11:47:38 2024 +0100 [minor] Add Kafka Cluster CA certificate validityDays (#1543) commit ee3a2272b4a3418339eb2c4df77598d56f1f0179 Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Wed Nov 6 11:57:00 2024 -0800 [patch] MAXMIS-886: disable stitching deploy when civil is not enabled2 (#1548) Co-authored-by: Brendan More O'Ferrall commit 3ee251228c962a156692391bceb063c22ebac459 Author: Tremaine Hart Date: Wed Nov 6 11:41:34 2024 -0600 [patch] [MASCORE-4344] Grafana role overwriting critical k8s configuration resource (#1549) commit 482b3868b31468f220da901d5d23458eb47d8afd Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Tue Nov 5 21:51:50 2024 -0800 [minor] #MAXMIS-748: add role to auto configure image stitching (#1530) Co-authored-by: Brendan More O'Ferrall Co-authored-by: Alexandre Quinteiro Co-authored-by: unnati-solanki-git <131964221+unnati-solanki-git@users.noreply.github.com> commit dd13898e98f238b724da63f476d59312c494f139 Author: Tremaine Hart Date: Tue Nov 5 14:45:45 2024 -0600 [patch] [MASCORE-4344] Grafana role overwriting critical k8s configuration resource (#1547) commit ade2a0ea54989b2b2ce55a47890f5d249d13484a Author: Andrew Whitfield Date: Tue Nov 5 17:18:24 2024 +0000 [patch] wait for olm to update operatorcondition (#1545) commit c267990a26a599d405fc78fe248181ab13ad258a Author: Sanjay Prabhakar Date: Tue Nov 5 16:33:07 2024 +0000 [patch] prevent mongo upgrade when featureCompatibilityVersion is not defined or set incorrectly (#1544) Co-authored-by: Sanjay Prabhakar commit f4c04d569cbad2268ad763b344fd297399c5e19a Author: Ian Boden <82514609+IanBoden@users.noreply.github.com> Date: Tue Nov 5 11:21:08 2024 +0000 [patch] Fix gpu-certified-operator default channel (#1542) commit 733b595cebbd00232c415a1d62d456266936ed58 Author: David Parker Date: Mon Nov 4 15:49:12 2024 +0000 [patch] Fix oc exec commands (#1541) commit 224a9a8707befe26612523397a2564a1f31b4071 Author: David Parker Date: Thu Oct 31 18:10:39 2024 +0000 [patch] Reduce useless logging (#1540) --- .github/workflows/docs.yml | 2 +- build/bin/build-collection.sh | 4 +- build/bin/copy-role-docs.sh | 2 +- docs/execution-environment.md | 180 +---- docs/extra.css | 48 -- docs/index.md | 27 +- docs/playbooks/backup-restore.md | 334 ++------- .../backup_restore/before_run_tasks.yml | 18 - .../backup_restore/check_backup_vars.yml | 27 +- .../backup_restore/check_common_vars.yml | 137 +--- .../copy_local_files_to_storage.yml | 77 +-- .../copy_pod_files_to_storage.yml | 364 ++-------- .../copy_storage_files_to_local.yml | 73 +- .../copy_storage_files_to_pod.yml | 470 ++----------- .../create_copy_cloud_files_job.yml | 145 ---- .../backup_restore/create_run_tasks_job.yml | 211 ------ .../delete_storage_job_folder.yml | 46 +- .../list_storage_job_folders.yml | 84 +-- .../rename_storage_job_folder.yml | 59 +- .../backup_restore/send_notification.yml | 35 - .../backup_restore/update_job_status.yml | 22 - .../common_tasks/create_subscription.yml | 141 ---- .../common_tasks/default_storage_classes.yml | 15 + .../common_tasks/get_version_from_channel.yml | 1 + .../backup_restore/run_task_job.yml.j2 | 149 ---- ibm/mas_devops/common_tasks/wait_for_crd.yml | 1 + ibm/mas_devops/common_vars/backup_restore.yml | 21 - .../casebundles/v8-230414-amd64.yml | 72 -- .../casebundles/v8-230518-amd64.yml | 72 -- .../casebundles/v8-230526-amd64.yml | 72 -- .../casebundles/v8-230616-amd64.yml | 72 -- .../casebundles/v8-230627-amd64.yml | 84 --- .../casebundles/v8-230721-amd64.yml | 84 --- .../casebundles/v8-230725-amd64.yml | 88 --- .../casebundles/v8-230829-amd64.yml | 88 --- .../casebundles/v8-230926-amd64.yml | 88 --- .../casebundles/v8-231004-amd64.yml | 98 --- .../casebundles/v8-231031-amd64.yml | 103 --- .../casebundles/v8-231128-amd64.yml | 103 --- .../casebundles/v8-231228-amd64.yml | 103 --- .../casebundles/v8-240130-amd64.yml | 103 --- .../casebundles/v8-240227-amd64.yml | 103 --- .../casebundles/v8-240326-amd64.yml | 103 --- .../casebundles/v8-240405-amd64.yml | 103 --- .../casebundles/v8-240430-amd64.yml | 104 --- .../casebundles/v8-240528-amd64.yml | 103 --- .../casebundles/v9-240625-amd64.yml | 99 --- .../casebundles/v9-240730-amd64.yml | 100 --- .../casebundles/v9-240827-amd64.yml | 99 --- .../casebundles/v9-241003-amd64.yml | 100 --- .../common_vars/default_storage_classes.yml | 23 - ibm/mas_devops/playbooks/br_core.yml | 38 +- ibm/mas_devops/playbooks/br_db2.yml | 33 +- ibm/mas_devops/playbooks/br_health.yml | 72 +- ibm/mas_devops/playbooks/br_iot.yml | 62 +- ibm/mas_devops/playbooks/br_manage.yml | 68 +- ibm/mas_devops/playbooks/br_mongodb.yml | 35 +- ibm/mas_devops/playbooks/br_monitor.yml | 72 +- ibm/mas_devops/playbooks/br_optimizer.yml | 72 +- .../playbooks/br_visualinspection.yml | 49 +- .../playbooks/mirror_add_assist.yml | 34 +- .../playbooks/mirror_add_hputilities.yml | 38 -- ibm/mas_devops/playbooks/mirror_add_iot.yml | 34 +- .../playbooks/mirror_add_manage.yml | 26 +- .../playbooks/mirror_add_monitor.yml | 34 +- .../playbooks/mirror_add_optimizer.yml | 26 +- .../playbooks/mirror_add_predict.yml | 26 +- .../playbooks/mirror_add_visualinspection.yml | 26 +- ibm/mas_devops/playbooks/mirror_core.yml | 26 +- .../playbooks/mirror_dependencies.yml | 245 ++++--- .../plugins/action/apply_subscription.py | 67 ++ .../plugins/action/get_catalog_info.py | 50 ++ .../action/get_default_storage_classes.py | 50 ++ .../plugins/action/get_newest_catalog_tag.py | 25 + .../plugins/action/update_ibm_entitlement.py | 53 ++ ibm/mas_devops/plugins/filter/filters.py | 42 +- ibm/mas_devops/requirements.txt | 1 + .../roles/aibroker/defaults/main.yml | 4 + .../roles/aibroker/tasks/aibroker/main.yml | 19 +- ibm/mas_devops/roles/appconnect/README.md | 1 + .../tasks/determine-storage-classes.yml | 31 +- ibm/mas_devops/roles/arcgis/defaults/main.yml | 4 + ibm/mas_devops/roles/arcgis/tasks/main.yml | 19 +- .../arcgis/templates/operator-group.yml.j2 | 14 - .../arcgis/templates/subscription.yml.j2 | 12 - .../tasks/provider/redhat/install.yml | 8 +- ibm/mas_devops/roles/cos/README.md | 90 ++- ibm/mas_devops/roles/cos/defaults/main.yml | 1 + ibm/mas_devops/roles/cos/tasks/main.yml | 2 +- .../cos/tasks/providers/ibm/deprovision.yml | 12 +- .../cos/tasks/providers/ibm/provision.yml | 22 +- .../roles/cos_bucket/defaults/main.yml | 2 + .../cos_bucket/tasks/providers/ibm/create.yml | 12 +- .../create_cross_region_location_bucket.yml | 4 +- .../ibm/create_region_location_bucket.yml | 4 +- .../ibm/create_service_credentials.yml | 4 +- .../cos_bucket/tasks/providers/ibm/delete.yml | 16 +- ibm/mas_devops/roles/cp4d/README.md | 81 ++- .../cp4d/tasks/determine-ibmcatalog-tag.yml | 71 +- .../cp4d/tasks/determine-storage-classes.yml | 53 +- .../roles/cp4d/tasks/install-cp4d.yml | 9 - .../cp4d/tasks/wait/wait-zenmetastore-edb.yml | 112 --- ibm/mas_devops/roles/cp4d_service/README.md | 638 +++++------------- .../tasks/determine-storage-classes.yml | 50 +- .../cp4d_service/tasks/wait/wait-ccs.yml | 11 +- .../cp4d_service/tasks/wait/wait-couchdb.yml | 108 --- ibm/mas_devops/roles/db2/README.md | 89 +-- .../roles/db2/tasks/backup/main.yml | 21 - .../install/determine-storage-classes.yml | 53 +- .../roles/db2/tasks/install/main.yml | 12 +- .../tasks/restore/copy-db2-backup-file.yml | 2 +- .../roles/db2/tasks/restore/main.yml | 21 - .../upgrade/run-db2-subscription-upgrade.yml | 8 +- .../db2/templates/certs/certificate.yml.j2 | 4 +- .../roles/db2/templates/db2ucluster.yml.j2 | 2 +- ibm/mas_devops/roles/dro/README.md | 3 +- .../install-dro/determine-storage-classes.yml | 22 +- ibm/mas_devops/roles/grafana/README.md | 2 +- .../install/determine-storage-classes.yml | 15 +- .../roles/grafana/tasks/install/main.yml | 33 +- .../tasks/update/determine-storage-config.yml | 28 +- .../cluster-monitoring-config.yml.j2 | 3 +- .../roles/ibm_catalogs/tasks/install/main.yml | 8 +- .../templates/offline-catalog.yml.j2 | 2 +- .../roles/install_operator/README.md | 30 - .../roles/install_operator/defaults/main.yml | 14 - .../roles/install_operator/tasks/main.yml | 91 --- .../templates/custom_labels.json.j2 | 7 - .../ibm-entitlement-with-artifactory.json.j2 | 32 - ibm/mas_devops/roles/kafka/README.md | 4 +- .../kafka/tasks/determine-storage-classes.yml | 26 +- .../templates/redhat/clusters/large.yml.j2 | 5 + .../templates/redhat/clusters/small.yml.j2 | 5 + .../tasks/determine-storage-classes.yml | 22 +- .../vars/amlen_1.1.2.yml | 16 + .../templates/imagesetconfiguration.yml.j2 | 4 +- ibm/mas_devops/roles/mongodb/README.md | 77 +-- .../tasks/determine-ibmcatalog-tag.yml | 106 ++- .../after-backup-restore.yml | 0 .../{ => backup-restore}/backup-database.yml | 7 +- .../before-backup-restore.yml | 5 +- .../{ => backup-restore}/create-role-user.yml | 8 +- .../{ => backup-restore}/get-mongo-info.yml | 0 .../restore-database-patch.yml | 0 .../restore-database-perform.yml | 3 +- .../{ => backup-restore}/restore-database.yml | 8 +- .../tasks/providers/community/backup.yml | 32 +- .../community/determine-storage-classes.yml | 18 +- .../providers/community/install-mongo.yml | 99 ++- .../tasks/providers/community/restore.yml | 32 +- .../providers/community/validate-upgrade.yml | 10 + ibm/mas_devops/roles/nvidia_gpu/README.md | 21 +- .../roles/nvidia_gpu/defaults/main.yml | 7 +- .../roles/nvidia_gpu/tasks/main.yml | 149 ++-- .../roles/nvidia_gpu/tasks/nfd_setup.yml | 128 ++-- ....j2 => clusterpolicy-customversion.yml.j2} | 53 +- .../templates/clusterpolicy-v2.yml.j2 | 33 +- .../roles/ocp_cluster_monitoring/README.md | 7 +- .../tasks/determine-storage-classes.yml | 19 +- .../README.md | 1 + .../roles/ocp_verify/tasks/main.yml | 73 -- .../roles/odh/tasks/authorino-operator.yml | 18 +- .../odh/tasks/determine-storage-classes.yml | 24 +- .../roles/odh/tasks/odh-operator.yml | 12 +- .../roles/odh/tasks/pipeline-operator.yml | 15 +- .../roles/odh/tasks/serverless-operator.yml | 16 +- .../roles/odh/tasks/servicemesh-operator.yml | 18 +- .../templates/authorino/operator-group.yml.j2 | 7 - .../templates/authorino/subscription.yml.j2 | 12 - .../odh/templates/odh/operator-group.yml.j2 | 7 - .../odh/templates/odh/subscription.yml.j2 | 13 - .../templates/pipelines/operator-group.yml.j2 | 7 - .../templates/pipelines/subscription.yml.j2 | 12 - .../serverless/operator-group.yml.j2 | 7 - .../templates/serverless/subscription.yml.j2 | 12 - .../servicemesh/operator-group.yml.j2 | 7 - .../templates/servicemesh/subscription.yml.j2 | 12 - .../roles/opentelemetry/tasks/install.yml | 8 +- ibm/mas_devops/roles/registry/README.md | 4 +- .../tasks/setup/determine-storage-classes.yml | 27 +- ibm/mas_devops/roles/sls/defaults/main.yml | 4 + .../roles/sls/tasks/install/main.yml | 22 +- .../roles/sls/templates/operator-group.yml.j2 | 9 - .../roles/sls/templates/subscription.yml.j2 | 12 - .../roles/suite_app_backup_restore/README.md | 75 +- .../tasks/get-app-info.yml | 7 +- .../suite_app_backup_restore/tasks/main.yml | 19 - .../tasks/manage/pv-info.yml | 4 +- .../tasks/manage/restore-namespace.yml | 33 +- .../tasks/restore-namespace.yml | 2 +- .../roles/suite_app_config/README.md | 5 +- .../tasks/determine-storage-classes.yml | 61 +- .../tasks/manage/post-config/main.yml | 7 + .../roles/suite_app_config/vars/health.yml | 6 - .../roles/suite_app_config/vars/manage.yml | 6 - .../roles/suite_app_install/README.md | 22 +- .../roles/suite_app_install/defaults/main.yml | 4 + .../roles/suite_app_install/tasks/iot.yml | 40 +- .../roles/suite_app_install/tasks/main.yml | 19 +- .../tasks/visualinspection.yml | 20 +- .../templates/operator-group.yml.j2 | 14 - .../templates/subscription.yml.j2 | 17 - .../roles/suite_backup_restore/README.md | 73 +- .../roles/suite_backup_restore/tasks/main.yml | 15 - .../tasks/restore-namespace.yml | 17 +- .../tasks/apply-db2-config-version.yml | 9 - .../suite_db2_setup_for_manage/vars/main.yml | 4 +- .../roles/suite_install/defaults/main.yml | 4 + .../roles/suite_install/tasks/main.yml | 19 +- .../templates/operator-group.yml.j2 | 15 - .../templates/subscription.yml.j2 | 18 - .../suite_manage_attachments_config/README.md | 2 +- .../README.md | 82 +++ .../defaults/main.yml | 16 + .../meta/main.yml | 5 +- .../tasks/configure-stitching.yml | 195 ++++++ .../tasks/main.yml | 21 + .../tasks/manage-set-system-property.yml | 56 ++ .../tasks/set-imagestitching-properties.yml | 53 ++ .../templates/imagestitching-pvc.yml.j2 | 15 + .../roles/suite_manage_pvc_config/README.md | 1 + .../tasks/determine-storage-classes.yml | 18 +- .../roles/suite_upgrade/tasks/upgrade.yml | 6 + ibm/mas_devops/roles/uds/README.md | 2 +- .../install/determine-storage-classes.yml | 20 +- mkdocs.yml | 63 +- 226 files changed, 2583 insertions(+), 7850 deletions(-) delete mode 100644 docs/extra.css delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/create_copy_cloud_files_job.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/create_run_tasks_job.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/send_notification.yml delete mode 100644 ibm/mas_devops/common_tasks/create_subscription.yml create mode 100644 ibm/mas_devops/common_tasks/default_storage_classes.yml delete mode 100644 ibm/mas_devops/common_tasks/templates/backup_restore/run_task_job.yml.j2 delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230414-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230518-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230526-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230616-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230627-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230721-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230725-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230829-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-230926-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-231004-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-231031-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-231128-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-231228-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-240130-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-240227-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-240326-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-240405-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-240430-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v8-240528-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v9-240625-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v9-240730-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v9-240827-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/casebundles/v9-241003-amd64.yml delete mode 100644 ibm/mas_devops/common_vars/default_storage_classes.yml delete mode 100644 ibm/mas_devops/playbooks/mirror_add_hputilities.yml create mode 100644 ibm/mas_devops/plugins/action/apply_subscription.py create mode 100644 ibm/mas_devops/plugins/action/get_catalog_info.py create mode 100644 ibm/mas_devops/plugins/action/get_default_storage_classes.py create mode 100644 ibm/mas_devops/plugins/action/get_newest_catalog_tag.py create mode 100644 ibm/mas_devops/plugins/action/update_ibm_entitlement.py create mode 100644 ibm/mas_devops/requirements.txt delete mode 100644 ibm/mas_devops/roles/arcgis/templates/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/arcgis/templates/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml delete mode 100644 ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml delete mode 100644 ibm/mas_devops/roles/install_operator/README.md delete mode 100644 ibm/mas_devops/roles/install_operator/defaults/main.yml delete mode 100644 ibm/mas_devops/roles/install_operator/tasks/main.yml delete mode 100644 ibm/mas_devops/roles/install_operator/templates/custom_labels.json.j2 delete mode 100644 ibm/mas_devops/roles/install_operator/templates/ibm-entitlement-with-artifactory.json.j2 create mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/amlen_1.1.2.yml rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{ => backup-restore}/after-backup-restore.yml (100%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{ => backup-restore}/backup-database.yml (98%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{ => backup-restore}/before-backup-restore.yml (98%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{ => backup-restore}/create-role-user.yml (95%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{ => backup-restore}/get-mongo-info.yml (100%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{ => backup-restore}/restore-database-patch.yml (100%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{ => backup-restore}/restore-database-perform.yml (98%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{ => backup-restore}/restore-database.yml (95%) rename ibm/mas_devops/roles/nvidia_gpu/templates/{clusterpolicy.yml.j2 => clusterpolicy-customversion.yml.j2} (56%) delete mode 100644 ibm/mas_devops/roles/odh/templates/authorino/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/authorino/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/odh/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/odh/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/pipelines/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/pipelines/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/serverless/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/serverless/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/servicemesh/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/odh/templates/servicemesh/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/sls/templates/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/sls/templates/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/suite_app_install/templates/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/suite_app_install/templates/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/suite_install/templates/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/suite_install/templates/subscription.yml.j2 create mode 100644 ibm/mas_devops/roles/suite_manage_imagestitching_config/README.md create mode 100644 ibm/mas_devops/roles/suite_manage_imagestitching_config/defaults/main.yml rename ibm/mas_devops/roles/{install_operator => suite_manage_imagestitching_config}/meta/main.yml (58%) create mode 100644 ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/configure-stitching.yml create mode 100644 ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/main.yml create mode 100644 ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/manage-set-system-property.yml create mode 100644 ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/set-imagestitching-properties.yml create mode 100644 ibm/mas_devops/roles/suite_manage_imagestitching_config/templates/imagestitching-pvc.yml.j2 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c07856617b..338e59c302 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -15,7 +15,7 @@ jobs: - name: Install and Build run: | bash build/bin/copy-role-docs.sh - python -m pip install -q mkdocs + python -m pip install -q mkdocs mkdocs-carbon mkdocs-glightbox mkdocs build --verbose --clean --strict - name: Deploy diff --git a/build/bin/build-collection.sh b/build/bin/build-collection.sh index 769d8f6185..b09fadd02e 100644 --- a/build/bin/build-collection.sh +++ b/build/bin/build-collection.sh @@ -13,8 +13,8 @@ cat $GITHUB_WORKSPACE/ibm/mas_devops/galaxy.yml # Update this when we have new catalog -MAS_PREVIOUS_CATALOG='v9-240827-amd64' -MAS_LATEST_CATALOG='v9-241003-amd64' +MAS_PREVIOUS_CATALOG='v9-241003-amd64' +MAS_LATEST_CATALOG='v9-241107-amd64' # Update all the placeholders in the playbooks diff --git a/build/bin/copy-role-docs.sh b/build/bin/copy-role-docs.sh index 8bc33ad3a4..018ae96846 100644 --- a/build/bin/copy-role-docs.sh +++ b/build/bin/copy-role-docs.sh @@ -40,7 +40,6 @@ copyDoc gencfg_workspace copyDoc grafana copyDoc ibm_catalogs copyDoc ibmcloud_resource_key -copyDoc install_operator copyDoc kafka copyDoc key_rotation copyDoc kmodels @@ -86,6 +85,7 @@ copyDoc suite_manage_bim_config copyDoc suite_manage_birt_report_config copyDoc suite_manage_customer_files_config copyDoc suite_manage_attachments_config +copyDoc suite_manage_imagestitching_config copyDoc suite_manage_import_certs_config copyDoc suite_manage_load_dbc_scripts copyDoc suite_manage_logging_config diff --git a/docs/execution-environment.md b/docs/execution-environment.md index ea29a198c7..8e4db2e24d 100644 --- a/docs/execution-environment.md +++ b/docs/execution-environment.md @@ -1,6 +1,6 @@ Execution Environment =============================================================================== -Details on the Red Hat Ansible Automation Platform Execution Environment for ansible-devops. +Details on the [Red Hat Ansible Automation Platform](https://www.redhat.com/en/technologies/management/ansible) Execution Environment for the **ibm.mas_devops** Ansible Collection. Execution Environment Image @@ -228,7 +228,7 @@ If you need to set an environment variable then you can do this in the playbook ocp_version: 4.14.35 rosa_compute_nodes: 3 - environment: + environment: AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}" AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}" @@ -245,148 +245,6 @@ If you need to set an environment variable then you can do this in the playbook - ibm.mas_devops.ocp_efs ``` -### backup/restore -The backup and restore tasks and playbooks provided in the `ibm.mas_devops` collection has a number of differences with other roles/playbooks that can make creating the playbook difficult. The following playbook is an example of how a backup for core playbook can be written to work with AAP, along with a rclone.conf.j2 file that should be stored in the same location as the playbook in your SCM. The playbook needs to set both the `environment` so that the `ansible_env` var is set, and for task vars to be set to override the common_vars that are looking for environment variables on the local controller (note that `environment` does not update the local controller environment so lookups using the ansible builtin `env` won't pick up those values). - -This playbook requires the following survey questions to be setup: - -```yaml -{ - "aws_access_key_id": "$encrypted$", - "aws_secret_access_key": "$encrypted$", - "ocp_token": "$encrypted$", - "ocp_server": "https://api.your_ocp_cluster:6443" -} -``` - -```yaml ---- -- name: "Backup/Restore MAS Core" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: aap1 - - # Define what action to perform - masbr_action: backup - - # Define storage type - masbr_storage_type: cloud - masbr_storage_cloud_rclone_file: rclone.conf - masbr_storage_cloud_rclone_name: masbr - masbr_storage_cloud_bucket: mas-backup - - # Define what to backup/restore - masbr_job_component: - name: "core" - instance: "{{ mas_instance_id }}" - namespace: "mas-{{ mas_instance_id }}-core" - - # Configure path to backup_restore tasks that should be present in the execution environment - role_path: "/usr/share/ansible/collections/ansible_collections/ibm/mas_devops/roles/suite_backup_restore" - - # Set here to get into ansible_env - environment: - MASBR_STORAGE_TYPE: cloud - MASBR_STORAGE_CLOUD_RCLONE_FILE: rclone.conf - MASBR_STORAGE_CLOUD_RCLONE_NAME: masbr - MASBR_STORAGE_CLOUD_BUCKET: mas-backup - - pre_tasks: - - name: "Asserts aws_access_key_id secret defined" - assert: - that: aws_access_key_id is defined - fail_msg: "aws_access_key_id not defined" - - - name: "Asserts aws_secret_access_key secret defined" - assert: - that: aws_secret_access_key is defined - fail_msg: "aws_secret_access_key not defined" - - # Template out the rclone.conf so no credentials are stored in git - - name: "Create rclone.conf" - template: - src: "{{ masbr_storage_cloud_rclone_file }}.j2" - dest: "{{ masbr_storage_cloud_rclone_file }}" - - roles: - - ibm.mas_devops.ocp_login - - tasks: - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: - file: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - masbr_storage_type: cloud - masbr_storage_cloud_rclone_file: rclone.conf - masbr_storage_cloud_rclone_name: masbr - masbr_storage_cloud_bucket: mas-backup - - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: - file: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" - vars: - _rt_playbook_name: "br_core" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - masbr_storage_type: cloud - masbr_storage_cloud_rclone_file: rclone.conf - masbr_storage_cloud_rclone_name: masbr - masbr_storage_cloud_bucket: mas-backup - - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "core" - masbr_storage_type: cloud - masbr_storage_cloud_rclone_file: rclone.conf - masbr_storage_cloud_rclone_name: masbr - masbr_storage_cloud_bucket: mas-backup - - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore - vars: - masbr_storage_type: cloud - masbr_storage_cloud_rclone_file: rclone.conf - masbr_storage_cloud_rclone_name: masbr - masbr_storage_cloud_bucket: mas-backup -``` - -The following file is stored in your SCM and is a template to be injected with the credentials needed during the play. This allows the file to be stored in SCM without exposing credentials. - -rclone.conf.j2 -``` -[masbr] -type = s3 -provider = Minio -endpoint = http://minio-api.apps.mydomain.com -access_key_id = "{{ aws_access_key_id }}" -secret_access_key = "{{ aws_secret_access_key }}" -region = minio -``` - Troubleshooting ------------------------------------------------------------------------------- @@ -459,37 +317,3 @@ Example setting role vars: vars: kafka_version: 3.7.0 ``` - -One caveat to the above is the `backup/restore` tasks as these use a combination of ansible vars and expecting environment variables to be set in the `ansible_env` variable. In order to work with `the backup/restore` tasks you need to set both `environment` and role/task vars (not play vars). - -Example, showing `environment` and task vars set. See [backup exmaple](#examples-of-playbooks) for more complete play: -```yaml -- name: "Backup/Restore MAS Core" - hosts: localhost - any_errors_fatal: true - - vars: - # Configure path to backup_restore tasks - role_path: "/usr/share/ansible/collections/ansible_collections/ibm/mas_devops/roles/suite_backup_restore" - - # Set here to get into ansible_env - environment: - MASBR_STORAGE_TYPE: cloud - MASBR_STORAGE_CLOUD_RCLONE_FILE: rclone.conf - MASBR_STORAGE_CLOUD_RCLONE_NAME: masbr - MASBR_STORAGE_CLOUD_BUCKET: mas-backup - - tasks: - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: - file: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - masbr_storage_type: cloud - masbr_storage_cloud_rclone_file: rclone.conf - masbr_storage_cloud_rclone_name: masbr - masbr_storage_cloud_bucket: mas-backup -``` \ No newline at end of file diff --git a/docs/extra.css b/docs/extra.css deleted file mode 100644 index fedf8fb6cc..0000000000 --- a/docs/extra.css +++ /dev/null @@ -1,48 +0,0 @@ -table { - margin-bottom: 2em -} - -table tr th { - padding: 5px 10px; - background-color: #ddd; - border:1px solid #aaa !important -} - -table tr td { - padding: 5px 10px; - border:1px solid #aaa !important -} - -table tr:nth-child(even) { - background-color: #eee; -} - -/* If an unordered list immediately follows a normal paragraph, set negative - margin so that it flows better */ -div[role=main] p + ul { - margin-top: -20px -} - -/* Ensure we keep the margin if the previous paragraph is an admonition title */ -div[role=main] p.admonition-title + ul { - margin-top: 0 -} - -img { - display: block; - margin-left: auto; - margin-right: auto; -} - -.wy-nav-content { - max-width: 1200px !important; -} - -.wy-table-responsive table td, .wy-table-responsive table th { - white-space: normal !important; -} - -h2 { - margin-top: 2em; - border-bottom: 2px solid black -} \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 5a53c1ab7e..d38113e392 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,26 +1,18 @@ -MAS Devops Ansible Collection +MAS DevOps Ansible Collection =============================================================================== - -MAS Compatability -------------------------------------------------------------------------------- -The collection supports IBM Maximo Application Suite version 8.10 and above. - - -Releases -------------------------------------------------------------------------------- -[https://github.com/ibm-mas/ansible-devops/releases](https://github.com/ibm-mas/ansible-devops/releases) +The **ibm.mas_devops** Ansible Collection is published on [Ansible Galaxy](https://galaxy.ansible.com/ui/repo/published/ibm/mas_devops/) and works with all supported releases of IBM Maximo Application Suite. Release information for the collection can be found in [GitHub](https://github.com/ibm-mas/ansible-devops/releases). Usage ------------------------------------------------------------------------------- -### Run a playbook +### Run a Playbook The collection includes a number of playbooks that string together multiple roles, you can directly invoke them after installing the collection: ```bash ansible-playbook ibm.mas_devops.lite_core_roks ``` -### Run a role +### Run a Role If you only want to perform a single action, you can directly invoke one of our roles from the command line without the need to build a playbook: ```bash @@ -75,7 +67,6 @@ curl -sL https://raw.githubusercontent.com/IBM-Cloud/ibm-cloud-developer-tools/m ibmcloud version` ``` - ### Install the Ansible Collection Install the collection direct from [Ansible Galaxy](https://galaxy.ansible.com/ibm/mas_devops) @@ -90,7 +81,7 @@ ansible-galaxy collection install ibm.mas_devops:18.10.4 Ansible Automation Platform ------------------------------------------------------------------------------- -If you wish to use Ansible Automation Platform then a Automation Execution Environment image is available at [quay.io/ibmmas/ansible-devops-ee](https://quay.io/repository/ibmmas/ansible-devops-ee?tab=tags&tag=latest) that contains the `ibm.mas_devops` collection at the same release level, plus required client packages and access to the automation content collections supported by Red Hat. +If you wish to use [Red Hat Ansible Automation Platform](https://www.redhat.com/en/technologies/management/ansible) then a Automation Execution Environment image is available at [quay.io/ibmmas/ansible-devops-ee](https://quay.io/repository/ibmmas/ansible-devops-ee?tab=tags&tag=latest) that contains the `ibm.mas_devops` collection at the same release level, plus required client packages and access to the automation content collections supported by Red Hat. More details on how to use the ansible-devops execution environment can be found [here](execution-environment.md) @@ -98,11 +89,3 @@ More details on how to use the ansible-devops execution environment can be found Support ------------------------------------------------------------------------------- This Ansible collection is developed by the IBM Maximo Application Suite development team, customers may raise support tickets via the same routes they would an issue with the product itself, or [raise an issue directly in the GitHub repository](https://github.com/ibm-mas/ansible-devops/issues). - - - diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index e254780a2d..d5d6a820ee 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -16,113 +16,37 @@ MAS Devops Collection includes playbooks for backing up and restoring of the fol - [Visual Inspection](#backuprestore-for-visual-inspection) -Creation of both **full** and **incremental** backups are supported. Backup and restore actions can be executed locally, or by generating **on-demand** or **scheduled** jobs that will allow the work to be performed on your OpenShift cluster using the [MAS CLI container image](https://github.ibm.com/ibm-mas/cli). - -!!! tip - The backup and restore Ansible roles can also be used individually, allowing you to build your own customized backup and restore playbook covering exactly what you need. For example, you can only [backup/restore Manage attachments](../roles/suite_app_backup_restore.md). - +Creation of both **full** and **incremental** backups are supported. The backup and restore Ansible roles can also be used individually, allowing you to build your own customized backup and restore playbook covering exactly what you need. For example, you can only [backup/restore Manage attachments](../roles/suite_app_backup_restore.md). For more information about backup and restore for Maximo Application Suite, please refer to [Backing up and restoring Maximo Application Suite](https://www.ibm.com/docs/en/mas-cd/continuous-delivery?topic=administering-backing-up-restoring-maximo-application-suite) in the product documentation. -Configuration +Configuration - Storage ------------------------------------------------------------------------------- +You can save the backup files to a folder on your local file system by setting the following environment variables: -### Storage - -| Envrionment variable | Required (Default Value) | Description | -| ------------------------------------ | -------------------------------------- | ----------- | -| MASBR_STORAGE_TYPE | **Yes** | Type of storage system for saving the backup files | -| MASBR_STORAGE_LOCAL_FOLDER | **Yes**, if `MASBR_STORAGE_TYPE=local` | The local path to save the backup files | -| MASBR_STORAGE_CLOUD_RCLONE_FILE | **Yes**, if `MASBR_STORAGE_TYPE=cloud` | The path of `rclone.conf` file | -| MASBR_STORAGE_CLOUD_RCLONE_NAME | **Yes** if `MASBR_STORAGE_TYPE=cloud` | The configuration name defined in `rclone.conf` file | -| MASBR_STORAGE_CLOUD_BUCKET | **Yes**, if `MASBR_STORAGE_TYPE=cloud` | Object storage bucket for saving backup files | -| MASBR_LOCAL_TEMP_FOLDER | No (`/tmp/masbr`) | Local folder for saving the temporary backup/restore data, the data in this folder will be deleted after the backup/restore job completed. | - -You need to set the environment variable `MASBR_STORAGE_TYPE` before you perform a backup or restore job. This variable indicates what type of storage systems that you are using for saving the backup files. Currently, it supports below types: - -- `local`: use the local file system, e.g. a folder on your laptop or workstation. -- `cloud`: use the cloud object storage, such as IBM Cloud Object Storage, AWS S3, etc. - -###### Use Local Folder - -You can save the backup files to a folder on your local file system by setting the following environment variables: -``` -MASBR_STORAGE_TYPE=local -MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -``` -- `MASBR_STORAGE_LOCAL_FOLDER`: the path for saving the backup files - -###### Use Cloud Object Storage - -The backup playbooks use [Rclone](https://rclone.org/) to copy backup files from data stores to cloud object storage. It requires a Rclone configuration file which you can either create it manually, or you can install the Rclone tool and create the configuration file by running the `rclone config` command. For more information about the rclone config command and configuration file format, please refer to the [Rclone documentation](https://rclone.org/s3/#configuration). - -Below is a sample Rclone configuration file that using MinIO object storage: -``` -[masbr] -type = s3 -provider = Minio -endpoint = http://minio-api.apps.mydomain.com -access_key_id = Qfx9YGnykJapxL7pzUyA -secret_access_key = qKRGSnxsJ7z6pIA74sVxJ6fkEh4Fq5m4fo0inDuJ -region = minio -``` - -Set the following environment variables to indicate the playbooks to use cloud object storage for saving backup files: -``` -MASBR_STORAGE_TYPE=cloud -MASBR_STORAGE_CLOUD_RCLONE_FILE=/mnt/configmap/rclone.conf -MASBR_STORAGE_CLOUD_RCLONE_NAME=masbr -MASBR_STORAGE_CLOUD_BUCKET=mas-backup -``` - -- `MASBR_STORAGE_CLOUD_RCLONE_FILE`: the path where your rclone.conf file is located -- `MASBR_STORAGE_CLOUD_RCLONE_NAME`: the Rclone configuration name (`[masbr]` from above sample) defined in the rclone.conf file -- `MASBR_STORAGE_CLOUD_BUCKET`: the bucket name you created on the object storage for saving the backup files - - -### Kubernetes Jobs -| Envrionment variable | Required (Default Value) | Description | -| ------------------------------------ | ------------------------ | ----------- | -| MASBR_CONFIRM_CLUSTER | No (`false`) | Set `true` or `false` to indicate the playbook whether to confirm the currently connected cluster before running the backup or restore job | -| MASBR_CREATE_TASK_JOB | No (`true`) | Whether to run backup/restore process in kubernetes Job | -| MASBR_COPY_TIMEOUT_SEC | No (`43200`) | The transfer files timeout in seconds, default timeout value is 12 hours. | -| MASBR_JOB_TIMEZONE | No | The [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) for creating scheduled job. If not set a value for this variable, this role will use UTC time zone when creating a CronJob for running scheduled job. | -| MASBR_CLEANUP_SCHEDULE | No (`0 1 * * *`) | Cron expression of cleanup Job (default to run at 1:00 every day) | -| MASBR_CLEANUP_TTL_SEC | No (`604800`) | All completed jobs that exceed this TTL(time-to-live) in seconds will be deleted (default TTL is 1 week: 3600 * 24 * 7) | -| MASBR_MASCLI_IMAGE_TAG | No (`latest`) | MAS CLI docker image tag | -| MASBR_MASCLI_IMAGE_PULL_POLICY | No | MAS CLI docker [image pull policy](https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting) | - -When the playbook starts running, it will first perform some checks, such as checking the required environment variables, get the source or target cluster information, etc. Then, depending on the value of `MASBR_CREATE_TASK_JOB`, the remaining backup/restore process can be ran in different ways: - -- `MASBR_CREATE_TASK_JOB=false`, run backup/restore process in your current terminal, and you can view the terminal output to get the progress of the backup/restore. -- `MASBR_CREATE_TASK_JOB=true`, a new kubernetes Job will be created to run the backup/restore process in the cluster, then you can check the log of the created kubernetes Job in the cluster to monitor the backup/restore progress. - -The environment variable `MASBR_CREATE_TASK_JOB` is only valid when using cloud object storage (`MASBR_STORAGE_TYPE=cloud`). The playbooks will always run backup/restore process in your local terminal when using local storage system (`MASBR_STORAGE_TYPE=local`). +| Envrionment variable | Required (Default Value) | Description | +| ------------------------------------ | -------------------------- | ----------- | +| MASBR_STORAGE_LOCAL_FOLDER | **Yes** | The local path to save the backup files | +| MASBR_LOCAL_TEMP_FOLDER | No (`/tmp/masbr`) | Local folder for saving the temporary backup/restore data, the data in this folder will be deleted after the backup/restore job completed. | -During the backup/restore process, the playbook will copy backup files between different data stores and specified backup storage systems. Set a suitable value for the environment variable `MASBR_COPY_TIMEOUT_SEC` to avoid the playbook entering a waiting state due to some errors, such as the specified storage system network speed is too slow or cannot be connected. - -!!! warning - Set a suitable value for `MASBR_COPY_TIMEOUT_SEC` based on the estimated size of the backup/restore data and the network conditions, setting it too low can result in the data copying process being interrupted. - -The playbook will create an additional CronJob `masbr-cleanup` for each namespace that has backup/restore jobs created. This cleanup CronJob will periodically delete the completed jobs that exceed a certain priod of time which specified by `MASBR_CLEANUP_TTL_SEC`. You can also specify when to run the cleanup CronJob by setting Cron expression for `MASBR_CLEANUP_SCHEDULE`. +Configuration - Backup +------------------------------------------------------------------------------- -### Backups | Envrionment variable | Required (Default Value) | Description | | ------------------------------------ | ------------------------ | ----------- | | MASBR_ACTION | **Yes** | Whether to run the playbook to perform a `backup` or a `restore` | | MASBR_BACKUP_TYPE | No (`full`) | Set `full` or `incr` to indicate the playbook to create a **full** backup or **incremental** backup. | | MASBR_BACKUP_FROM_VERSION | No | Set the full backup version to use in the incremental backup, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`). | -| MASBR_BACKUP_SCHEDULE | No | Set [Cron expression](ttps://en.wikipedia.org/wiki/Cron) to create a scheduled backup. If not set a value for this varialbe, the playbook will create an on-demand backup. | The playbooks are switched to backup mode by setting `MASBR_ACTION` to `backup`. -###### Full Backups -If you set environment variable `MASBR_BACKUP_TYPE=full` or do not specify a value for this variable, the playbook will take a full backup. +### Full Backups +If you set environment variable `MASBR_BACKUP_TYPE=full` or do not specify a value for this variable, the playbook will take a full backup. -###### Incremental Backups -You can set environment variable `MASBR_BACKUP_TYPE=incr` to indicate the playbook to take an incremental backup. +### Incremental Backups +You can set environment variable `MASBR_BACKUP_TYPE=incr` to indicate the playbook to take an incremental backup. !!! important Only supports creating incremental backup for MonogDB, Db2 and persistent volume data. The playbook will always create a full backup for other type of data regardless of whether this variable be set to `incr`. @@ -132,30 +56,10 @@ The environment variable `MASBR_BACKUP_FROM_VERSION` is only valid if `MASBR_BAC !!! important The backup files you specified by `MASBR_BACKUP_FROM_VERSION` must be a Full backup. And the component name and data types in the specified Full backup file must be same as the current incremental backup job. -###### Scheduled Backups -In addition to create an on-demand backup job, you can also set environment variable `MASBR_BACKUP_SCHEDULE` to indicate the playbook to create a kubernetes CronJob to run the backup process periodically. - -The value of `MASBR_BACKUP_SCHEDULE` is a [Cron expression](https://en.wikipedia.org/wiki/Cron): - -``` - ┌───────────── minute (0–59) - │ ┌───────────── hour (0–23) - │ │ ┌───────────── day of the month (1–31) - │ │ │ ┌───────────── month (1–12) - │ │ │ │ ┌───────────── day of the week (0–6) (Sunday to Saturday; - │ │ │ │ │ 7 is also Sunday on some systems) - │ │ │ │ │ - │ │ │ │ │ - * * * * * -``` - -For example, set below value to create a scheduled backup job that will run at 1:00 a.m. from Monday to Friday: -`MASBR_BACKUP_SCHEDULE="0 1 * * 1-5"` - -By default, the kubernetes CronJob use UTC time zone, so maybe you want to set environment variable `MASBR_JOB_TIMEZONE` with the Cron expression based on your local [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) +Configuration - Restore +------------------------------------------------------------------------------- -### Restore | Envrionment variable | Required (Default Value) | Description | | ------------------------------------ | ------------------------ | ----------- | | MASBR_ACTION | **Yes** | Whether to run the playbook to perform a `backup` or a `restore` | @@ -166,22 +70,6 @@ The playbooks are switched to restore mode by setting `MASBR_ACTION` to `restore In the case of restoring from an incremental backup, the corresponding full backup will be restored first before continuing to restore the incremental backup. -### Slack Notifications -| Envrionment variable | Required (Default Value) | Description | -| ------------------------------------ | ---------------------------------- | ----------- | -| MASBR_SLACK_ENABLED | No (`false`) | Set `true` or `false` to indicate whether the playbook will send Slack notification messages of the backup and restore progress | -| MASBR_SLACK_LEVEL | No (`info`) | Set `failure`, `info` or `verbose` to indicate the playbook to send Slack notification messages in which backup and resore phases | -| MASBR_SLACK_TOKEN | Yes, if `MASBR_SLACK_ENABLED=true` | Slack integration token | -| MASBR_SLACK_CHANNEL | Yes, if `MASBR_SLACK_ENABLED=true` | Channel to send the message to | -| MASBR_SLACK_USER | No (`MASBR`) | Sender of the message | - -Integration with Slack is supported with below notification levels: - -- `verbose`: send notifications when job in the phase `InProgress`, `Completed`, `Failed` or `PartiallyFailed`. -- `info`: send notifications when job in the phase `Completed`, `Failed` or `PartiallyFailed`. -- `failure`: send notifications only when job in the phase `Failed` or `PartiallyFailed` - - Backup/Restore for MongoDB ------------------------------------------------------------------------------- This playbook `ibm.mas_devops.br_mongodb` will invoke the role [mongodb](../roles/mongodb.md) to backup/restore the MongoDB databases. @@ -189,7 +77,7 @@ This playbook `ibm.mas_devops.br_mongodb` will invoke the role [mongodb](../role This playbook supports backing up and restoring databases for an in-cluster MongoDB CE instance. If you are using other MongoDB venders, such as IBM Cloud Databases for MongoDB, Amazon DocumentDB or MongoDB Altas Database, please refer to the corresponding vender's documentation for more information about their provided backup/restore service. ### Environment Variables - +- `MONGODB_NAMESPACE`: By default the backup and restore processes will use a namespace of `mongoce`, if you have customized the install of MongoDb CE you must set this environment variable to the appropriate namespace you wish to backup from/restore to. - `MAS_INSTANCE_ID`: **Required**. This playbook supports backup/restore MongoDB databases that belong to a specific MAS instance, call the playbook multiple times with different values for `MAS_INSTANCE_ID` if you wish to back up multiple MAS instances that use the same MongoDB CE instance. - `MAS_APP_ID`: **Optional**. By default, this playbook will backup all databases belonging to the specified MAS instance. You can backup the databases only belong to a specific MAS application by setting this environment variable to a supported MAS application id `core`, `manage`, `iot`, `monitor`, `health`, `optimizer` or `visualinspection`. @@ -197,39 +85,26 @@ This playbook supports backing up and restoring databases for an in-cluster Mong ```bash # Full backup all MongoDB data for the dev1 instance export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev ansible-playbook ibm.mas_devops.br_mongodb # Incremental backup all MongoDB data for the dev1 instance export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 -ansible-playbook ibm.mas_devops.br_mongodb - -# Create a scheduled backup Job for the dev1 instance -export MASBR_ACTION=backup -export MASBR_BACKUP_SCHEDULE="50 0 * * *" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev ansible-playbook ibm.mas_devops.br_mongodb # Restore all MongoDB data for the dev1 instance export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev ansible-playbook ibm.mas_devops.br_mongodb # Backup just the IoT MongoDB data for the dev2 instance export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MAS_INSTANCE_ID=dev2 export MAS_APP_ID=iot @@ -249,7 +124,6 @@ This playbook `ibm.mas_devops.br_db2` will invoke the role [db2](../roles/db2.md ```bash # Full backup for the db2w-shared Db2 instance export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export DB2_INSTANCE_NAME=db2w-shared ansible-playbook ibm.mas_devops.br_db2 @@ -257,23 +131,12 @@ ansible-playbook ibm.mas_devops.br_db2 # Incremental backup for the db2w-shared Db2 instance export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_db2 - -# Create a scheduled backup Job for the db2w-shared Db2 instance -export MASBR_ACTION=backup -export MASBR_BACKUP_SCHEDULE="50 0 * * *" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export DB2_INSTANCE_NAME=db2w-shared ansible-playbook ibm.mas_devops.br_db2 # Restore for the db2w-shared Db2 instance export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 export DB2_INSTANCE_NAME=db2w-shared @@ -298,34 +161,22 @@ This playbook `ibm.mas_devops.br_core` will backup the following components that ```bash # Full backup all core data for the dev1 instance export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev ansible-playbook ibm.mas_devops.br_core # Incremental backup all core data for the dev1 instance export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 -ansible-playbook ibm.mas_devops.br_core - -# Create a scheduled backup Job for the dev1 instance -export MASBR_ACTION=backup -export MASBR_BACKUP_SCHEDULE="50 0 * * *" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev ansible-playbook ibm.mas_devops.br_core # Restore all core data for the dev1 instance export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev ansible-playbook ibm.mas_devops.br_core ``` @@ -346,51 +197,35 @@ This playbook `ibm.mas_devops.br_manage` will backup the following components th - `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. - `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by Manage, you need to set the correct Db2 instance name for this environment variable. +- `DB2_INSTANCE_NAME` **Optional**. When defined, this playbook will backup the Db2 instance used by Manage. DB2 role is skipped when environment variable is not defined.. ### Examples ```bash # Full backup all manage data for the dev1 instance and ws1 workspace export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role ansible-playbook ibm.mas_devops.br_manage # Incremental backup all manage data for the dev1 instance and ws1 workspace export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role ansible-playbook ibm.mas_devops.br_manage # Restore all manage data for the dev1 instance and ws1 workspace export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_manage - -# Create a scheduled backup of all manage data for the dev1 instance and ws1 workspace -# This will run at 01:00, Monday through Friday -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_BACKUP_SCHEDULE="0 1 * * 1-5" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role ansible-playbook ibm.mas_devops.br_manage ``` @@ -417,9 +252,8 @@ This playbook `ibm.mas_devops.br_iot` will backup the following components that ```bash # Full backup all iot data for the dev1 instance export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=db2w-shared ansible-playbook ibm.mas_devops.br_iot @@ -427,35 +261,21 @@ ansible-playbook ibm.mas_devops.br_iot # Incremental backup all iot data for the dev1 instance export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=db2w-shared ansible-playbook ibm.mas_devops.br_iot # Restore all iot data for the dev1 instance export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=db2w-shared ansible-playbook ibm.mas_devops.br_iot -# Create a scheduled backup of all iot data for the dev1 instance -# This will run at 01:00, Monday through Friday -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_BACKUP_SCHEDULE="0 1 * * 1-5" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_iot ``` @@ -483,9 +303,8 @@ This playbook `ibm.mas_devops.br_monitor` will backup the following components t ```bash # Full backup all monitor data for the dev1 instance export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=db2w-shared ansible-playbook ibm.mas_devops.br_monitor @@ -493,32 +312,17 @@ ansible-playbook ibm.mas_devops.br_monitor # Incremental backup all monitor data for the dev1 instance export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=db2w-shared ansible-playbook ibm.mas_devops.br_monitor # Restore all monitor data for the dev1 instance export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev1 -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_monitor - -# Create a scheduled backup of all monitor data for the dev1 instance -# This will run at 01:00, Monday through Friday -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_BACKUP_SCHEDULE="0 1 * * 1-5" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=db2w-shared ansible-playbook ibm.mas_devops.br_monitor @@ -548,9 +352,8 @@ This playbook `ibm.mas_devops.br_health` will backup the following components th ```bash # Full backup all health data for the dev1 instance and ws1 workspace export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=mas-dev1-ws1-manage ansible-playbook ibm.mas_devops.br_health @@ -558,32 +361,17 @@ ansible-playbook ibm.mas_devops.br_health # Incremental backup all health data for the dev1 instance and ws1 workspace export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=mas-dev1-ws1-manage ansible-playbook ibm.mas_devops.br_health # Restore all health data for the dev1 instance and ws1 workspace export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev1 -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_health - -# Create a scheduled backup of all health data for the dev1 instance and ws1 workspace -# This will run at 01:00, Monday through Friday -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_BACKUP_SCHEDULE="0 1 * * 1-5" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=mas-dev1-ws1-manage ansible-playbook ibm.mas_devops.br_health @@ -613,9 +401,8 @@ This playbook `ibm.mas_devops.br_optimizer` will backup the following components ```bash # Full backup all optimizer data for the dev1 instance and ws1 workspace export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=mas-dev1-ws1-manage ansible-playbook ibm.mas_devops.br_optimizer @@ -623,32 +410,17 @@ ansible-playbook ibm.mas_devops.br_optimizer # Incremental backup all optimizer data for the dev1 instance and ws1 workspace export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=mas-dev1-ws1-manage ansible-playbook ibm.mas_devops.br_optimizer # Restore all optimizer data for the dev1 instance and ws1 workspace export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev1 -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_optimizer - -# Create a scheduled backup of all optimizer data for the dev1 instance and ws1 workspace -# This will run at 01:00, Monday through Friday -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_BACKUP_SCHEDULE="0 1 * * 1-5" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 export DB2_INSTANCE_NAME=mas-dev1-ws1-manage ansible-playbook ibm.mas_devops.br_optimizer @@ -675,45 +447,31 @@ This playbook `ibm.mas_devops.br_visualinspection` will backup the following com ```bash # Full backup all visual inspection data for the dev1 instance and ws1 workspace export MASBR_ACTION=backup -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 ansible-playbook ibm.mas_devops.br_visualinspection # Incremental backup all visual inspection data for the dev1 instance and ws1 workspace export MASBR_ACTION=backup export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 ansible-playbook ibm.mas_devops.br_visualinspection # Restore all visual inspection data for the dev1 instance and ws1 workspace export MASBR_ACTION=restore -export MASBR_STORAGE_TYPE=local export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev1 -export MAS_WORKSPACE_ID=ws1 -ansible-playbook ibm.mas_devops.br_visualinspection - -# Create a scheduled backup of all visual inspection data for the dev1 instance and ws1 workspace -# This will run at 01:00, Monday through Friday -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_BACKUP_SCHEDULE="0 1 * * 1-5" -export MASBR_JOB_TIMEZONE="Asia/Shanghai" -export MASBR_STORAGE_TYPE=local -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev1 +export MAS_INSTANCE_ID=dev export MAS_WORKSPACE_ID=ws1 ansible-playbook ibm.mas_devops.br_visualinspection ``` -## Reference +Reference +------------------------------------------------------------------------------- ### Directory Structure No matter what kind of storage systems you choose, the folder structure created in the storage system is same. diff --git a/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml b/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml index 5fd0736311..7d037d7744 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml @@ -24,24 +24,6 @@ include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/check_{{ _job_type }}_vars.yml" -# Add labels to current Job -# ----------------------------------------------------------------------------- -- name: "Add lables to current Job" - when: - - masbr_running_in_task_job - - masbr_task_type == "schedule" - - masbr_included_before_run_tasks is not defined - shell: >- - job_name=$(oc get job -n {{ _job_namespace }} --sort-by=.metadata.creationTimestamp --no-headers - | grep {{ _job_name }} | awk '{print $1}' | tail -n 1); - oc label job ${job_name} -n {{ _job_namespace }} masbr-job={{ masbr_job_name }}; - pod_name=$(oc get pod -n {{ _job_namespace }} -l job-name=${job_name} --no-headers | awk '{print $1}'); - oc label pod ${pod_name} -n {{ _job_namespace }} masbr-job={{ masbr_job_name }} - vars: - _job_namespace: "{{ lookup('env', 'MASBR_TASK_JOB_NAMESPACE') }}" - _job_name: "{{ lookup('env', 'MASBR_TASK_JOB_NAME') }}" - - # Before backup/restore component # ------------------------------------------------------------------------- - name: "Before {{ _job_type }} {{ masbr_job_component.name }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml b/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml index 152b5e33d6..abb99d4b69 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml @@ -1,6 +1,6 @@ --- # Set below common job facts: -# masbr_task_type: backup, restore, schedule +# masbr_task_type: backup, restore # masbr_job_type: backup, restore # masbr_job_name, masbr_job_name_final # @@ -23,10 +23,6 @@ # only used when masbr_backup_type='incr' masbr_backup_from_version: "{{ lookup('env', 'MASBR_BACKUP_FROM_VERSION') | default('', true) }}" - # Cron expression for scheduled backup - # https://en.wikipedia.org/wiki/Cron - masbr_backup_schedule: "{{ lookup('env', 'MASBR_BACKUP_SCHEDULE') | default('', true) }}" - # Check 'masbr_job_component' # ----------------------------------------------------------------------------- @@ -87,28 +83,9 @@ # Set 'masbr_task_type' # ----------------------------------------------------------------------------- -- name: "Set fact: running task type (schedule)" - when: masbr_backup_schedule is defined and masbr_backup_schedule | length > 0 - set_fact: - masbr_task_type: "schedule" - -- name: "Set fact: always create a new Job version for each Job of the CronJob" - when: - - masbr_backup_schedule is defined and masbr_backup_schedule | length > 0 - - masbr_included_before_run_tasks is not defined - set_fact: - masbr_job_version: "{{ masbr_timestamp_format | strftime }}" - -- name: "Set fact: running task type (backup)" - when: masbr_task_type is not defined - set_fact: - masbr_task_type: "backup" - - -# Set backup job variables -# ----------------------------------------------------------------------------- - name: "Set fact: backup job variables" set_fact: + masbr_task_type: "backup" masbr_job_type: "backup" - name: "Set fact: job name include instance" diff --git a/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml b/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml index 1744b521bc..a29a1c995d 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml @@ -9,9 +9,6 @@ # ONLY FOR DEV __masbr_dev_create_env_file: "{{ lookup('env', '__MASBR_DEV_CREATE_ENV_FILE') | default(false, true) | bool }}" - # Whether the tasks are already running in a k8s Job/CronJob - masbr_running_in_task_job: "{{ lookup('env', 'MASBR_RUNNING_IN_TASK_JOB') | default(false, true) | bool }}" - # Temp folder in the Pod for backup/restore masbr_pod_temp_folder: "/tmp/masbr" @@ -34,132 +31,12 @@ # Storage location # ----------------------------------------------------------------------------- -- name: "Set fact: default values of storage location variables" - set_fact: - masbr_use_local_storage: false - masbr_use_pvc_storage: false - masbr_use_cloud_storage: false - -# Must specified supported storage location type -- name: "Fail if masbr_storage_type is not provided" +- name: "Fail if masbr_storage_local_folder is not provided" assert: - that: masbr_storage_type is defined and masbr_storage_type != "" - fail_msg: "masbr_storage_type is required" - -- name: "Fail if masbr_storage_type is not supported" - assert: - that: masbr_storage_type in ['local', 'pvc', 'cloud'] - fail_msg: "masbr_storage_type must be one of the supported value: local, pvc, cloud" - -# Specified local storage location -- name: "Check variables for local backup storage" - when: masbr_storage_type == "local" - block: - - name: "Fail if masbr_storage_local_folder is not provided" - assert: - that: masbr_storage_local_folder is defined and masbr_storage_local_folder != "" - fail_msg: "masbr_storage_local_folder is required" - - - name: "Set fact: use local backup storage" - set_fact: - masbr_use_local_storage: true - # Not support creating k8s Job/CronJob to run tasks when using local storage - masbr_create_task_job: false - - - name: "Debug: variables for local backup storage" - debug: - msg: - - "Use local storage .................. {{ masbr_use_local_storage }}" - - "Local storage folder ............... {{ masbr_storage_local_folder }}" - - "Run tasks in k8s Job ............... {{ masbr_create_task_job }}" - -# Specified pvc storage location -- name: "Check variables for pvc backup storage" - when: masbr_storage_type == "pvc" - block: - - name: "Fail if masbr_storage_pvc_name is not provided" - assert: - that: masbr_storage_pvc_name is defined and masbr_storage_pvc_name != "" - fail_msg: "masbr_storage_pvc_name is required" - - - name: "Fail if masbr_storage_pvc_path is not provided" - assert: - that: masbr_storage_pvc_path is defined and masbr_storage_pvc_path != "" - fail_msg: "masbr_storage_pvc_path is required" - - - name: "Set fact: use pvc backup storage" - set_fact: - masbr_use_pvc_storage: true - # By default, we will create k8s Job/CronJob to run tasks when using pvc storage - # You can set 'MASBR_CREATE_TASK_JOB' to 'false' when developing and running tasks on local workstation - masbr_create_task_job: "{{ lookup('env', 'MASBR_CREATE_TASK_JOB') | default(true, true) | bool }}" - - - name: "Debug: variables for pvc backup storage" - debug: - msg: - - "Use PVC storage .................... {{ masbr_use_pvc_storage }}" - - "PVC name ........................... {{ masbr_storage_pvc_name }}" - - "PVC mount path ..................... {{ masbr_storage_pvc_path }}" - - "Run tasks in k8s Job ............... {{ masbr_create_task_job }}" - -# Specified cloud storage location -- name: "Check variables for cloud backup storage" - when: masbr_storage_type == "cloud" - block: - - name: "Fail if masbr_storage_cloud_rclone_file is not provided" - assert: - that: masbr_storage_cloud_rclone_file is defined and masbr_storage_cloud_rclone_file != "" - fail_msg: "masbr_storage_cloud_rclone_file is required" - - - name: "Fail if masbr_storage_cloud_rclone_name is not provided" - assert: - that: masbr_storage_cloud_rclone_name is defined and masbr_storage_cloud_rclone_name != "" - fail_msg: "masbr_storage_cloud_rclone_name is required" - - - name: "Fail if masbr_storage_cloud_bucket is not provided" - assert: - that: masbr_storage_cloud_bucket is defined and masbr_storage_cloud_bucket != "" - fail_msg: "masbr_storage_cloud_bucket is required" - - - name: "Set fact: use cloud backup storage" - set_fact: - masbr_use_cloud_storage: true - # By default, we will create k8s Job/CronJob to run tasks when using cloud storage - # You can set 'MASBR_CREATE_TASK_JOB' to 'false' when developing and running tasks on local workstation - masbr_create_task_job: "{{ lookup('env', 'MASBR_CREATE_TASK_JOB') | default(true, true) | bool }}" - - - name: "Debug: variables for cloud backup storage" - debug: - msg: - - "Use cloud storage .................. {{ masbr_use_cloud_storage }}" - - "Rclone config file ................. {{ masbr_storage_cloud_rclone_file }}" - - "Rclone config name ................. {{ masbr_storage_cloud_rclone_name }}" - - "Bucket name ........................ {{ masbr_storage_cloud_bucket }}" - - "Run tasks in k8s Job ............... {{ masbr_create_task_job }}" - - -# Sending notification -# ----------------------------------------------------------------------------- -- name: "Check variables for Slack" - when: masbr_slack_enabled - block: - - name: "Fail if masbr_slack_token is not provided" - assert: - that: masbr_slack_token is defined and masbr_slack_token != "" - fail_msg: "masbr_slack_token is required when masbr_slack_enabled set to true" - - - name: "Fail if masbr_slack_channel is not provided" - assert: - that: masbr_slack_channel is defined and masbr_slack_channel != "" - fail_msg: "masbr_slack_channel is required when masbr_slack_enabled set to true" - - - name: "Debug: variables for sending Slack notification" - debug: - msg: - - "Slack channel ...................... {{ masbr_slack_channel }}" - - "Slack user ......................... {{ masbr_slack_user }}" - - "Notification level ................. {{ masbr_slack_level }} ({{ masbr_notification_levels[masbr_slack_level] | join(',') }})" + that: masbr_storage_local_folder is defined and masbr_storage_local_folder != "" + fail_msg: "masbr_storage_local_folder is required" - - name: "Set fact: reset masbr_sent_notifications" - set_fact: - masbr_sent_notifications: [] +- name: "Debug: variables for local backup storage" + debug: + msg: + - "Local storage folder ............... {{ masbr_storage_local_folder }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_local_files_to_storage.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_local_files_to_storage.yml index 44dd7fb4bf..058043a510 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_local_files_to_storage.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/copy_local_files_to_storage.yml @@ -1,59 +1,24 @@ --- # Copy local job files to local storage # ----------------------------------------------------------------------------- -- name: "Copy local job files to local storage" - when: masbr_use_local_storage - block: - - name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - - name: "Debug: local storage job folder" - debug: - msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" - - - name: "Copy local job files to local storage job folder" - shell: >- - mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && - cp -rf {{ [masbr_local_job_folder, item.src_file] | path_join }} - {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && - ls -lA {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - loop: "{{ masbr_cf_paths }}" - register: _local_copy_output - - - name: "Debug: copy local job files to local storage job folder" - debug: - msg: "{{ _local_copy_output | json_query('results[*].stdout_lines') }}" - - -# Copy local job files to cloud storage -# ----------------------------------------------------------------------------- -- name: "Copy local job files to cloud storage" - when: masbr_use_cloud_storage - block: - - name: "Set fact: cloud storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - - name: "Debug: cloud storage job folder" - debug: - msg: "Cloud storage job folder .......... {{ masbr_storage_job_folder }}" - - - name: "Copy local job files to cloud storage job folder" - shell: >- - rclone --links --progress --no-check-certificate --config {{ masbr_storage_cloud_rclone_file }} - copy --no-traverse - {{ [masbr_local_job_folder, item.src_file] | path_join }} - {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - loop: "{{ masbr_cf_paths }}" - register: _rclone_copy_output - - - name: "Debug: copy local job files to cloud storage job folder" - debug: - msg: "{{ _rclone_copy_output | json_query('results[*].stdout_lines') }}" - - -# Copy local job files to pvc storage -# ----------------------------------------------------------------------------- +- name: "Set fact: local storage job folder" + set_fact: + masbr_storage_job_folder: >- + {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} + +- name: "Debug: local storage job folder" + debug: + msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" + +- name: "Copy local job files to local storage job folder" + shell: >- + mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && + cp -rf {{ [masbr_local_job_folder, item.src_file] | path_join }} + {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && + ls -lA {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} + loop: "{{ masbr_cf_paths }}" + register: _local_copy_output + +- name: "Debug: copy local job files to local storage job folder" + debug: + msg: "{{ _local_copy_output | json_query('results[*].stdout_lines') }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml index 3f10c4be27..2601a6557b 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml @@ -1,323 +1,47 @@ --- # Copy files from pod to local storage # ----------------------------------------------------------------------------- -- name: "Copy files from pod to local storage" - when: masbr_use_local_storage - block: - # Local storage job folder - - name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - - name: "Debug: local storage job folder" - debug: - msg: "Local storage job folder ......... {{ masbr_storage_job_folder }}" - - # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* - - name: "Copy files from pod folder to local storage folder" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_folder }} - {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - loop: "{{ masbr_cf_paths }}" - - # Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file - - name: "Copy file from pod to local storage folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_file }} - {{ [masbr_storage_job_folder, item.dest_folder, item.src_file|basename] | path_join }} - loop: "{{ masbr_cf_paths }}" - - # Condition 3. src_file -> dest_file - - name: "Copy file from pod to local storage file" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - shell: >- - mkdir -p {{ [masbr_storage_job_folder, item.dest_file|dirname] | path_join }} && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_file }} - {{ [masbr_storage_job_folder, item.dest_file] | path_join }} - loop: "{{ masbr_cf_paths }}" - - -# Copy files from pod to cloud storage -# ----------------------------------------------------------------------------- -- name: "Copy files from pod to cloud storage" - when: masbr_use_cloud_storage - block: - # Cloud storage job folder - - name: "Set fact: cloud storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - - name: "Debug: cloud storage job folder" - debug: - msg: "Cloud storage job folder ......... {{ masbr_storage_job_folder }}" - - # Cloud storage from job folder - # (only used for incremental backup 'src_folder -> dest_folder') - - name: "Set fact: cloud storage from job folder" - when: masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0 - set_fact: - masbr_storage_from_job_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_from_job_name }} - - - name: "Debug: cloud storage from job folder" - when: masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0 - debug: - msg: "Cloud storage from job folder ......... {{ masbr_storage_from_job_folder }}" - - # The source paths are not located in pvc, so we need to copy files to mounted pvc job folder first, - # then create a Job pod to mount the same pvc and copy files from pvc to cloud storage folder. - - name: "Copy files to mounted pvc job folder" - when: masbr_cf_are_pvc_paths is not defined or not masbr_cf_are_pvc_paths - block: - # pvc job folder - - name: "Set fact: pvc job folder" - set_fact: - masbr_cf_paths_from_pvc: [] - masbr_cf_pvc_job_folder: >- - {{ [masbr_cf_pvc_mount_path, masbr_cf_job_name] | path_join }} - - - name: "Debug: pvc job folder" - debug: - msg: "PVC job folder .................... {{ masbr_cf_pvc_job_folder }}" - - - name: "Set fact: list dest files in pvc" - set_fact: - masbr_cf_ls_dest_files: [] - - # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* - - name: "Copy files from pod folder to pvc job folder" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'mkdir -p {{ [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join }} && - cp -rf {{ item.src_folder }}/* {{ [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join }} && - ls -lA {{ [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join }}' - loop: "{{ masbr_cf_paths }}" - register: _ls_output - - - name: "Set fact: append dest files in pvc" - set_fact: - masbr_cf_ls_dest_files: "{{ masbr_cf_ls_dest_files + _ls_output | json_query('results[*].stdout_lines') }}" - - - name: "Set fact: change source paths based on pvc job folder" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - set_fact: - masbr_cf_paths_from_pvc: >- - {{ masbr_cf_paths_from_pvc + [{ - 'src_folder': [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join, - 'dest_folder': item.dest_folder - }] }} - loop: "{{ masbr_cf_paths }}" - - # Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file - - name: "Copy file from pod to pvc job folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'mkdir -p {{ [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join }} && - cp -f {{ item.src_file }} {{ [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join }} && - ls -lA {{ [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join }}' - loop: "{{ masbr_cf_paths }}" - register: _ls_output - - - name: "Set fact: append dest files in pvc" - set_fact: - masbr_cf_ls_dest_files: "{{ masbr_cf_ls_dest_files + _ls_output | json_query('results[*].stdout_lines') }}" - - - name: "Set fact: change source paths based on pvc job folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - set_fact: - masbr_cf_paths_from_pvc: >- - {{ masbr_cf_paths_from_pvc + [{ - 'src_file': [masbr_cf_pvc_job_folder, item.dest_folder|basename, item.src_file|basename] | path_join, - 'dest_folder': item.dest_folder - }] }} - loop: "{{ masbr_cf_paths }}" - - # Condition 3. src_file -> dest_file - - name: "Copy file from pod to pvc job file" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'mkdir -p {{ [masbr_cf_pvc_job_folder, item.dest_file|dirname] | path_join }} && - cp -f {{ item.src_file }} {{ [masbr_cf_pvc_job_folder, item.dest_file|dirname] | path_join }} && - ls -lA {{ [masbr_cf_pvc_job_folder, item.dest_file|dirname] | path_join }}' - loop: "{{ masbr_cf_paths }}" - register: _ls_output - - - name: "Set fact: append dest files in pvc" - set_fact: - masbr_cf_ls_dest_files: "{{ masbr_cf_ls_dest_files + _ls_output | json_query('results[*].stdout_lines') }}" - - - name: "Set fact: change source paths based on pvc job folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - set_fact: - masbr_cf_paths_from_pvc: >- - {{ masbr_cf_paths_from_pvc + [{ - 'src_file': [masbr_cf_pvc_job_folder, item.dest_file|dirname, item.src_file|basename] | path_join, - 'dest_file': item.dest_file - }] }} - loop: "{{ masbr_cf_paths }}" - - - name: "Debug: list files in pvc paths" - debug: - msg: "{{ masbr_cf_ls_dest_files }}" - - - name: "Set fact: file paths in pvc job folder" - when: masbr_cf_are_pvc_paths is defined and masbr_cf_are_pvc_paths - set_fact: - masbr_cf_paths_from_pvc: "{{ masbr_cf_paths }}" - - - name: "Debug: file paths in pvc job folder" - debug: - msg: "{{ masbr_cf_paths_from_pvc }}" - - - # Set job variables - # ----------------------------------------------------------------------------- - - name: "Set fact: copy file Job name" - set_fact: - # must be no more than 63 characters and in lower case - # format 'copy--', e.g. - # 'copy-20240424210110-20240424210110' - masbr_cf_k8s_name: >- - copy-{{ masbr_job_version }}-{{ masbr_timestamp_format | strftime }} - - - name: "Debug: copy file Job name" - debug: - msg: - - "Copy file Job name ..................... {{ masbr_cf_k8s_name }}" - - - # Set copy file commands: mounted pvc -> cloud storage - # ------------------------------------------------------------------------- - # Copy files from mounted pvc folder to cloud storage - - name: "Set fact: initial rclone copy file command" - set_fact: - masbr_cf_cmds: >- - mkdir -p /tmp/logs; - rclone version 2>&1 | tee /tmp/logs/rclone-version.log - - # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* - - name: "Set fact: copy all files from pvc folder to cloud storage folder (full)" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - - masbr_cf_from_job_name is not defined or masbr_cf_from_job_name | length == 0 - set_fact: - masbr_cf_cmds: >- - {{ masbr_cf_cmds }} && - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copy --no-traverse - {{ item.src_folder }} {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - 2>&1 | tee /tmp/logs/rclone-copy.log - loop: "{{ masbr_cf_paths_from_pvc }}" - - - name: "Set fact: copy changed files from pvc folder to cloud storage folder (incremental)" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - - masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0 - set_fact: - masbr_cf_cmds: >- - {{ masbr_cf_cmds }} && - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - sync {{ item.src_folder }} {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - --compare-dest {{ [masbr_storage_from_job_folder, item.dest_folder] | path_join }} - 2>&1 | tee /tmp/logs/rclone-sync.log; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - check {{ item.src_folder }} {{ [masbr_storage_from_job_folder, item.dest_folder] | path_join }} - --combined /tmp/masbr-changes.txt - 2>&1 | tee /tmp/logs/rclone-check.log; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copy /tmp/masbr-changes.txt {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - 2>&1 | tee /tmp/logs/rclone-copy.log - loop: "{{ masbr_cf_paths_from_pvc }}" - - # Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file - - name: "Set fact: copy file from pvc to cloud storage folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - set_fact: - masbr_cf_cmds: >- - {{ masbr_cf_cmds }} && - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copy --no-traverse - {{ item.src_file }} {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - 2>&1 | tee /tmp/logs/rclone-copy.log - loop: "{{ masbr_cf_paths_from_pvc }}" - - # Condition 3. src_file -> dest_file - - name: "Set fact: copy file from pvc to cloud storage file" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - set_fact: - masbr_cf_cmds: >- - {{ masbr_cf_cmds }} && - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copyto {{ item.src_file }} {{ [masbr_storage_job_folder, item.dest_file] | path_join }} - 2>&1 | tee /tmp/logs/rclone-copyto.log - loop: "{{ masbr_cf_paths_from_pvc }}" - - - name: "Set fact: command for copying logs" - set_fact: - masbr_cf_cmds: >- - {{ masbr_cf_cmds }}; - ls -lA /tmp/logs; - tar -czf /tmp/{{ masbr_cf_k8s_name }}-log.tar.gz -C /tmp/logs .; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copy /tmp/{{ masbr_cf_k8s_name }}-log.tar.gz - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_job_type }}s/{{ masbr_job_name_final }}/log - - - name: "Debug: rclone copy file command" - debug: - msg: "{{ masbr_cf_cmds }}" - - - # Create copy file Job: mounted pvc -> cloud storage - # ------------------------------------------------------------------------- - # 1. The Job pod will mount the PVC where the files will be copied from - # 2. Use rclone to copy files from PVC to COS - - name: "Create copy file Job" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_copy_cloud_files_job.yml" - - always: - # Clean up - - name: "Delete pvc job folder" - when: masbr_cf_pvc_job_folder is defined and masbr_cf_pvc_job_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'rm -rf {{ masbr_cf_pvc_job_folder }}' - - -# Copy files from pod to pvc storage -# ----------------------------------------------------------------------------- +- name: "Set fact: local storage job folder" + set_fact: + masbr_storage_job_folder: >- + {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} + +- name: "Debug: local storage job folder" + debug: + msg: "Local storage job folder ......... {{ masbr_storage_job_folder }}" + +# Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* +- name: "Copy files from pod folder to local storage folder" + when: + - item.src_folder is defined and item.src_folder | length > 0 + - item.dest_folder is defined and item.dest_folder | length > 0 + shell: >- + mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && + oc cp --retries=50 -c {{ masbr_cf_container_name }} + {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_folder }} + {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} + loop: "{{ masbr_cf_paths }}" + +# Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file +- name: "Copy file from pod to local storage folder" + when: + - item.src_file is defined and item.src_file | length > 0 + - item.dest_folder is defined and item.dest_folder | length > 0 + shell: >- + mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && + oc cp --retries=50 -c {{ masbr_cf_container_name }} + {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_file }} + {{ [masbr_storage_job_folder, item.dest_folder, item.src_file|basename] | path_join }} + loop: "{{ masbr_cf_paths }}" + +# Condition 3. src_file -> dest_file +- name: "Copy file from pod to local storage file" + when: + - item.src_file is defined and item.src_file | length > 0 + - item.dest_file is defined and item.dest_file | length > 0 + shell: >- + mkdir -p {{ [masbr_storage_job_folder, item.dest_file|dirname] | path_join }} && + oc cp --retries=50 -c {{ masbr_cf_container_name }} + {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_file }} + {{ [masbr_storage_job_folder, item.dest_file] | path_join }} + loop: "{{ masbr_cf_paths }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_local.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_local.yml index fff540b3af..363e4516ca 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_local.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_local.yml @@ -1,59 +1,24 @@ --- # Copy job files from local storage to local job folder # ----------------------------------------------------------------------------- -- name: "Copy job files from local storage to local job folder" - when: masbr_use_local_storage - block: - - name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - - name: "Debug: local storage job folder" - debug: - msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" - - - name: "Copy job files from local storage to local job folder" - shell: >- - mkdir -p {{ [masbr_local_job_folder, item.dest_folder] | path_join }} && - cp -rf {{ [masbr_storage_job_folder, item.src_file] | path_join }} - {{ [masbr_local_job_folder, item.dest_folder] | path_join }} && - ls -lA {{ [masbr_local_job_folder, item.dest_folder] | path_join }} - loop: "{{ masbr_cf_paths }}" - register: _local_copy_output - - - name: "Debug: copy job files from local storage to local job folder" - debug: - msg: "{{ _local_copy_output | json_query('results[*].stdout_lines') }}" - - -# Copy job files from cloud storage to local job folder -# ----------------------------------------------------------------------------- -- name: "Copy job files from cloud storage to local job folder" - when: masbr_use_cloud_storage - block: - - name: "Set fact: cloud storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - - name: "Debug: cloud storage job folder" - debug: - msg: "Cloud storage job folder .......... {{ masbr_storage_job_folder }}" - - - name: "Copy job files from cloud storage to local job folder" - shell: >- - rclone --links --progress --no-check-certificate --config {{ masbr_storage_cloud_rclone_file }} - copy --no-traverse - {{ [masbr_storage_job_folder, item.src_file] | path_join }} - {{ [masbr_local_job_folder, item.dest_folder] | path_join }} - loop: "{{ masbr_cf_paths }}" - register: _rclone_copy_output +- name: "Set fact: local storage job folder" + set_fact: + masbr_storage_job_folder: >- + {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - name: "Debug: copy job files from cloud storage to local job folder" - debug: - msg: "{{ _rclone_copy_output | json_query('results[*].stdout_lines') }}" +- name: "Debug: local storage job folder" + debug: + msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" - -# Copy job files from pvc storage to local job folder -# ----------------------------------------------------------------------------- +- name: "Copy job files from local storage to local job folder" + shell: >- + mkdir -p {{ [masbr_local_job_folder, item.dest_folder] | path_join }} && + cp -rf {{ [masbr_storage_job_folder, item.src_file] | path_join }} + {{ [masbr_local_job_folder, item.dest_folder] | path_join }} && + ls -lA {{ [masbr_local_job_folder, item.dest_folder] | path_join }} + loop: "{{ masbr_cf_paths }}" + register: _local_copy_output + +- name: "Debug: copy job files from local storage to local job folder" + debug: + msg: "{{ _local_copy_output | json_query('results[*].stdout_lines') }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml index 1531c6251f..6aa3eb0247 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml @@ -1,427 +1,51 @@ --- # Copy files from local storage to pod # ----------------------------------------------------------------------------- -- name: "Copy files from local storage to pod" - when: masbr_use_local_storage - block: - # Local storage job folder - - name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - name: "Debug: local storage job folder" - debug: - msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" - - # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* - - name: "Copy files from local storage folder to pod folder" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'temp_dest_folder={{ [item.dest_folder, masbr_job_version] | path_join }} && - mkdir -p ${temp_dest_folder} && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ [masbr_storage_job_folder, item.src_folder] | path_join }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:${temp_dest_folder} && - mv -f ${temp_dest_folder}/* {{ item.dest_folder }} && - rm -rf ${temp_dest_folder}' - loop: "{{ masbr_cf_paths }}" - - # Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file - - name: "Copy file from local storage folder to pod folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'mkdir -p {{ item.dest_folder }}' && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ [masbr_storage_job_folder, item.src_file] | path_join }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.dest_folder }} - loop: "{{ masbr_cf_paths }}" - - # Condition 3. src_file -> dest_file - - name: "Copy file from local storage folder to pod file" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'temp_dest_folder={{ [item.dest_file|dirname, masbr_job_version] | path_join }} && - mkdir -p ${temp_dest_folder} && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ [masbr_storage_job_folder, item.src_file] | path_join }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:${temp_dest_folder} && - mv -f ${temp_dest_folder}/{{ item.src_file|basename }} {{ item.dest_file }} && - rm -rf ${temp_dest_folder}' - loop: "{{ masbr_cf_paths }}" - - -# Copy files from cloud storage to pod -# ----------------------------------------------------------------------------- -- name: "Copy files from cloud storage to pod" - when: masbr_use_cloud_storage - block: - # Cloud storage job folder - - name: "Set fact: cloud storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - - - name: "Debug: cloud storage job folder" - debug: - msg: "Cloud storage job folder .......... {{ masbr_storage_job_folder }}" - - # Cloud storage from job folder - # (only used for restoring from incremental backup 'src_folder -> dest_folder') - - name: "Set fact: cloud storage from job folder" - when: masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0 - set_fact: - masbr_storage_from_job_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_from_job_name }} - - - name: "Debug: cloud storage from job folder" - when: masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0 - debug: - msg: "Cloud storage from job folder ......... {{ masbr_storage_from_job_folder }}" - - - name: "Set fact: pvc paths" - set_fact: - # Copy to PVC paths by default - masbr_cf_paths_to_pvc: "{{ masbr_cf_paths }}" - masbr_cf_paths_from_pvc: [] - - # The target paths are not located in pvc, so we need to create a Job to copy files from cloud storage - # to mounted pvc job folder first, then copy files from pvc to pod destination folder. - - name: "Set pvc paths" - when: masbr_cf_are_pvc_paths is not defined or not masbr_cf_are_pvc_paths - block: - # pvc job folder - - name: "Set fact: pvc job folder" - set_fact: - masbr_cf_paths_to_pvc: [] - masbr_cf_pvc_job_folder: >- - {{ [masbr_cf_pvc_mount_path, masbr_cf_job_name] | path_join }} - - - name: "Debug: pvc job folder" - debug: - msg: "PVC job folder .................... {{ masbr_cf_pvc_job_folder }}" - - # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* - - name: "Set fact: change destination paths based on pvc job folder" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - set_fact: - masbr_cf_paths_to_pvc: >- - {{ masbr_cf_paths_to_pvc + [{ - 'src_folder': item.src_folder, - 'dest_folder': [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join - }] }} - masbr_cf_paths_from_pvc: >- - {{ masbr_cf_paths_from_pvc + [{ - 'src_folder': [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join, - 'dest_folder': item.dest_folder - }] }} - loop: "{{ masbr_cf_paths }}" - - # Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file - - name: "Set fact: change destination paths based on pvc job folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - set_fact: - masbr_cf_paths_to_pvc: >- - {{ masbr_cf_paths_to_pvc + [{ - 'src_file': item.src_file, - 'dest_folder': [masbr_cf_pvc_job_folder, item.dest_folder|basename] | path_join - }] }} - masbr_cf_paths_from_pvc: >- - {{ masbr_cf_paths_from_pvc + [{ - 'src_file': [masbr_cf_pvc_job_folder, item.dest_folder|basename, item.src_file|basename] | path_join, - 'dest_folder': item.dest_folder - }] }} - loop: "{{ masbr_cf_paths }}" - - # Condition 3. src_file -> dest_file - - name: "Set fact: change destination paths based on pvc job folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - set_fact: - masbr_cf_paths_to_pvc: >- - {{ masbr_cf_paths_to_pvc + [{ - 'src_file': item.src_file, - 'dest_file': [masbr_cf_pvc_job_folder, item.src_file|dirname, item.dest_file|basename] | path_join, - }] }} - masbr_cf_paths_from_pvc: >- - {{ masbr_cf_paths_from_pvc + [{ - 'src_file': [masbr_cf_pvc_job_folder, item.dest_file|dirname, item.src_file|basename] | path_join, - 'dest_file': item.dest_file - }] }} - loop: "{{ masbr_cf_paths }}" - - - name: "Debug: paths of copying files from cloud storage to pvc" - debug: - msg: "{{ masbr_cf_paths_to_pvc }}" - - - # Set job variables - # ----------------------------------------------------------------------------- - - name: "Set fact: copy file Job name" - set_fact: - # must be no more than 63 characters and in lower case - # format 'copy--', e.g. - # 'copy-20240424210110-20240424210110' - masbr_cf_k8s_name: >- - copy-{{ masbr_job_version }}-{{ masbr_timestamp_format | strftime }} - - - name: "Debug: copy file Job name" - debug: - msg: - - "Copy file Job name ..................... {{ masbr_cf_k8s_name }}" - - - # Set copy file commands: cloud storage -> mounted pvc - # ------------------------------------------------------------------------- - - name: "Set fact: dest paths in pvc" - set_fact: - # For debugging only - masbr_cf_dest_folders: [] - - - name: "Set fact: initial rclone copy file command" - set_fact: - masbr_cf_cmds: >- - mkdir -p /tmp/logs; - rclone version 2>&1 | tee /tmp/logs/rclone-version.log - - # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* - # When restoring from a full backup: use 'rclone sync' - # When restoring from an incremental backup: - # 1. use 'rclone copy' to copy full backup, use 'rclone check' to create 'missing-on-src-full.txt' - # 2. use 'rclone copy' to copy incremental backup, use 'rclone check' to create 'missing-on-src-incr.txt' - # 3. create 'masbr-deletions.txt' based on 'masbr-changes.txt' and the intersect lines in - # 'missing-on-src-full.txt' and 'missing-on-src-incr.txt' - # 4. 'rclone delete' based on 'masbr-deletions.txt' - - name: "Set fact: type of the restoring backup" - set_fact: - _restoring_type: >- - {{ 'incr' if (masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0) else 'full' }} - - - name: "Set fact: command for copying files from cloud storage folder to pvc folder ({{ _restoring_type }})" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - set_fact: - masbr_cf_dest_folders: "{{ masbr_cf_dest_folders + [item.dest_folder] }}" - masbr_cf_cmds: >- - {{ masbr_cf_cmds }}; - mkdir -p {{ item.dest_folder }}; - {{ 'rm -rf ' + item.dest_folder + '/*;' if masbr_cf_delete_dest is defined and masbr_cf_delete_dest }} - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - {{ 'copy --no-traverse' if (masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0) - else 'sync' }} - {{ [masbr_storage_job_folder, item.src_folder] | path_join }} {{ item.dest_folder }} - 2>&1 | tee /tmp/logs/rclone-copy-{{ _restoring_type }}.log - loop: "{{ masbr_cf_paths_to_pvc }}" - - - name: "Set fact: command for copying files from cloud storage folder to pvc folder (full)" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - - masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0 - set_fact: - masbr_cf_dest_folders: "{{ masbr_cf_dest_folders + [item.dest_folder] }}" - masbr_cf_cmds: >- - {{ masbr_cf_cmds }}; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copy --no-traverse - {{ [masbr_storage_from_job_folder, item.src_folder] | path_join }} {{ item.dest_folder }} - 2>&1 | tee /tmp/logs/rclone-copy-full.log - loop: "{{ masbr_cf_paths_to_pvc }}" - - - name: "Set fact: command for creating changes file" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - - masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0 - set_fact: - masbr_cf_cmds: >- - {{ masbr_cf_cmds }}; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - check {{ [masbr_storage_job_folder, item.src_folder] | path_join }} {{ item.dest_folder }} - --missing-on-src /tmp/logs/missing-on-src-incr.txt - 2>&1 | tee /tmp/logs/rclone-check-incr.log; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - check {{ [masbr_storage_from_job_folder, item.src_folder] | path_join }} {{ item.dest_folder }} - --missing-on-src /tmp/logs/missing-on-src-full.txt - 2>&1 | tee /tmp/logs/rclone-check-full.log - loop: "{{ masbr_cf_paths_to_pvc }}" - - - name: "Set fact: command for deleting files" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - - masbr_cf_from_job_name is defined and masbr_cf_from_job_name | length > 0 - set_fact: - masbr_cf_cmds: >- - {{ masbr_cf_cmds }}; - mv {{ [item.dest_folder, 'masbr-changes.txt'] | path_join }} /tmp/logs/masbr-changes.txt; - cat /tmp/logs/masbr-changes.txt | grep "^-" | awk '{print $2}' > /tmp/logs/masbr-deletions.txt; - sort /tmp/logs/missing-on-src-full.txt /tmp/logs/missing-on-src-incr.txt - | uniq -d >> /tmp/logs/masbr-deletions.txt; - rclone --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - delete {{ item.dest_folder }} --files-from /tmp/logs/masbr-deletions.txt - 2>&1 | tee /tmp/logs/rclone-delete.log; - rclone --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - rmdirs {{ item.dest_folder }} --leave-root - 2>&1 | tee /tmp/logs/rclone-rmdirs.log - loop: "{{ masbr_cf_paths_to_pvc }}" - - # Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file - - name: "Set fact: copy file from cloud storage folder to pvc folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - set_fact: - masbr_cf_dest_folders: "{{ masbr_cf_dest_folders + [item.dest_folder] }}" - masbr_cf_cmds: >- - {{ masbr_cf_cmds }}; - mkdir -p {{ item.dest_folder }}; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copy --no-traverse - {{ [masbr_storage_job_folder, item.src_file] | path_join }} {{ item.dest_folder }} - 2>&1 | tee /tmp/logs/rclone-copy.log - loop: "{{ masbr_cf_paths_to_pvc }}" - - # Condition 3. src_file -> dest_file - - name: "Set fact: copy file from cloud storage to pvc file" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - set_fact: - masbr_cf_dest_folders: "{{ masbr_cf_dest_folders + [item.dest_file | dirname] }}" - masbr_cf_cmds: >- - {{ masbr_cf_cmds }}; - mkdir -p {{ item.dest_file | dirname }}; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copyto {{ [masbr_storage_job_folder, item.src_file] | path_join }} {{ item.dest_file }} - 2>&1 | tee /tmp/logs/rclone-copyto.log - loop: "{{ masbr_cf_paths_to_pvc }}" - - - name: "Set fact: command for copying logs" - set_fact: - masbr_cf_cmds: >- - {{ masbr_cf_cmds }}; - ls -lA /tmp/logs; - tar -czf /tmp/{{ masbr_cf_k8s_name }}-log.tar.gz -C /tmp/logs .; - rclone --links --progress --no-check-certificate --config ${MASBR_STORAGE_CLOUD_RCLONE_FILE} - copy /tmp/{{ masbr_cf_k8s_name }}-log.tar.gz - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_job_type }}s/{{ masbr_job_name_final }}/log - - - name: "Debug: rclone copy file command" - debug: - msg: "Rclone copy file command .......... {{ masbr_cf_cmds }}" - - - # Create copy file Job: cloud storage -> mounted pvc - # ------------------------------------------------------------------------- - # 1. The Job pod will mount the PVC where the files will be saved to - # 2. Use rclone to copy files from COS to PVC - - name: "Create copy file Job" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_copy_cloud_files_job.yml" - - - name: "List copied files in pvc paths" - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'ls -lA {{ item }}' - loop: "{{ masbr_cf_dest_folders }}" - register: _ls_pvc_paths_output - - - name: "Debug: list files in pvc paths" - debug: - msg: "{{ _ls_pvc_paths_output | json_query('results[*].stdout_lines') }}" - - - # Copy files: mounted pvc -> pod temp folder - # ------------------------------------------------------------------------- - - name: "Copy files from pvc to pod" - when: masbr_cf_are_pvc_paths is not defined or not masbr_cf_are_pvc_paths - block: - - name: "Set fact: list dest files in pod" - set_fact: - masbr_cf_ls_dest_files: [] - - - name: "Debug: paths of copying files from pvc to pod" - debug: - msg: "{{ masbr_cf_paths_from_pvc }}" - - # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* - - name: "Copy files from pvc folder to pod folder" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'mkdir -p {{ item.dest_folder }} && - cp -rf {{ item.src_folder }}/* {{ item.dest_folder }} && - ls -lA {{ item.dest_folder }}' - loop: "{{ masbr_cf_paths_from_pvc }}" - register: _ls_output - - - name: "Set fact: append dest files in pod" - set_fact: - masbr_cf_ls_dest_files: "{{ masbr_cf_ls_dest_files + _ls_output | json_query('results[*].stdout_lines') }}" - - # Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file - - name: "Copy file from pvc to pod folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'mkdir -p {{ item.dest_folder }} && - cp -r {{ item.src_file }} {{ item.dest_folder }} && - ls -lA {{ item.dest_folder }}' - loop: "{{ masbr_cf_paths_from_pvc }}" - register: _ls_output - - - name: "Set fact: append dest files in pod" - set_fact: - masbr_cf_ls_dest_files: "{{ masbr_cf_ls_dest_files + _ls_output | json_query('results[*].stdout_lines') }}" - - # Condition 3. src_file -> dest_file - - name: "Copy file from pvc to pod file" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'mkdir -p {{ item.dest_file | direname }} && - cp -r {{ item.src_file }} {{ item.dest_file }} && - ls -lA {{ item.dest_file | direname }}' - loop: "{{ masbr_cf_paths_from_pvc }}" - register: _ls_output - - - name: "Set fact: append dest files in pod" - set_fact: - masbr_cf_ls_dest_files: "{{ masbr_cf_ls_dest_files + _ls_output | json_query('results[*].stdout_lines') }}" - - - name: "Debug: list files in pod paths" - debug: - msg: "{{ masbr_cf_ls_dest_files }}" - - always: - # Clean up - - name: "Delete pvc job folder" - when: masbr_cf_pvc_job_folder is defined and masbr_cf_pvc_job_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c - 'rm -rf {{ masbr_cf_pvc_job_folder }}' - - -# Copy files from pvc storage to pod -# ----------------------------------------------------------------------------- +# Local storage job folder +- name: "Set fact: local storage job folder" + set_fact: + masbr_storage_job_folder: >- + {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} + +- name: "Debug: local storage job folder" + debug: + msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" + +# Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* +# +# - exec into masbr_cf_pod_name/masbr_cf_container_name, create temp folder +# - cp from src_folder to temp folder inside masbr_cf_pod_name/masbr_cf_container_name +# - exec into masbr_cf_pod_name/masbr_cf_container_name, move temp_dest_folder to dest_folder and delete temp_dest_folder +- name: "Copy files from local storage folder to pod folder" + when: + - item.src_folder is defined and item.src_folder | length > 0 + - item.dest_folder is defined and item.dest_folder | length > 0 + shell: >- + oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'temp_dest_folder={{ [item.dest_folder, masbr_job_version] | path_join }} && mkdir -p ${temp_dest_folder}' \ + && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_folder] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:${temp_dest_folder} \ + && oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mv -f ${temp_dest_folder}/* {{ item.dest_folder }} && rm -rf ${temp_dest_folder}' + loop: "{{ masbr_cf_paths }}" + +# Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file +- name: "Copy file from local storage folder to pod folder" + when: + - item.src_file is defined and item.src_file | length > 0 + - item.dest_folder is defined and item.dest_folder | length > 0 + shell: >- + oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mkdir -p {{ item.dest_folder }}' \ + && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_file] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.dest_folder }} + loop: "{{ masbr_cf_paths }}" + +# Condition 3. src_file -> dest_file +# - exec into masbr_cf_pod_name/masbr_cf_container_name, create temp folder +# - cp from src_folder to temp folder inside masbr_cf_pod_name/masbr_cf_container_name +# - exec into masbr_cf_pod_name/masbr_cf_container_name, move temp_dest_folder to dest_folder and delete temp_dest_folder +- name: "Copy file from local storage folder to pod file" + when: + - item.src_file is defined and item.src_file | length > 0 + - item.dest_file is defined and item.dest_file | length > 0 + shell: >- + oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'temp_dest_folder={{ [item.dest_file|dirname, masbr_job_version] | path_join }} && mkdir -p ${temp_dest_folder}' \ + && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_file] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:${temp_dest_folder} \ + && oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mv -f ${temp_dest_folder}/{{ item.src_file|basename }} {{ item.dest_file }} && rm -rf ${temp_dest_folder}' diff --git a/ibm/mas_devops/common_tasks/backup_restore/create_copy_cloud_files_job.yml b/ibm/mas_devops/common_tasks/backup_restore/create_copy_cloud_files_job.yml deleted file mode 100644 index 6770420375..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/create_copy_cloud_files_job.yml +++ /dev/null @@ -1,145 +0,0 @@ ---- -# Create k8s job to copy files between pod and cloud storage -# Related facts (name staring with 'masbr_cf_'): -# masbr_cf_job_type -# masbr_cf_job_name -# masbr_cf_from_job_name -# masbr_cf_namespace -# masbr_cf_are_pvc_paths: true|false -# masbr_cf_pod_name: only used when masbr_cf_are_pvc_paths=false -# masbr_cf_container_name: only used when masbr_cf_are_pvc_paths=false -# masbr_cf_affinity: true|false -# masbr_cf_pvc_name -# masbr_cf_pvc_mount_path -# masbr_cf_pvc_sub_path -# masbr_cf_paths -# masbr_cf_delete_dest: true|false, only used when coping files from cloud storage to pvc - - -# Get pod information -# ----------------------------------------------------------------------------- -- name: "Get pod information" - kubernetes.core.k8s_info: - api_version: v1 - kind: Pod - name: "{{ masbr_cf_pod_name }}" - namespace: "{{ masbr_cf_namespace }}" - register: _pod_info - -- name: "Set fact: pod infomation" - set_fact: - masbr_cf_service_account_name: "{{ _pod_info.resources[0].spec.serviceAccountName | default('') }}" - masbr_cf_service_account: "{{ _pod_info.resources[0].spec.serviceAccount | default('') }}" - masbr_cf_pod_security_context: "{{ _pod_info.resources[0].spec.securityContext }}" - masbr_cf_container_security_context: >- - {{ _pod_info.resources[0] | json_query('spec.containers[?name==`' + masbr_cf_container_name + '`].securityContext') | first }} - - -- name: "Debug: pod information" - debug: - msg: - - "Pod service account .................... {{ masbr_cf_service_account }}" - - "Pod service account name ............... {{ masbr_cf_service_account_name }}" - - "Pod security context ................... {{ masbr_cf_pod_security_context }}" - - "Container name ......................... {{ masbr_cf_container_name }}" - - "Container security context ............. {{ masbr_cf_container_security_context }}" - - -# Create ConfigMap -# ----------------------------------------------------------------------------- -- name: "Get rclone config" - shell: > - cat {{ masbr_storage_cloud_rclone_file }} - register: _rclone_config_content - -- name: "Create configmap to save rclone config" - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: "{{ masbr_cf_k8s_name }}" - namespace: "{{ masbr_cf_namespace }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "copy" - masbr-job: "{{ masbr_job_name }}" - data: - rclone.conf: "{{ _rclone_config_content.stdout }}" - wait: true - -- name: "Set fact: reset masbr_cf_env" - set_fact: - masbr_cf_env: [] - -- name: "Set fact: add rclone config file to env variables" - set_fact: - masbr_cf_env: >- - {{ masbr_cf_env + [ - {'name': 'MASBR_STORAGE_CLOUD_RCLONE_FILE', 'value': '/mnt/configmap/rclone.conf'}, - {'name': 'RCLONE_VERBOSE', 'value': '2'} - ] }} - - -# Create Job -# ----------------------------------------------------------------------------- -- name: "Create copying file Job" - kubernetes.core.k8s: - template: "{{ role_path }}/../../common_tasks/templates/backup_restore/copy_cloud_files_job.yml.j2" - state: present - wait: true - - -# Save Job log -# ----------------------------------------------------------------------------- -- name: "Save copy file Job log" - block: - - name: "Wait for Job to be Completed or Failed (10s delay)" - kubernetes.core.k8s_info: - api_version: batch/v1 - kind: Job - name: "{{ masbr_cf_k8s_name }}" - namespace: "{{ masbr_cf_namespace }}" - register: _job_info - until: - - _job_info.resources is defined - - _job_info.resources | length > 0 - - (_job_info.resources | json_query('[*].status.conditions[?type==`Complete`][].status') | select ('match','True') | list | length == 1) or - (_job_info.resources | json_query('[*].status.conditions[?type==`Failed`][].status') | select ('match','True') | list | length == 1) - retries: "{{ (masbr_copy_timeout_sec|int // 10) | int }}" - delay: 10 - - always: - - name: "Get Job pod information" - shell: >- - mkdir -p {{ masbr_local_job_folder }}/log/{{ masbr_cf_k8s_name }}; - pod_name=$(oc get pod -n {{ masbr_cf_namespace }} --no-headers=true -l job-name={{ masbr_cf_k8s_name }} | awk '{print $1}' | head -n 1); - oc describe -n {{ masbr_cf_namespace }} pod ${pod_name} > {{ masbr_local_job_folder }}/log/{{ masbr_cf_k8s_name }}/${pod_name}-describe.txt; - oc get -n {{ masbr_cf_namespace }} pod ${pod_name} -o yaml > {{ masbr_local_job_folder }}/log/{{ masbr_cf_k8s_name }}/${pod_name}.yaml; - ls -lA {{ masbr_local_job_folder }}/log/{{ masbr_cf_k8s_name }} - register: _get_log_files_output - - - name: "Debug: list Job log files" - debug: - msg: - - "Local job log folder ............... {{ masbr_local_job_folder }}/log/{{ masbr_cf_k8s_name }}" - - "{{ _get_log_files_output.stdout_lines }}" - - - name: "Set fact: copy file Job log name" - set_fact: - masbr_cf_log_file_name: >- - {{ masbr_cf_k8s_name }}-describe.tar.gz - - - name: "Create a tar.gz archive of all log files" - shell: >- - tar -czf {{ masbr_local_job_folder }}/log/{{ masbr_cf_log_file_name }} - -C {{ masbr_local_job_folder }}/log/{{ masbr_cf_k8s_name }} . - - - name: "Copy log file from local to storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "{{ masbr_job_type }}" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "log/{{ masbr_cf_log_file_name }}" - dest_folder: "log" diff --git a/ibm/mas_devops/common_tasks/backup_restore/create_run_tasks_job.yml b/ibm/mas_devops/common_tasks/backup_restore/create_run_tasks_job.yml deleted file mode 100644 index 94009bc97c..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/create_run_tasks_job.yml +++ /dev/null @@ -1,211 +0,0 @@ ---- -# Input parameters: -# _rt_playbook_name -# _rt_role_name -# _rt_env - - -# Set task variables -# ----------------------------------------------------------------------------- -- name: "Set fact: task variables" - set_fact: - masbr_rt_namespace: "{{ masbr_job_component.namespace }}" - -- name: "Set fact: task name (backup)" - when: masbr_task_type == "backup" - set_fact: - # Format '--' - # - Job name must be no more than 63 characters and in lower case: - # 'backup-full-20240508151053' - masbr_rt_k8s_name: >- - {{ masbr_task_type }}-{{ masbr_backup_type }}-{{ masbr_job_version }} - -- name: "Set fact: task name (schedule)" - when: masbr_task_type == "schedule" - set_fact: - # Format '--' - # - CronJob name must be no longer than 52 characters and in lower case: - # 'schedule-incr-20240424210110' - masbr_rt_k8s_name: >- - {{ masbr_task_type }}-{{ masbr_backup_type }}-{{ masbr_timestamp_format | strftime }} - -- name: "Set fact: task name (restore)" - when: masbr_task_type == "restore" - set_fact: - # Format '---' - # - Job name must be no more than 63 characters and in lower case: - # 'restore-full-20240508151704' - masbr_rt_k8s_name: >- - {{ masbr_task_type }}-{{ masbr_restore_from_yaml.type }}-{{ masbr_restore_from_version }}-{{ masbr_job_version }} - -- name: "Debug: run task Job name" - debug: - msg: "{{ masbr_rt_k8s_name }}" - -# masbr_rt_cmds -- name: "Set fact: run playbook command" - when: _rt_playbook_name is defined and _rt_playbook_name | length > 0 - set_fact: - masbr_rt_cmds: > - ansible-playbook ibm.mas_devops.{{ _rt_playbook_name }} - -- name: "Set fact: run role command" - when: _rt_role_name is defined and _rt_role_name | length > 0 - set_fact: - masbr_rt_cmds: > - ROLE_NAME={{ _rt_role_name }} ansible-playbook ibm.mas_devops.run_role - -- name: "Debug: run task Job command" - debug: - msg: "{{ masbr_rt_cmds }}" - -# masbr_rt_env -- name: "Set fact: default common env variables" - set_fact: - masbr_rt_common_env: - - name: "ANSIBLE_LOG_PATH" - value: "/tmp/ansible.log" - - name: "MASBR_CONFIRM_CLUSTER" - value: "false" - - name: "MASBR_CREATE_TASK_JOB" - value: "false" - - name: "MASBR_RUNNING_IN_TASK_JOB" - value: "true" - - name: "MASBR_TASK_JOB_NAMESPACE" - value: "{{ masbr_rt_namespace }}" - - name: "MASBR_TASK_JOB_NAME" - value: "{{ masbr_rt_k8s_name }}" - - name: "MASBR_ACTION" - value: "{{ masbr_action | default('backup') }}" - -- name: "Set fact: add additional common env variables (staring with 'MASBR_')" - set_fact: - masbr_rt_common_env: "{{ masbr_rt_common_env + [item] }}" - with_items: >- - {{ ansible_env | - ansible.utils.keep_keys(target=['MASBR_'], matching_parameter='starts_with') | - dict2items(key_name='name', value_name='value') }} - # Exclude below 'MASBR_' env vars because we will override them - when: >- - item.name not in [ - 'MASBR_CONFIRM_CLUSTER', - 'MASBR_CREATE_TASK_JOB', - 'MASBR_RUNNING_IN_TASK_JOB', - 'MASBR_TASK_JOB_NAMESPACE', - 'MASBR_TASK_JOB_NAME', - 'MASBR_STORAGE_CLOUD_RCLONE_FILE' - ] - -- name: "Set fact: add role specific env variables" - set_fact: - masbr_rt_env: "{{ masbr_rt_common_env + _rt_env | default([]) }}" - -- name: "Debug: run task Job env" - debug: - msg: "{{ masbr_rt_env }}" - - -# Create configmap to save rclone config -# ----------------------------------------------------------------------------- -- name: "Create configmap to save rclone config" - when: masbr_use_cloud_storage - block: - # rclone config - - name: "Get rclone config" - shell: > - cat {{ masbr_storage_cloud_rclone_file }} - register: _rclone_config_content - - - name: "Create configmap to save rclone config" - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: "{{ masbr_rt_k8s_name }}" - namespace: "{{ masbr_rt_namespace }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "{{ masbr_task_type}}" - masbr-job: "{{ masbr_job_name }}" - data: - rclone.conf: "{{ _rclone_config_content.stdout }}" - wait: true - - - name: "Set fact: add rclone config file to env variables" - set_fact: - masbr_rt_env: >- - {{ masbr_rt_env + [ - {'name': 'MASBR_STORAGE_CLOUD_RCLONE_FILE', 'value': '/mnt/configmap/rclone.conf'} - ] }} - - -# Create Job -# ----------------------------------------------------------------------------- -- name: "Create cleanup Job" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_cleanup_job.yml" - vars: - masbr_cleanup_namespace: "{{ masbr_rt_namespace }}" - -- name: "Create run tasks Job" - kubernetes.core.k8s: - template: "{{ role_path }}/../../common_tasks/templates/backup_restore/run_task_job.yml.j2" - state: present - wait: true - - -# ONLY FOR DEV -# ----------------------------------------------------------------------------- -- name: "Create env file for running tasks in docker" - when: __masbr_dev_create_env_file - block: - - name: "Create env file" - shell: > - rm -f /tmp/run_task_job.env; - touch /tmp/run_task_job.env - - - name: "Write env file" - shell: > - echo "{{ item.name }}={{ item.value }}" >> /tmp/run_task_job.env - loop: "{{ masbr_rt_env }}" - - - name: "Get OCP login command" - shell: > - echo "oc login --server=$(oc whoami --show-server) --token=$(oc whoami --show-token)" - register: _oc_login_output - - - name: "Run docker command" - debug: - msg: - - >- - Step 1: podman run -ti -v {{ masbr_storage_cloud_rclone_file | dirname }}:/mnt/configmap - --env-file=/tmp/run_task_job.env quay.io/ibmmas/cli:local - - "Step 2: {{ _oc_login_output.stdout }}" - - "Step 3: {{ masbr_rt_cmds }}" - - -# Display information of the created task -# ----------------------------------------------------------------------------- -- name: "Get OCP URL" - shell: >- - echo "$(oc whoami --show-console)/k8s/ns/{{ masbr_rt_namespace }}/{{ 'cronjobs' - if masbr_task_type == 'schedule' else 'jobs' }}/{{ masbr_rt_k8s_name }}" - register: _oc_url_output - -- name: "Summary of backup job" - when: masbr_job_type == "backup" - debug: - msg: - - "Backup version ..................... {{ masbr_job_version }}" - - "Backup from ........................ {{ masbr_backup_from | default('', true) }}" - - "Job name ........................... {{ masbr_rt_k8s_name }}" - - "Job link ........................... {{ _oc_url_output.stdout }}" - -- name: "Summary of restore job" - when: masbr_job_type == "restore" - debug: - msg: - - "Restore version .................... {{ masbr_job_version }}" - - "Restore from ....................... {{ masbr_restore_from | default('', true) }}" - - "Job name ........................... {{ masbr_rt_k8s_name }}" - - "Job link ........................... {{ _oc_url_output.stdout }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/delete_storage_job_folder.yml b/ibm/mas_devops/common_tasks/backup_restore/delete_storage_job_folder.yml index 29902f8431..60d6a71fa0 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/delete_storage_job_folder.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/delete_storage_job_folder.yml @@ -1,41 +1,15 @@ --- # Delete the job folder from local storage # ----------------------------------------------------------------------------- -- name: "Delete the job folder from local storage" - when: masbr_use_local_storage - block: - - name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name }}" - - - name: "Debug: local storage job folder" - debug: - msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" - - - name: "Delete the job folder from local storage" - command: rm -rf {{ masbr_storage_job_folder }} - args: - removes: "{{ masbr_storage_job_folder }}" - - -# Delete the job folder from cloud storage -# ----------------------------------------------------------------------------- -- name: "Delete the job folder from cloud storage" - when: masbr_use_cloud_storage - block: - - name: "Set fact: cloud storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_job_type }}s/{{ masbr_job_name }} - - - name: "Debug: cloud storage job folder" - debug: - msg: "Cloud storage job folder .......... {{ masbr_storage_job_folder }}" - - - name: "Delete the job folder from cloud storage" - shell: > - rclone --config {{ masbr_storage_cloud_rclone_file }} delete {{ masbr_storage_job_folder }} +- name: "Set fact: local storage job folder" + set_fact: + masbr_storage_job_folder: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name }}" +- name: "Debug: local storage job folder" + debug: + msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" -# Delete the job folder from pvc storage -# ----------------------------------------------------------------------------- +- name: "Delete the job folder from local storage" + command: rm -rf {{ masbr_storage_job_folder }} + args: + removes: "{{ masbr_storage_job_folder }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/list_storage_job_folders.yml b/ibm/mas_devops/common_tasks/backup_restore/list_storage_job_folders.yml index 45916544e0..7115813af6 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/list_storage_job_folders.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/list_storage_job_folders.yml @@ -1,67 +1,27 @@ --- # List job folders in local storage # ----------------------------------------------------------------------------- -- name: "List job folders in local storage" - when: masbr_use_local_storage - block: - - name: "Set fact: local storage job type folder" - set_fact: - masbr_storage_job_type_folder: "{{ masbr_storage_local_folder }}/{{ masbr_ls_job_type }}s" - - - name: "Debug: list job folders variables" - debug: - msg: - - "Search folder ...................... {{ masbr_storage_job_type_folder }}" - - "Search filter ...................... {{ masbr_ls_filter | default('', true) }}" - - - name: "List job folders in local storage" - changed_when: false - shell: >- - ls {{ masbr_storage_job_type_folder }} {{ masbr_ls_filter | default('') }}; - exit 0 - register: _ls_output - - - name: "Set fact: results of list job folders" - set_fact: - masbr_ls_results: "{{ _ls_output.stdout_lines }}" - - - name: "Debug: results of list job folders" - debug: - msg: "Results of list job folders ....... {{ masbr_ls_results }}" - - -# List job folders in cloud storage -# ----------------------------------------------------------------------------- -- name: "List job folders in cloud storage" - when: masbr_use_cloud_storage - block: - - name: "Set fact: cloud storage job type folder" - set_fact: - masbr_storage_job_type_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_ls_job_type }}s - - - name: "Debug: list job folders variables" - debug: - msg: - - "Search folder ...................... {{ masbr_storage_job_type_folder }}" - - "Search filter ...................... {{ masbr_ls_filter | default('', true) }}" - - - name: "List job folders in cloud storage" - changed_when: false - shell: >- - rclone lsd --no-check-certificate --config {{ masbr_storage_cloud_rclone_file }} - {{ masbr_storage_job_type_folder }} | awk '{print $5}' {{ masbr_ls_filter | default('') }}; - exit 0 - register: _ls_output - - - name: "Set fact: results of list job folders" - set_fact: - masbr_ls_results: "{{ _ls_output.stdout_lines }}" +- name: "Set fact: local storage job type folder" + set_fact: + masbr_storage_job_type_folder: "{{ masbr_storage_local_folder }}/{{ masbr_ls_job_type }}s" - - name: "Debug: results of list job folders" - debug: - msg: "Results of list job folders ....... {{ masbr_ls_results }}" +- name: "Debug: list job folders variables" + debug: + msg: + - "Search folder ...................... {{ masbr_storage_job_type_folder }}" + - "Search filter ...................... {{ masbr_ls_filter | default('', true) }}" - -# List job folders in pvc storage -# ----------------------------------------------------------------------------- +- name: "List job folders in local storage" + changed_when: false + shell: >- + ls {{ masbr_storage_job_type_folder }} {{ masbr_ls_filter | default('') }}; + exit 0 + register: _ls_output + +- name: "Set fact: results of list job folders" + set_fact: + masbr_ls_results: "{{ _ls_output.stdout_lines }}" + +- name: "Debug: results of list job folders" + debug: + msg: "Results of list job folders ....... {{ masbr_ls_results }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml b/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml index 5e7d046018..fa271acc47 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml @@ -6,50 +6,19 @@ # Rename the job folder in local storage # ----------------------------------------------------------------------------- -- name: "Rename the job folder in local storage" - when: masbr_use_local_storage - block: - - name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name }}" - masbr_storage_job_folder_final: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name_final }}" - - - name: "Debug: rename local storage job folder" - debug: - msg: - - "Source job folder .................. {{ masbr_storage_job_folder }}" - - "Dest job folder .................... {{ masbr_storage_job_folder_final }}" - - - name: "Rename the job folder in local storage" - command: mv {{ masbr_storage_job_folder }} {{ masbr_storage_job_folder_final }} - args: - removes: "{{ masbr_storage_job_folder }}" - creates: "{{ masbr_storage_job_folder_final }}" - - -# Rename the job folder in cloud storage -# ----------------------------------------------------------------------------- -- name: "Rename the job folder in cloud storage" - when: masbr_use_cloud_storage - block: - - name: "Set fact: cloud storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_job_type }}s/{{ masbr_job_name }} - masbr_storage_job_folder_final: >- - {{ masbr_storage_cloud_rclone_name }}:{{ masbr_storage_cloud_bucket }}/{{ masbr_job_type }}s/{{ masbr_job_name_final }} - - - name: "Debug: rename cloud storage job folder" - debug: - msg: - - "Source job folder .................. {{ masbr_storage_job_folder }}" - - "Dest job folder .................... {{ masbr_storage_job_folder_final }}" - - - name: "Rename the job folder in cloud storage" - shell: >- - rclone --config {{ masbr_storage_cloud_rclone_file }} - move --no-traverse {{ masbr_storage_job_folder }} {{ masbr_storage_job_folder_final }} +- name: "Set fact: local storage job folder" + set_fact: + masbr_storage_job_folder: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name }}" + masbr_storage_job_folder_final: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name_final }}" +- name: "Debug: rename local storage job folder" + debug: + msg: + - "Source job folder .................. {{ masbr_storage_job_folder }}" + - "Dest job folder .................... {{ masbr_storage_job_folder_final }}" -# Rename the job folder in pvc storage -# ----------------------------------------------------------------------------- +- name: "Rename the job folder in local storage" + command: mv {{ masbr_storage_job_folder }} {{ masbr_storage_job_folder_final }} + args: + removes: "{{ masbr_storage_job_folder }}" + creates: "{{ masbr_storage_job_folder_final }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/send_notification.yml b/ibm/mas_devops/common_tasks/backup_restore/send_notification.yml deleted file mode 100644 index c7e97b1524..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/send_notification.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -# Send Slack notification -# ----------------------------------------------------------------------------- -- name: "Send Slack notification" - when: - - masbr_slack_enabled - - masbr_job_status.phase in masbr_notification_levels[masbr_slack_level] - block: - - name: "Set fact: Slack message" - set_fact: - slack_msg: | - {{ masbr_job_type | capitalize }} {{ masbr_job_component.name }} is {{ masbr_job_status.phase }} - Job name: `{{ masbr_job_name }}` - slack_color: >- - {% if masbr_job_status.phase == 'InProgress' %} warning - {% elif masbr_job_status.phase == 'Completed' %} good - {% elif masbr_job_status.phase in ['Failed', 'PartiallyFailed'] %} danger - {% endif %} - - - name: "Send Slack message" - community.general.slack: - token: "{{ masbr_slack_token }}" - channel: "{{ masbr_slack_channel }}" - msg: "{{ slack_msg }}" - color: "{{ slack_color | trim }}" - - - name: "Set fact: append to masbr_job_status.sentNotifications (Slack)" - set_fact: - masbr_sent_notifications: >- - {{ masbr_sent_notifications + [{ - "type": "Slack", - "channel": masbr_slack_channel, - "timestamp": "%Y-%m-%dT%H:%M:%S" | strftime, - "phase": masbr_job_status.phase - }] }} diff --git a/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml b/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml index 3562a588e2..fd61b8675c 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml @@ -125,28 +125,6 @@ }) }} -# Send notification -# ----------------------------------------------------------------------------- -- name: "Send notification" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/send_notification.yml" - -- name: "Update fact: masbr_job_status.sentNotifications" - when: - - masbr_sent_notifications is defined - - masbr_sent_notifications | length > 0 - block: - - name: "Update fact: masbr_job_status.sentNotifications" - ansible.utils.update_fact: - updates: - - path: masbr_job_status.sentNotifications - value: "{{ masbr_sent_notifications }}" - register: _job_status_updated - - - name: "Set fact: masbr_job_status.sentNotifications" - set_fact: - masbr_job_status: "{{ _job_status_updated.masbr_job_status }}" - - # Create job file # ----------------------------------------------------------------------------- - name: "Debug: update job variables" diff --git a/ibm/mas_devops/common_tasks/create_subscription.yml b/ibm/mas_devops/common_tasks/create_subscription.yml deleted file mode 100644 index b0e17f567f..0000000000 --- a/ibm/mas_devops/common_tasks/create_subscription.yml +++ /dev/null @@ -1,141 +0,0 @@ ---- - -# Usage Examples -# ----------------------------------------------------------------------------- -# - name: Create Red Hat Certificate Manager Subscription -# include_tasks: "{{ role_path }}/../../common_tasks/create_subscription.yml" -# vars: -# subscription_namespace: "{{ cert_manager_operator_namespace }}" -# package_name: openshift-cert-manager-operator -# channel_name: stable-v1 - -# Optionally provide a config object as well: -# -# - name: "install : Create Grafana v5 Subscription" -# include_tasks: "{{ role_path }}/../../common_tasks/create_subscription.yml" -# vars: -# package_name: grafana-operator -# channel_name: v5 -# subscription_namespace: "{{ grafana_v5_namespace }}" -# subscription_config: -# env: -# - name: "WATCH_NAMESPACE" -# value: "" -# - name: "DASHBOARD_NAMESPACES_ALL" -# value: "true" - - -# 1. Validate inputs -# ----------------------------------------------------------------------------- -- name: "create_subscription: Assert that PackageManifest exists" - ansible.builtin.assert: - that: - - package_name is defined and package_name != "" - - subscription_namespace is defined and subscription_namespace != "" - fail_msg: "Unable to create subscription: package_name and subscription_namespace variables must both be set" - - -# 2. Lookup Required Subscription Information -# ----------------------------------------------------------------------------- -- name: "create_subscription : Get package manifest" - kubernetes.core.k8s_info: - api_version: packages.operators.coreos.com/v1 - kind: PackageManifest - name: "{{ package_name }}" - # Note: A namespace must be provided when calling packages.operators.coreos.com/v1 - namespace: openshift-marketplace - register: _manifest - -- name: "create_subscription: Assert that PackageManifest exists" - ansible.builtin.assert: - that: - - _manifest is defined - - _manifest.resources is defined - - _manifest.resources | length == 1 - fail_msg: "Unable to create subscription, packageManifest not found: {{ package_name }}" - -- name: "create_subscription : Set the subscription information" - set_fact: - _source: "{{ _manifest.resources[0].status.catalogSource }}" - _source_namespace: "{{ _manifest.resources[0].status.catalogSourceNamespace }}" - _default_channel: "{{ _manifest.resources[0].status.defaultChannel }}" - -- name: "create_subscription: Use default channel if none is provided" - when: channel_name is not defined or channel_name == "" - set_fact: - channel_name: "{{ _default_channel }}" - - -# 3. Determine whether a new OperatorGroup must be created -# ----------------------------------------------------------------------------- -- name: "create_subscription: Check if operator group is present in {{ subscription_namespace }} namespace" - kubernetes.core.k8s_info: - namespace: "{{ subscription_namespace }}" - kind: OperatorGroup - register: _og_info - - -# 4. Create Subscription -# ----------------------------------------------------------------------------- -- name: "create_subscription: Debug settings" - debug: - msg: - - "Package Name ............................ {{ package_name }}" - - "Namespace ............................... {{ subscription_namespace }}" - - "OperatorGroup Exists .................... {{ _og_info.resources | length != 0 }}" - - "Catalog Source .......................... {{ _source }}" - - "Catalog Source Namespace ................ {{ _source_namespace }}" - - "Default Channel ......................... {{ _default_channel }}" - - "Channel ................................. {{ channel_name }}" - - "Configuration ........................... {{ subscription_config | default('', True) }}" - -- name: "create_subscription: Create {{ package_name }} subscription in {{ subscription_namespace }}" - kubernetes.core.k8s: - template: "templates/subscription.yml.j2" - wait: yes - wait_timeout: 120 - - -# 5. Wait for InstallPlan to complete -# ----------------------------------------------------------------------------- -- name: "create_subscription : Lookup InstallPlan" - kubernetes.core.k8s_info: - api_version: operators.coreos.com/v1alpha1 - kind: InstallPlan - namespace: "{{ subscription_namespace }}" - label_selectors: - - operators.coreos.com/{{ package_name }}.{{ subscription_namespace }} - register: _installplan_info - retries: 30 - delay: 60 # Retry for approx 30 minutes (60s * 30 attempts) before giving up - until: _installplan_info.resources | length > 0 - -- name: "create_subscription : Wait for InstallPlan to complete" - when: _installplan_info.resources[0].status.phase != "Complete" - kubernetes.core.k8s_info: - api_version: operators.coreos.com/v1alpha1 - kind: InstallPlan - name: "{{ _installplan_info.resources[0].metadata.name }}" - namespace: "{{ subscription_namespace }}" - register: _installplan_info - retries: 20 - delay: 30 # Retry for approx 10 minutes (20s * 30 attempts) before giving up - until: - - _installplan_info.resources[0].status.phase is defined - - _installplan_info.resources[0].status.phase == "Complete" - -# 5. Wait for Subscription to complete -# ----------------------------------------------------------------------------- -- name: "create_subscription : Wait for Subscription state to become 'AtLatestKnown'" - kubernetes.core.k8s_info: - api_version: operators.coreos.com/v1alpha1 - kind: Subscription - namespace: "{{ subscription_namespace }}" - name: "{{ package_name }}" - register: _subscription_info - retries: 30 - delay: 30 # Retry for approx 15 minutes (30s * 30 attempts) before giving up - until: - - _subscription_info.resources | length > 0 - - _subscription_info.resources[0].status.state is defined - - _subscription_info.resources[0].status.state == "AtLatestKnown" diff --git a/ibm/mas_devops/common_tasks/default_storage_classes.yml b/ibm/mas_devops/common_tasks/default_storage_classes.yml new file mode 100644 index 0000000000..f31bc5c7cd --- /dev/null +++ b/ibm/mas_devops/common_tasks/default_storage_classes.yml @@ -0,0 +1,15 @@ +--- + +# Lookup & use default supported storage class +# ----------------------------------------------------------------------------- +# See: ibm/mas_devops/plugins/action/get_default_storage_classes.py +- name: "default-storage-classes : Lookup default storage classes" + ibm.mas_devops.get_default_storage_classes: + register: defaultStorageClasses + +- name: "default-storage-classes : Debug default storage classes" + debug: + msg: + - "Provider ............................... {{ defaultStorageClasses.provider }}" + - "Default RWX ............................ {{ defaultStorageClasses.rwx }}" + - "Default RWO ............................ {{ defaultStorageClasses.rwo }}" diff --git a/ibm/mas_devops/common_tasks/get_version_from_channel.yml b/ibm/mas_devops/common_tasks/get_version_from_channel.yml index 58caeff5fd..8700f298f7 100644 --- a/ibm/mas_devops/common_tasks/get_version_from_channel.yml +++ b/ibm/mas_devops/common_tasks/get_version_from_channel.yml @@ -19,6 +19,7 @@ name: "{{ op_pm_name }}" namespace: openshift-marketplace register: op_pm_lookup + no_log: true # This generates a huge amount of (useless) logging - name: Assert that PackageManifest exists ansible.builtin.assert: diff --git a/ibm/mas_devops/common_tasks/templates/backup_restore/run_task_job.yml.j2 b/ibm/mas_devops/common_tasks/templates/backup_restore/run_task_job.yml.j2 deleted file mode 100644 index 767b34c708..0000000000 --- a/ibm/mas_devops/common_tasks/templates/backup_restore/run_task_job.yml.j2 +++ /dev/null @@ -1,149 +0,0 @@ ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: "masbr-sa" - namespace: "{{ masbr_rt_namespace }}" - labels: - mas.ibm.com/masbr: "" - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: "masbr-{{ masbr_rt_namespace }}" - labels: - mas.ibm.com/masbr: "" -subjects: - - kind: ServiceAccount - name: "masbr-sa" - namespace: "{{ masbr_rt_namespace }}" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin - -{% if masbr_use_cloud_storage %} ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: "masbr-network-policy" - namespace: "{{ masbr_rt_namespace }}" - labels: - mas.ibm.com/masbr: "" -spec: - podSelector: - matchLabels: - mas.ibm.com/masbr: "" - egress: - - {} - policyTypes: - - Egress -{% endif %} - -{% if masbr_task_type in ['backup', 'restore'] %} ---- -kind: Job -apiVersion: batch/v1 -metadata: - name: "{{ masbr_rt_k8s_name }}" - namespace: "{{ masbr_rt_namespace }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "{{ masbr_task_type}}" - masbr-job: "{{ masbr_job_name }}" -spec: - backoffLimit: 1 - template: - metadata: - name: "{{ masbr_rt_k8s_name }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "{{ masbr_task_type}}" - masbr-job: "{{ masbr_job_name }}" - spec: - serviceAccountName: masbr-sa - containers: - - name: main - image: quay.io/ibmmas/cli:{{ masbr_mascli_image_tag }} -{% if masbr_mascli_image_pull_policy is defined and masbr_mascli_image_pull_policy | length > 0 %} - imagePullPolicy: "{{ masbr_mascli_image_pull_policy }}" -{% endif %} - command: - - sh - - '-c' - - >- - {{ masbr_rt_cmds }} -{% if masbr_rt_env is defined and masbr_rt_env | length > 0 %} - env: {{ masbr_rt_env }} -{% endif %} -{% if masbr_use_cloud_storage %} - volumeMounts: - - name: cm-volume - readOnly: true - mountPath: /mnt/configmap -{% endif %} - restartPolicy: Never -{% if masbr_use_cloud_storage %} - volumes: - - name: cm-volume - configMap: - name: "{{ masbr_rt_k8s_name }}" -{% endif %} -{% endif %} - -{% if masbr_task_type == 'schedule' %} ---- -kind: CronJob -apiVersion: batch/v1 -metadata: - name: "{{ masbr_rt_k8s_name }}" - namespace: "{{ masbr_rt_namespace }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "{{ masbr_task_type}}" -spec: - schedule: "{{ masbr_backup_schedule }}" -{% if masbr_job_timezone is defined and masbr_job_timezone | length > 0 %} - timeZone: "{{ masbr_job_timezone }}" -{% endif %} - jobTemplate: - spec: - backoffLimit: 1 - template: - metadata: - name: "{{ masbr_rt_k8s_name }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "{{ masbr_task_type}}" - spec: - serviceAccountName: masbr-sa - containers: - - name: main - image: quay.io/ibmmas/cli:{{ masbr_mascli_image_tag }} -{% if masbr_mascli_image_pull_policy is defined and masbr_mascli_image_pull_policy | length > 0 %} - imagePullPolicy: "{{ masbr_mascli_image_pull_policy }}" -{% endif %} - command: - - sh - - '-c' - - >- - {{ masbr_rt_cmds }} -{% if masbr_rt_env is defined and masbr_rt_env | length > 0 %} - env: {{ masbr_rt_env }} -{% endif %} -{% if masbr_use_cloud_storage %} - volumeMounts: - - name: cm-volume - readOnly: true - mountPath: /mnt/configmap -{% endif %} - restartPolicy: Never -{% if masbr_use_cloud_storage %} - volumes: - - name: cm-volume - configMap: - name: "{{ masbr_rt_k8s_name }}" -{% endif %} -{% endif %} diff --git a/ibm/mas_devops/common_tasks/wait_for_crd.yml b/ibm/mas_devops/common_tasks/wait_for_crd.yml index 6eece69111..e78ecfecb5 100644 --- a/ibm/mas_devops/common_tasks/wait_for_crd.yml +++ b/ibm/mas_devops/common_tasks/wait_for_crd.yml @@ -25,3 +25,4 @@ until: - _crd_info.resources is defined - _crd_info.resources | length > 0 + no_log: true # This generates a huge amount of (useless) logging diff --git a/ibm/mas_devops/common_vars/backup_restore.yml b/ibm/mas_devops/common_vars/backup_restore.yml index 5c4788aeaa..32f0140fb1 100644 --- a/ibm/mas_devops/common_vars/backup_restore.yml +++ b/ibm/mas_devops/common_vars/backup_restore.yml @@ -32,22 +32,8 @@ masbr_mascli_image_pull_policy: "{{ lookup('env', 'MASBR_MASCLI_IMAGE_PULL_POLIC # ----------------------------------------------------------------------------- # Local temp folder for backup/restore masbr_local_temp_folder: "{{ lookup('env', 'MASBR_LOCAL_TEMP_FOLDER') | default('/tmp/masbr', true) }}" - -# -masbr_storage_type: "{{ lookup('env', 'MASBR_STORAGE_TYPE') }}" - -# when masbr_storage_type='local' masbr_storage_local_folder: "{{ lookup('env', 'MASBR_STORAGE_LOCAL_FOLDER') }}" -# when masbr_storage_type='pvc' -masbr_storage_pvc_name: "{{ lookup('env', 'MASBR_STORAGE_PVC_NAME') }}" -masbr_storage_pvc_path: "{{ lookup('env', 'MASBR_STORAGE_PVC_PATH') | default('masbr', true) }}" - -# when masbr_storage_type='cloud' -masbr_storage_cloud_rclone_file: "{{ lookup('env', 'MASBR_STORAGE_CLOUD_RCLONE_FILE') }}" -masbr_storage_cloud_rclone_name: "{{ lookup('env', 'MASBR_STORAGE_CLOUD_RCLONE_NAME') }}" -masbr_storage_cloud_bucket: "{{ lookup('env', 'MASBR_STORAGE_CLOUD_BUCKET') }}" - # Notification variables # ----------------------------------------------------------------------------- @@ -68,10 +54,3 @@ masbr_notification_levels: failure: - "Failed" - "PartiallyFailed" - -# Slack notification -masbr_slack_enabled: "{{ lookup('env', 'MASBR_SLACK_ENABLED') | default(false, true) | bool }}" -masbr_slack_level: "{{ lookup('env', 'MASBR_SLACK_LEVEL') | default('info', true) }}" -masbr_slack_token: "{{ lookup('env', 'MASBR_SLACK_TOKEN') }}" -masbr_slack_channel: "{{ lookup('env', 'MASBR_SLACK_CHANNEL') }}" -masbr_slack_user: "{{ lookup('env', 'MASBR_SLACK_USER') | default('MASBR', true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230414-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230414-amd64.yml deleted file mode 100644 index d7942f13da..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230414-amd64.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230411 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:4e7a81ee11bd0667f1cadc1ea1da44865e412fb0597186cfc8baa9ceb3015592 - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.0 # Operator version 3.23.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.1 # Operator version 110508.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.4.0 # Operator version 4.4.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.9 # Operator version 2.0.9 -sls_version: 3.6.0 # Operator version 3.6.0 -tsm_version: 1.5.0 # Operator version 1.5.0 -dd_version: 1.1.3 # Operator version 1.1.3 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.7.0 # Operator version 3.6.0 -wsl_version: 6.3.0 # Operator version 6.3.0 -wml_version: 6.3.0 # Operator version 3.3.0 -spark_version: 6.3.0 # Operator version 3.3.0 -wd_version: 5.3.0 # Operator version 4.6.3 -cognos_version: 23.3.0 # Operator version 23.3.0 - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.9.x: 8.9.4 # Updated - 8.10.x: 8.10.0 # No Update -mas_assist_version: - 8.9.x: 8.6.3 # Updated - 8.10.x: 8.7.0 # No Update -mas_hputilities_version: - 8.9.x: 8.5.1 # Updated - 8.10.x: 8.6.0 # Updated -mas_iot_version: - 8.9.x: 8.6.5 # Updated - 8.10.x: 8.7.0 # No Update -mas_manage_version: - 8.9.x: 8.5.4 # Updated - 8.10.x: 8.6.0 # No Update -mas_monitor_version: - 8.9.x: 8.9.4 # No Update - 8.10.x: 8.10.0 # No Update -mas_optimizer_version: - 8.9.x: 8.3.3 # Updated - 8.10.x: 8.4.0 # No Update -mas_predict_version: - 8.9.x: 8.7.2 # Updated - 8.10.x: 8.8.0 # No Update -mas_visualinspection_version: - 8.9.x: 8.7.0 # No Update - 8.10.x: 8.8.0 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.2.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.3 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230518-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230518-amd64.yml deleted file mode 100644 index 4d667cd90f..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230518-amd64.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230518 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:14cba8ea86a045901cc506f260e4f7aaa5d7a60a1922b927c30353f55e1c5cec - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.0 # Operator version 3.23.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.1 # Operator version 110508.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.4.0 # Operator version 4.4.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.9 # Operator version 2.0.9 -sls_version: 3.7.0 # Operator version 3.7.0 -tsm_version: 1.5.0 # Operator version 1.5.0 -dd_version: 1.1.4 # Operator version 1.1.4 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.7.0 # Operator version 3.6.0 -wsl_version: 6.3.0 # Operator version 6.3.0 -wml_version: 6.3.0 # Operator version 3.3.0 -spark_version: 6.3.0 # Operator version 3.3.0 -wd_version: 5.3.0 # Operator version 4.6.3 -cognos_version: 23.3.0 # Operator version 23.3.0 - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.9.x: 8.9.5 # Updated - 8.10.x: 8.10.1 # Updated -mas_assist_version: - 8.9.x: 8.6.3 # No Update - 8.10.x: 8.7.0 # No Update -mas_hputilities_version: - 8.9.x: 8.5.1 # No Update - 8.10.x: 8.6.1 # Updated -mas_iot_version: - 8.9.x: 8.6.6 # Updated - 8.10.x: 8.7.1 # Updated -mas_manage_version: - 8.9.x: 8.5.5 # Updated - 8.10.x: 8.6.1 # Updated -mas_monitor_version: - 8.9.x: 8.9.5 # Updated - 8.10.x: 8.10.1 # Updated -mas_optimizer_version: - 8.9.x: 8.3.3 # No Update - 8.10.x: 8.4.1 # Updated -mas_predict_version: - 8.9.x: 8.7.2 # No Update - 8.10.x: 8.8.0 # No Update -mas_visualinspection_version: - 8.9.x: 8.7.1 # Updated - 8.10.x: 8.8.0 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.2.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.3 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230526-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230526-amd64.yml deleted file mode 100644 index ba27b89d71..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230526-amd64.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230518 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:bdae03cfd469399d29a962448dcf790dcf81d309c4bc233637c3acecf228aa1f - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.0 # Operator version 3.23.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.1 # Operator version 110508.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.4.0 # Operator version 4.4.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.9 # Operator version 2.0.9 -sls_version: 3.7.0 # Operator version 3.7.0 -tsm_version: 1.5.0 # Operator version 1.5.0 -dd_version: 1.1.4 # Operator version 1.1.4 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.7.0 # Operator version 3.6.0 -wsl_version: 6.3.0 # Operator version 6.3.0 -wml_version: 6.3.0 # Operator version 3.3.0 -spark_version: 6.3.0 # Operator version 3.3.0 -wd_version: 5.3.0 # Operator version 4.6.3 -cognos_version: 23.3.0 # Operator version 23.3.0 - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.9.x: 8.9.5 # Updated - 8.10.x: 8.10.1 # Updated -mas_assist_version: - 8.9.x: 8.6.3 # No Update - 8.10.x: 8.7.0 # No Update -mas_hputilities_version: - 8.9.x: 8.5.1 # No Update - 8.10.x: 8.6.1 # Updated -mas_iot_version: - 8.9.x: 8.6.6 # Updated - 8.10.x: 8.7.1 # Updated -mas_manage_version: - 8.9.x: 8.5.5 # Updated - 8.10.x: 8.6.1 # Updated -mas_monitor_version: - 8.9.x: 8.9.5 # Updated - 8.10.x: 8.10.2 # Updated -mas_optimizer_version: - 8.9.x: 8.3.3 # No Update - 8.10.x: 8.4.1 # Updated -mas_predict_version: - 8.9.x: 8.7.2 # No Update - 8.10.x: 8.8.1 # Updated -mas_visualinspection_version: - 8.9.x: 8.7.1 # Updated - 8.10.x: 8.8.0 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.2.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.3 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230616-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230616-amd64.yml deleted file mode 100644 index 2ebbede3e1..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230616-amd64.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230616 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:cb0a38132a1e16964d9aa9b7eb5d247543df5feea218bb1100757290a07bc042 - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.2 # Operator version 3.23.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.1 # Operator version 110508.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.0 # Operator version 4.6.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.10 # Operator version 2.0.10 -sls_version: 3.7.0 # Operator version 3.7.0 -tsm_version: 1.5.0 # Operator version 1.5.0 -dd_version: 1.1.4 # Operator version 1.1.4 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.8.0+20230417.110116 # Operator version 3.7.0 -wsl_version: 6.4.0 # Operator version 6.4.0 -wml_version: 6.4.0 # Operator version 3.4.0 -spark_version: 6.4.0 # Operator version 3.4.0 -wd_version: 5.3.0 # Operator version 4.6.3 -cognos_version: 23.4.0 # Operator version 23.4.0 - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.9.x: 8.9.6 # Updated - 8.10.x: 8.10.2 # Updated -mas_assist_version: - 8.9.x: 8.6.4 # Updated - 8.10.x: 8.7.1 # Updated -mas_hputilities_version: - 8.9.x: 8.5.1 # No Update - 8.10.x: 8.6.1 # No Update -mas_iot_version: - 8.9.x: 8.6.7 # Updated - 8.10.x: 8.7.2 # Updated -mas_manage_version: - 8.9.x: 8.5.6 # Updated - 8.10.x: 8.6.2 # Updated -mas_monitor_version: - 8.9.x: 8.9.5 # No Update - 8.10.x: 8.10.3 # Updated -mas_optimizer_version: - 8.9.x: 8.3.3 # No Update - 8.10.x: 8.4.1 # No Update -mas_predict_version: - 8.9.x: 8.7.2 # No Update - 8.10.x: 8.8.1 # No Update -mas_visualinspection_version: - 8.9.x: 8.7.1 # No Update - 8.10.x: 8.8.1 # Updated - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.3.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.4 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230627-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230627-amd64.yml deleted file mode 100644 index 184dcf824c..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230627-amd64.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230627 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:42891d978163c24737f799ed870ad340f5b3e6cd9b14644eaf1c5810ed5ef7cf - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.2 # Operator version 3.23.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.1 # Operator version 110508.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.0 # Operator version 4.6.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.10 # Operator version 2.0.10 -sls_version: 3.7.0 # Operator version 3.7.0 -tsm_version: 1.5.0 # Operator version 1.5.0 -dd_version: 1.1.4 # Operator version 1.1.4 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.8.0+20230417.110116 # Operator version 3.7.0 -wsl_version: 6.4.0 # Operator version 6.4.0 -wml_version: 6.4.0 # Operator version 3.4.0 -spark_version: 6.4.0 # Operator version 3.4.0 -cognos_version: 23.4.0 # Operator version 23.4.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.3.0 # Operator version 4.6.3 -model_train_version: 1.2.5 # Operator version 1.1.7 -elasticsearch_version: 1.1.1336 # Operator version 1.1.1336 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.9.x: 8.9.7 # Updated - 8.10.x: 8.10.2 # No Update -mas_assist_version: - 8.9.x: 8.6.4 # No Update - 8.10.x: 8.7.1 # No Update -mas_hputilities_version: - 8.9.x: 8.5.1 # No Update - 8.10.x: 8.6.1 # No Update -mas_iot_version: - 8.9.x: 8.6.7 # No Update - 8.10.x: 8.7.2 # No Update -mas_manage_version: - 8.9.x: 8.5.6 # No Update - 8.10.x: 8.6.2 # No Update -mas_monitor_version: - 8.9.x: 8.9.5 # No Update - 8.10.x: 8.10.3 # No Update -mas_optimizer_version: - 8.9.x: 8.3.3 # No Update - 8.10.x: 8.4.1 # No Update -mas_predict_version: - 8.9.x: 8.7.2 # No Update - 8.10.x: 8.8.1 # No Update -mas_visualinspection_version: - 8.9.x: 8.7.1 # No Update - 8.10.x: 8.8.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.3.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.1 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.0 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.4 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230721-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230721-amd64.yml deleted file mode 100644 index c84946e854..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230721-amd64.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230721 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:200df0fe4723d2ec40433be5793ee3afe341f966f66b7acaa1fb43b412c90848 - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.2 # Operator version 3.23.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.1 # Operator version 110508.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.0 # Operator version 4.6.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.11 # Operator version 2.0.11 -sls_version: 3.7.0 # Operator version 3.7.0 -tsm_version: 1.5.0 # Operator version 1.5.0 -dd_version: 1.1.4 # Operator version 1.1.4 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.8.0+20230417.110116 # Operator version 3.7.0 -wsl_version: 6.4.0 # Operator version 6.4.0 -wml_version: 6.4.0 # Operator version 3.4.0 -spark_version: 6.4.0 # Operator version 3.4.0 -cognos_version: 23.4.0 # Operator version 23.4.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.3.0 # Operator version 4.6.3 -model_train_version: 1.2.5 # Operator version 1.1.7 -elasticsearch_version: 1.1.1336 # Operator version 1.1.1336 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.9.x: 8.9.7 # No Update - 8.10.x: 8.10.2 # No Update -mas_assist_version: - 8.9.x: 8.6.4 # No Update - 8.10.x: 8.7.1 # No Update -mas_hputilities_version: - 8.9.x: 8.5.1 # No Update - 8.10.x: 8.6.1 # No Update -mas_iot_version: - 8.9.x: 8.6.7 # No Update - 8.10.x: 8.7.2 # No Update -mas_manage_version: - 8.9.x: 8.5.6 # No Update - 8.10.x: 8.6.2 # No Update -mas_monitor_version: - 8.9.x: 8.9.5 # No Update - 8.10.x: 8.10.3 # No Update -mas_optimizer_version: - 8.9.x: 8.3.3 # No Update - 8.10.x: 8.4.1 # No Update -mas_predict_version: - 8.9.x: 8.7.2 # No Update - 8.10.x: 8.8.1 # No Update -mas_visualinspection_version: - 8.9.x: 8.7.1 # No Update - 8.10.x: 8.8.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.3.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.1 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.0 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.4 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230725-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230725-amd64.yml deleted file mode 100644 index 2df24ff94a..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230725-amd64.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230725 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:f0776894d5b584bfdd10c3de2f5e586ddafdaca9b247d13ef05dc23fe98cfe2a - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.2 # Operator version 3.23.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.1 # Operator version 110508.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.0 # Operator version 4.6.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.11 # Operator version 2.0.11 -sls_version: 3.7.0 # Operator version 3.7.0 -tsm_version: 1.5.0 # Operator version 1.5.0 -dd_version: 1.1.4 # Operator version 1.1.4 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.8.0+20230417.110116 # Operator version 3.7.0 -wsl_version: 6.4.0 # Operator version 6.4.0 -wml_version: 6.4.0 # Operator version 3.4.0 -spark_version: 6.4.0 # Operator version 3.4.0 -cognos_version: 23.4.0 # Operator version 23.4.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.3.0 # Operator version 4.6.3 -model_train_version: 1.2.5 # Operator version 1.1.7 -elasticsearch_version: 1.1.1336 # Operator version 1.1.1336 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.9.x: 8.9.8 # Updated - 8.10.x: 8.10.3 # Updated -mas_assist_version: - 8.9.x: 8.6.4 # No Update - 8.10.x: 8.7.1 # No Update -mas_hputilities_version: - 8.9.x: 8.5.1 # No Update - 8.10.x: 8.6.1 # No Update -mas_iot_version: - 8.9.x: 8.6.8 # Updated - 8.10.x: 8.7.3 # Updated -mas_manage_version: - 8.9.x: 8.5.7 # Updated - 8.10.x: 8.6.3 # Updated -mas_monitor_version: - 8.9.x: 8.9.6 # Updated - 8.10.x: 8.10.4 # Updated -mas_optimizer_version: - 8.9.x: 8.3.3 # No Update - 8.10.x: 8.4.1 # No Update -mas_predict_version: - 8.9.x: 8.7.2 # No Update - 8.10.x: 8.8.2 # Updated -mas_visualinspection_version: - 8.9.x: 8.7.1 # No Update - 8.10.x: 8.8.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.3.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.1 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.0 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.4 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230829-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230829-amd64.yml deleted file mode 100644 index b194a91712..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230829-amd64.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230829 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:ecc076b3219db96ed78faac32c815282515d830bc5c6025268be96ca15618f7e - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.4 # Operator version 3.23.4 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.4 # Operator version 110508.0.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.1 # Operator version 4.6.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.11 # Operator version 2.0.11 -sls_version: 3.7.0 # Operator version 3.7.0 -tsm_version: 1.5.0 # Operator version 1.5.0 -dd_version: 1.1.4 # Operator version 1.1.4 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.8.0+20230417.110116 # Operator version 3.7.0 -wsl_version: 6.4.0 # Operator version 6.4.0 -wml_version: 6.4.0 # Operator version 3.4.0 -spark_version: 6.4.0 # Operator version 3.4.0 -cognos_version: 23.4.0 # Operator version 23.4.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.3.0 # Operator version 4.6.3 -model_train_version: 1.2.7 # Operator version 1.1.9 -elasticsearch_version: 1.1.1541 # Operator version 1.1.1541 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.9.x: 8.9.9 # Updated - 8.10.x: 8.10.4 # Updated -mas_assist_version: - 8.9.x: 8.6.5 # Updated - 8.10.x: 8.7.2 # Updated -mas_hputilities_version: - 8.9.x: 8.5.3 # Updated - 8.10.x: 8.6.2 # Updated -mas_iot_version: - 8.9.x: 8.6.9 # Updated - 8.10.x: 8.7.4 # Updated -mas_manage_version: - 8.9.x: 8.5.8 # Updated - 8.10.x: 8.6.4 # Updated -mas_monitor_version: - 8.9.x: 8.9.6 # No Update - 8.10.x: 8.10.4 # No Update -mas_optimizer_version: - 8.9.x: 8.3.3 # No Update - 8.10.x: 8.4.1 # No Update -mas_predict_version: - 8.9.x: 8.7.2 # No Update - 8.10.x: 8.8.2 # No Update -mas_visualinspection_version: - 8.9.x: 8.7.1 # No Update - 8.10.x: 8.8.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.4.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.2 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.1 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.4 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-230926-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-230926-amd64.yml deleted file mode 100644 index a4288c72da..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-230926-amd64.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v230926 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:b3ad0d8d20eee9c7e48ba93b956a4f452e48ba0a648e76c39100c352f2cb6537 - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.4 # Operator version 3.23.4 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.4 # Operator version 110508.0.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.1 # Operator version 4.6.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.11 # Operator version 2.0.11 -sls_version: 3.8.0 # Operator version 3.8.0 -tsm_version: 1.5.1 # Operator version 1.5.1 -dd_version: 1.1.5 # Operator version 1.1.5 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.7 # Operator version 1.1.9 -elasticsearch_version: 1.1.1541 # Operator version 1.1.1541 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.5 # Updated - 8.11.x: 8.11.0 # Updated -mas_assist_version: - 8.10.x: 8.7.2 # No Update - 8.11.x: 8.8.0 # Updated -mas_hputilities_version: - 8.10.x: 8.6.2 # No Update - 8.11.x: "" # Not Supported -mas_iot_version: - 8.10.x: 8.7.4 # No Update - 8.11.x: 8.8.0 # Updated -mas_manage_version: - 8.10.x: 8.6.5 # Updated - 8.11.x: 8.7.0 # Updated -mas_monitor_version: - 8.10.x: 8.10.5 # Updated - 8.11.x: 8.11.0 # Updated -mas_optimizer_version: - 8.10.x: 8.4.1 # No Update - 8.11.x: 8.5.0 # Updated -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.0 # Updated -mas_visualinspection_version: - 8.10.x: 8.8.1 # No Update - 8.11.x: 8.9.0 # Updated - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.4.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version: 4.4.21 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.2 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.1 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-231004-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-231004-amd64.yml deleted file mode 100644 index 9c3f53e2e6..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-231004-amd64.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v231004 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:32b28d56327215dcab58664f10987b3e961c0ee9630744b9f66b710e9d879dca - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.4 # Operator version 3.23.4 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.4 # Operator version 110508.0.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.1 # Operator version 4.6.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.11 # Operator version 2.0.11 -sls_version: 3.8.1 # Operator version 3.8.1 -tsm_version: 1.5.1 # Operator version 1.5.1 -dd_version: 1.1.5 # Operator version 1.1.5 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.7 # Operator version 1.1.9 -elasticsearch_version: 1.1.1541 # Operator version 1.1.1541 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.5 # No Update - 8.11.x: 8.11.1 # Updated - 8.9.x: 8.9.10 # Updated -mas_assist_version: - 8.10.x: 8.7.2 # No Update - 8.11.x: 8.8.0 # No Update - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.2 # No Update - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.4 # No Update - 8.11.x: 8.8.0 # No Update - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.5 # No Update - 8.11.x: 8.7.0 # No Update - 8.9.x: 8.5.9 # Updated -mas_monitor_version: - 8.10.x: 8.10.5 # No Update - 8.11.x: 8.11.0 # No Update - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.1 # No Update - 8.11.x: 8.5.0 # No Update - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.1 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.4.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 4.4.21 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.2 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.1 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-231031-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-231031-amd64.yml deleted file mode 100644 index 14da09bf02..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-231031-amd64.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v231031 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:31f0f52a55cc97e7d6c80b844c1d13791efa303eeca87b41954dd2ab67d75378 - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.4 # Operator version 3.23.4 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.4 # Operator version 110508.0.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.1 # Operator version 4.6.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.8.1 # Operator version 3.8.1 -tsm_version: 1.5.1 # Operator version 1.5.1 -dd_version: 1.1.5 # Operator version 1.1.5 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.7 # Operator version 1.1.9 -elasticsearch_version: 1.1.1541 # Operator version 1.1.1541 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.6 # Updated - 8.11.x: 8.11.2 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.2 # No Update - 8.11.x: 8.8.1 # Updated - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.2 # No Update - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.5 # Updated - 8.11.x: 8.8.1 # Updated - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.6 # Updated - 8.11.x: 8.7.1 # Updated - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.5 # No Update - 8.11.x: 8.11.1 # Updated - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.1 # No Update - 8.11.x: 8.5.0 # No Update - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.1 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.4.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 4.4.21 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror extra mongo versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.21 -mongo_extras_version_6: 6.0.10 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.2 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.1 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-231128-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-231128-amd64.yml deleted file mode 100644 index f032710e11..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-231128-amd64.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v231128 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:e9f2439166ee18b540b8fc4484e3df5235bfaf7293dadd181b5755c3c79c602a - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.4 # Operator version 3.23.4 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.1.4 # Operator version 110508.0.2 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.6.1 # Operator version 4.6.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.8.1 # Operator version 3.8.1 -tsm_version: 1.5.1 # Operator version 1.5.1 -dd_version: 1.1.5 # Operator version 1.1.5 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.7 # Operator version 1.1.9 -elasticsearch_version: 1.1.1541 # Operator version 1.1.1541 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.7 # Updated - 8.11.x: 8.11.3 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.2 # No Update - 8.11.x: 8.8.1 # No Update - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.2 # No Update - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.6 # Updated - 8.11.x: 8.8.2 # Updated - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.7 # Updated - 8.11.x: 8.7.2 # Updated - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.6 # Updated - 8.11.x: 8.11.2 # Updated - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.1 # No Update - 8.11.x: 8.5.0 # No Update - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.1 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.4.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 5.0.21 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.21 -mongo_extras_version_6: 6.0.10 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.2 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.1 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.2 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-231228-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-231228-amd64.yml deleted file mode 100644 index 61bc1d2f6e..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-231228-amd64.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v231228 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:431656fe80e0d565d9b130bb53e6ef12ec0370e3422975f6f4ddbfe95f728cda - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.10 # Operator version 3.23.10 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.4.2 # Operator version 110508.0.3 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.9.0 # Operator version 4.9.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.8.1 # Operator version 3.8.1 -tsm_version: 1.5.1 # Operator version 1.5.1 -dd_version: 1.1.5 # Operator version 1.1.5 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.10 # Operator version 1.1.12 -elasticsearch_version: 1.1.1807 # Operator version 1.1.1807 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.8 # Updated - 8.11.x: 8.11.5 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.2 # No Update - 8.11.x: 8.8.1 # No Update - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.2 # No Update - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.7 # Updated - 8.11.x: 8.8.3 # Updated - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.8 # Updated - 8.11.x: 8.7.3 # Updated - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.6 # No Update - 8.11.x: 8.11.2 # No Update - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.2 # Updated - 8.11.x: 8.5.1 # Updated - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.1 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 5.0.23 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.3 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.2 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.2 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-240130-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-240130-amd64.yml deleted file mode 100644 index e6ac5b41b4..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-240130-amd64.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v240130 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:7944a630ce47776338238717cb04ff98e1faf90087fd339708a0785c683326ca - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.11 # Operator version 3.23.11 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.6.0 # Operator version 110509.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.9.0 # Operator version 4.9.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.8.1 # Operator version 3.8.1 -tsm_version: 1.5.1 # Operator version 1.5.1 -dd_version: 1.1.6 # Operator version 1.1.6 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.11 # Operator version 1.1.13 -elasticsearch_version: 1.1.1845 # Operator version 1.1.1845 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.9 # Updated - 8.11.x: 8.11.6 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.2 # No Update - 8.11.x: 8.8.1 # No Update - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.3 # Updated - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.8 # Updated - 8.11.x: 8.8.4 # Updated - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.9 # Updated - 8.11.x: 8.7.4 # Updated - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.6 # No Update - 8.11.x: 8.11.3 # Updated - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.2 # No Update - 8.11.x: 8.5.1 # No Update - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.1 # Updated - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.1 # No Update - 8.11.x: 8.9.0 # No Update - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 5.0.23 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.4 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.3 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.2 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-240227-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-240227-amd64.yml deleted file mode 100644 index e5a10afec3..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-240227-amd64.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v240227 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:8a2f2226b4aa47d42ccb564083d2a6e365fc4b116fb9a82ea83e269383a1efa1 - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.11 # Operator version 3.23.11 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.6.0 # Operator version 110509.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.9.0 # Operator version 4.9.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.8.1 # Operator version 3.8.1 -tsm_version: 1.5.1 # Operator version 1.5.1 -dd_version: 1.1.6 # Operator version 1.1.6 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.11 # Operator version 1.1.13 -elasticsearch_version: 1.1.1845 # Operator version 1.1.1845 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.10 # Updated - 8.11.x: 8.11.7 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.3 # Updated - 8.11.x: 8.8.2 # Updated - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.4 # Updated - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.9 # Updated - 8.11.x: 8.8.5 # Updated - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.10 # Updated - 8.11.x: 8.7.5 # Updated - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.7 # Updated - 8.11.x: 8.11.4 # Updated - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.3 # Updated - 8.11.x: 8.5.2 # Updated - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.2 # Updated - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.2 # Updated - 8.11.x: 8.9.1 # Updated - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 5.0.23 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.4 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.3 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.3 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-240326-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-240326-amd64.yml deleted file mode 100644 index 08f737199c..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-240326-amd64.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog v240326 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:9c891bb6fa8496e1ef514d29b253bf5b1e817fe95963f6e536170688aa95f1d2 - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.12 # Operator version 3.23.12 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.6.2 # Operator version 110509.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.9.0 # Operator version 4.9.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.8.1 # Operator version 3.8.1 -tsm_version: 1.5.3 # Operator version 1.5.3 -dd_version: 1.1.7 # Operator version 1.1.7 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.13 # Operator version 1.1.15 -elasticsearch_version: 1.1.1960 # Operator version 1.1.1960 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.11 # Updated - 8.11.x: 8.11.8 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.3 # No Update - 8.11.x: 8.8.2 # No Update - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.4 # No Update - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.10 # Updated - 8.11.x: 8.8.6 # Updated - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.11 # Updated - 8.11.x: 8.7.6 # Updated - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.8 # Updated - 8.11.x: 8.11.5 # Updated - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.4 # Updated - 8.11.x: 8.5.3 # Updated - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.2 # No Update - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.1 # No Update - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 5.0.23 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.5 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.4 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.3 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-240405-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-240405-amd64.yml deleted file mode 100644 index 3d1beb9103..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-240405-amd64.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog 240405 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:8f03470e84cad81cd5e7aaaf4bce3a08b6575d48b2d27de07c25872f73ecc59c - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.12 # Operator version 3.23.12 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.6.2 # Operator version 110509.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.9.0 # Operator version 4.9.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.8.1 # Operator version 3.8.1 -tsm_version: 1.5.3 # Operator version 1.5.3 -dd_version: 1.1.7 # Operator version 1.1.7 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.13 # Operator version 1.1.15 -elasticsearch_version: 1.1.1960 # Operator version 1.1.1960 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.11 # No Update - 8.11.x: 8.11.9 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.3 # No Update - 8.11.x: 8.8.2 # No Update - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.4 # No Update - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.10 # No Update - 8.11.x: 8.8.6 # No Update - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.11 # No Update - 8.11.x: 8.7.6 # No Update - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.8 # No Update - 8.11.x: 8.11.5 # No Update - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.4 # No Update - 8.11.x: 8.5.3 # No Update - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.2 # No Update - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.1 # No Update - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 5.0.23 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.5 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.4 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.3 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-240430-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-240430-amd64.yml deleted file mode 100644 index c7a3844f0a..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-240430-amd64.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog 240430 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:ce8e623505fbfca7fa207b341a442448b99ff44eeb527df5f3a4c687a47e47e4 - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.12 # Operator version 3.23.12 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.6.2 # Operator version 110509.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 4.9.0 # Operator version 4.9.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.9.0 # Operator version 3.9.0 -tsm_version: 1.5.3 # Operator version 1.5.3 -dd_version: 1.1.7 # Operator version 1.1.7 -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.13 # Operator version 1.1.15 -elasticsearch_version: 1.1.1960 # Operator version 1.1.1960 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.13 # Updated - 8.11.x: 8.11.10 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.4 # Updated - 8.11.x: 8.8.3 # Updated - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.5 # Updated - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.11 # Updated - 8.11.x: 8.8.7 # Updated - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.13 # Updated - 8.11.x: 8.7.7 # Updated - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.9 # Updated - 8.11.x: 8.11.6 # Updated - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.5 # Updated - 8.11.x: 8.5.4 # Updated - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.2 # No Update - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.3 # Updated - 8.11.x: 8.9.2 # Updated - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 5.0.23 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.5 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.4 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.0.3 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v8-240528-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v8-240528-amd64.yml deleted file mode 100644 index 61271033a2..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v8-240528-amd64.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog 240528 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:5ca51934b3aec978e261e6dc9704fd3d46a308010bd6c86252788d69aac3fa9f - -# Dependencies -# ----------------------------------------------------------------------------- -common_svcs_version: 1.19.13 # Operator version 3.23.13 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -db2u_version: 5.6.2 # Operator version 110509.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 5.0.0 # Operator version 5.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.9.1 # Operator version 3.9.1 (https://github.ibm.com/maximoappsuite/ibm-sls/releases) -tsm_version: 1.5.3 # Operator version 1.5.3 (https://github.ibm.com/maximoappsuite/ibm-truststore-mgr/releases) -dd_version: 1.1.8 # Operator version 1.1.8 (https://github.ibm.com/maximoappsuite/ibm-data-dictionary/releases) -appconnect_version: 6.2.0 # Operator version 6.2.0 -cp4d_platform_version: 2.9.0+20230524.165553 # Operator version 3.8.0 -wsl_version: 6.5.0 # Operator version 6.5.0 -wml_version: 6.5.0 # Operator version 3.5.0 -spark_version: 6.5.0 # Operator version 3.5.0 -cognos_version: 23.5.0 # Operator version 23.5.0 - -# Watson discovery and its dependencies -# Match corresponding case version for default versions in the catalog source -# ----------------------------------------------------------------------------- -wd_version: 5.5.0 # Operator version 4.6.5 -model_train_version: 1.2.13 # Operator version 1.1.15 -elasticsearch_version: 1.1.2071 # Operator version 1.1.2071 -couchdb_version: 1.0.13 # Operator version 2.2.1 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 8.10.x: 8.10.14 # Updated - 8.11.x: 8.11.11 # Updated - 8.9.x: 8.9.10 # No Update -mas_assist_version: - 8.10.x: 8.7.4 # No Update - 8.11.x: 8.8.3 # No Update - 8.9.x: 8.6.5 # No Update -mas_hputilities_version: - 8.10.x: 8.6.5 # No Update - 8.11.x: "" # Not Supported - 8.9.x: 8.5.3 # No Update -mas_iot_version: - 8.10.x: 8.7.13 # Updated - 8.11.x: 8.8.9 # Updated - 8.9.x: 8.6.9 # No Update -mas_manage_version: - 8.10.x: 8.6.14 # Updated - 8.11.x: 8.7.8 # Updated - 8.9.x: 8.5.9 # No Update -mas_monitor_version: - 8.10.x: 8.10.10 # Updated - 8.11.x: 8.11.7 # Updated - 8.9.x: 8.9.6 # No Update -mas_optimizer_version: - 8.10.x: 8.4.6 # Updated - 8.11.x: 8.5.5 # Updated - 8.9.x: 8.3.3 # No Update -mas_predict_version: - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.2 # No Update - 8.9.x: 8.7.2 # No Update -mas_visualinspection_version: - 8.10.x: 8.8.4 # Updated - 8.11.x: 8.9.3 # Updated - 8.9.x: 8.7.1 # No Update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 5.0.23 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.5 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -wd_extras_version: 1.0.4 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.1.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.6.6 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v9-240625-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v9-240625-amd64.yml deleted file mode 100644 index 56e20657a2..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v9-240625-amd64.yml +++ /dev/null @@ -1,99 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog 240625 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:e74f646327e728aa523199e9dfb2e95efb67385755c3ac9f0763ab6563e63843 - -# Dependencies -# ----------------------------------------------------------------------------- -ibm_licensing_version: 4.2.3 # Operator version 4.2.3 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-licensing) -common_svcs_version: 4.3.0 # Operator version 4.3.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -cp4d_platform_version: 4.0.0+20231213.115030 # Operator version 5.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-datacore/4.0.0%2B20231213.115030) - -db2u_version: 5.6.2 # Operator version 110509.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 5.0.0 # Operator version 5.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 -sls_version: 3.9.1 # Operator version 3.9.1 (https://github.ibm.com/maximoappsuite/ibm-sls/releases) -tsm_version: 1.5.4 # Operator version 1.5.4 (https://github.ibm.com/maximoappsuite/ibm-truststore-mgr/releases) -dd_version: 1.1.9 # Operator version 1.1.9 (https://github.ibm.com/maximoappsuite/ibm-data-dictionary/releases) -appconnect_version: 6.2.0 # Operator version 6.2.0 -wsl_version: 8.0.0 # Operator version 8.0.0 -wml_version: 8.0.0 # Operator version 5.0.0 -spark_version: 8.0.0 # Operator version 5.0.0 -cognos_version: 25.0.0 # Operator version 25.0.0 -couchdb_version: 1.0.13 # Operator version 2.2.1 (This is required for Assist 9.0, https://github.com/IBM/cloud-pak/blob/master/repo/case/ibm-couchdb/index.yaml) -elasticsearch_version: 1.1.2071 # Operator version 1.1.2071 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 9.0.x: 9.0.0 # Updated - 8.10.x: 8.10.15 # Updated - 8.11.x: 8.11.12 # Updated -mas_assist_version: - 9.0.x: 9.0.0 # Updated - 8.10.x: 8.7.5 # Updated - 8.11.x: 8.8.4 # Updated -mas_hputilities_version: - 9.0.x: "" # Not Supported - 8.10.x: 8.6.5 # No Update - 8.11.x: "" # Not Supported -mas_iot_version: - 9.0.x: 9.0.0 # Updated - 8.10.x: 8.7.14 # Updated - 8.11.x: 8.8.10 # Updated -mas_manage_version: - 9.0.x: 9.0.0 # Updated - 8.10.x: 8.6.15 # Updated - 8.11.x: 8.7.9 # Updated -mas_monitor_version: - 9.0.x: 9.0.0 # Updated - 8.10.x: 8.10.11 # Updated - 8.11.x: 8.11.8 # Updated -mas_optimizer_version: - 9.0.x: 9.0.0 # Updated - 8.10.x: 8.4.7 # Updated - 8.11.x: 8.5.6 # Updated -mas_predict_version: - 9.0.x: 9.0.0 # Updated - 8.10.x: 8.8.2 # No Update - 8.11.x: 8.9.3 # Updated -mas_visualinspection_version: - 9.0.x: 9.0.0 # Updated - 8.10.x: 8.8.4 # No update - 8.11.x: 8.9.3 # No update - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 6.0.12 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.5 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -#wd_extras_version: 1.0.4 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.1.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.8.0 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v9-240730-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v9-240730-amd64.yml deleted file mode 100644 index 7b494a00c6..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v9-240730-amd64.yml +++ /dev/null @@ -1,100 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog 240625 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:c38ceab72008bd8ae6ecb4bf8e29453e6a6e6d8ecbdb1e40b465fef401c51d56 - -# Dependencies -# ----------------------------------------------------------------------------- -ibm_licensing_version: 4.2.4 # Operator version 4.2.4 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-licensing) -common_svcs_version: 4.3.0 # Operator version 4.3.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -cp4d_platform_version: 4.0.0+20231213.115030 # Operator version 5.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-datacore/4.0.0%2B20231213.115030) - -db2u_version: 5.6.2 # Operator version 110509.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 5.0.1 # Operator version 5.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 # sticking to 2.0.12 version # Please do Not Change -sls_version: 3.9.1 # Operator version 3.9.1 (https://github.ibm.com/maximoappsuite/ibm-sls/releases) -tsm_version: 1.5.4 # Operator version 1.5.4 (https://github.ibm.com/maximoappsuite/ibm-truststore-mgr/releases) -dd_version: 1.1.10 # updated # Operator version 1.1.10 (https://github.ibm.com/maximoappsuite/ibm-data-dictionary/releases) -appconnect_version: 6.2.0 # Operator version 6.2.0 # sticking to 6.2.0 version # Please do Not Change -wsl_version: 8.0.0 # Operator version 8.0.0 -wml_version: 8.0.0 # Operator version 5.0.0 -spark_version: 8.0.0 # Operator version 5.0.0 -cognos_version: 25.0.0 # Operator version 25.0.0 -couchdb_version: 1.0.13 # Operator version 2.2.1 (This is required for Assist 9.0, https://github.com/IBM/cloud-pak/blob/master/repo/case/ibm-couchdb/index.yaml) -elasticsearch_version: 1.1.2153 # Operator version 1.1.2153 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 9.0.x: 9.0.1 # Updated - 8.10.x: 8.10.16 # Updated - 8.11.x: 8.11.13 # Updated -mas_assist_version: - 9.0.x: 9.0.1 # Updated - 8.10.x: 8.7.6 # Updated - 8.11.x: 8.8.5 # Updated -mas_hputilities_version: - 9.0.x: "" # Not Supported - 8.10.x: 8.6.6 # Updated - 8.11.x: "" # Not Supported -mas_iot_version: - 9.0.x: 9.0.1 # Updated - 8.10.x: 8.7.15 # Updated - 8.11.x: 8.8.11 # Updated -mas_manage_version: - 9.0.x: 9.0.1 # Updated - 8.10.x: 8.6.16 # Updated - 8.11.x: 8.7.10 # Updated -mas_monitor_version: - 9.0.x: 9.0.1 # Updated - 8.10.x: 8.10.12 # Updated - 8.11.x: 8.11.9 # Updated -mas_optimizer_version: - 9.0.x: 9.0.1 # Updated - 8.10.x: 8.4.8 # Updated - 8.11.x: 8.5.7 # Updated -mas_predict_version: - 9.0.x: 9.0.0 # No Update - 8.10.x: 8.8.3 # Updated - 8.11.x: 8.9.3 # No Update -mas_visualinspection_version: - 9.0.x: 9.0.1 # Updated - 8.10.x: 8.8.4 # No update - 8.11.x: 8.9.4 # Updated - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 6.0.12 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 -mongo_extras_version_7: 7.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.5 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -#wd_extras_version: 1.0.4 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.1.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.8.0 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v9-240827-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v9-240827-amd64.yml deleted file mode 100644 index 0c4847e130..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v9-240827-amd64.yml +++ /dev/null @@ -1,99 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog 240625 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:bcace8fa41d14ce1b818467243bbc8469f7538ac4bc46198139454626f99f367 - -# Dependencies -# ----------------------------------------------------------------------------- -ibm_licensing_version: 4.2.4 # Operator version 4.2.4 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-licensing) -common_svcs_version: 4.3.0 # Operator version 4.3.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -cp4d_platform_version: 4.0.0+20231213.115030 # Operator version 5.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-datacore/4.0.0%2B20231213.115030) - -db2u_version: 5.6.2 # Operator version 110509.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 5.0.1 # Operator version 5.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 # sticking to 2.0.12 version # Please do Not Change -sls_version: 3.10.0 # updated # Operator version 3.10.0 (https://github.ibm.com/maximoappsuite/ibm-sls/releases) -tsm_version: 1.5.4 # Operator version 1.5.4 (https://github.ibm.com/maximoappsuite/ibm-truststore-mgr/releases) -dd_version: 1.1.11 # updated # Operator version 1.1.11 (https://github.ibm.com/maximoappsuite/ibm-data-dictionary/releases) -appconnect_version: 6.2.0 # Operator version 6.2.0 # sticking to 6.2.0 version # Please do Not Change -wsl_version: 8.0.0 # Operator version 8.0.0 -wml_version: 8.0.0 # Operator version 5.0.0 -spark_version: 8.0.0 # Operator version 5.0.0 -cognos_version: 25.0.0 # Operator version 25.0.0 -couchdb_version: 1.0.13 # Operator version 2.2.1 (This is required for Assist 9.0, https://github.com/IBM/cloud-pak/blob/master/repo/case/ibm-couchdb/index.yaml) -elasticsearch_version: 1.1.2153 # Operator version 1.1.2153 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 9.0.x: 9.0.2 # Updated - 8.10.x: 8.10.17 # Updated - 8.11.x: 8.11.14 # Updated -mas_assist_version: - 9.0.x: 9.0.2 # Updated - 8.10.x: 8.7.7 # Updated - 8.11.x: 8.8.6 # Updated -mas_hputilities_version: - 9.0.x: "" # Not Supported - 8.10.x: 8.6.7 # Updated - 8.11.x: "" # Not Supported -mas_iot_version: - 9.0.x: 9.0.2 # Updated - 8.10.x: 8.7.16 # Updated - 8.11.x: 8.8.12 # Updated -mas_manage_version: - 9.0.x: 9.0.2 # Updated - 8.10.x: 8.6.17 # Updated - 8.11.x: 8.7.11 # Updated -mas_monitor_version: - 9.0.x: 9.0.2 # Updated - 8.10.x: 8.10.13 # Updated - 8.11.x: 8.11.10 # Updated -mas_optimizer_version: - 9.0.x: 9.0.2 # Updated - 8.10.x: 8.4.9 # Updated - 8.11.x: 8.5.8 # Updated -mas_predict_version: - 9.0.x: 9.0.1 # Updated - 8.10.x: 8.8.3 # No Update - 8.11.x: 8.9.3 # No Update -mas_visualinspection_version: - 9.0.x: 9.0.2 # Updated - 8.10.x: 8.8.4 # No update - 8.11.x: 8.9.5 # Updated - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 6.0.12 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.5 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -#wd_extras_version: 1.0.4 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.1.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.8.0 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/casebundles/v9-241003-amd64.yml b/ibm/mas_devops/common_vars/casebundles/v9-241003-amd64.yml deleted file mode 100644 index f713845c64..0000000000 --- a/ibm/mas_devops/common_vars/casebundles/v9-241003-amd64.yml +++ /dev/null @@ -1,100 +0,0 @@ ---- -# Case bundle configuration for IBM Maximo Operator Catalog 240625 -# ----------------------------------------------------------------------------- -# In the future this won't be necessary as we'll be able to mirror from the -# catalog itself, but not everything in the catalog supports this yet (including MAS) -# so we need to use the CASE bundle mirror process still. - -catalog_digest: sha256:ba2237481b2ce7407698775a71f44daaecd2db6f74855f20829ac1bceeddb3d9 - -# Dependencies -# ----------------------------------------------------------------------------- -ibm_licensing_version: 4.2.7 # Operator version 4.2.7 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-licensing) -common_svcs_version: 4.3.0 # check once why we have 4.3.0 in all previous versions since so long it's 4.3.0 # Operator version 4.3.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services) -cp4d_platform_version: 4.0.0+20231213.115030 # Operator version 5.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-datacore/4.0.0%2B20231213.115030) - -db2u_version: 6.0.1+20240704.142950.9960 # Operator version 110509.0.2 to find the version 6.0.1, search CASE_VERSION in db2u, catalog.yaml into ibm-maximo-operator-catalog (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator) -events_version: 5.0.1 # Operator version 5.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator) -uds_version: 2.0.12 # Operator version 2.0.12 # sticking to 2.0.12 version # Please do Not Change -sls_version: 3.10.0 # No Update # Operator version 3.10.0 (https://github.ibm.com/maximoappsuite/ibm-sls/releases) -tsm_version: 1.5.4 # Operator version 1.5.4 (https://github.ibm.com/maximoappsuite/ibm-truststore-mgr/releases) -dd_version: 1.1.12 # Updated # Operator version 1.1.11 (https://github.ibm.com/maximoappsuite/ibm-data-dictionary/releases) -appconnect_version: 6.2.0 # Operator version 6.2.0 # sticking to 6.2.0 version # Please do Not Change -wsl_version: 8.0.0 # Operator version 8.0.0 -wml_version: 8.0.0 # Operator version 5.0.0 -spark_version: 8.0.0 # Operator version 5.0.0 -cognos_version: 25.0.0 # Operator version 25.0.0 -couchdb_version: 1.0.13 # Operator version 2.2.1 (This is required for Assist 9.0, https://github.com/IBM/cloud-pak/blob/master/repo/case/ibm-couchdb/index.yaml) -elasticsearch_version: 1.1.2238 # Operator version 1.1.2238 - - -# Maximo Application Suite -# ----------------------------------------------------------------------------- -mas_core_version: - 9.0.x: 9.0.3 # Updated - 8.10.x: 8.10.18 # Updated - 8.11.x: 8.11.15 # Updated -mas_assist_version: - 9.0.x: 9.0.2 # No Update - 8.10.x: 8.7.7 # No Update - 8.11.x: 8.8.6 # No Update -mas_hputilities_version: - 9.0.x: "" # Not Supported - 8.10.x: 8.6.7 # No Update - 8.11.x: "" # Not Supported -mas_iot_version: - 9.0.x: 9.0.3 # Updated - 8.10.x: 8.7.17 # Updated - 8.11.x: 8.8.13 # Updated -mas_manage_version: - 9.0.x: 9.0.3 # Updated - 8.10.x: 8.6.18 # Updated - 8.11.x: 8.7.12 # Updated -mas_monitor_version: - 9.0.x: 9.0.3 # Updated - 8.10.x: 8.10.14 # Updated - 8.11.x: 8.11.11 # Updated -mas_optimizer_version: - 9.0.x: 9.0.3 # Updated - 8.10.x: 8.4.10 # Updated - 8.11.x: 8.5.9 # Updated -mas_predict_version: - 9.0.x: 9.0.2 # Updated - 8.10.x: 8.8.3 # No Update - 8.11.x: 8.9.5 # Updated -mas_visualinspection_version: - 9.0.x: 9.0.3 # Updated - 8.10.x: 8.8.4 # No update - 8.11.x: 8.9.6 # Updated - -# Extra Images for UDS -# ------------------------------------------------------------------------------ -uds_extras_version: 1.5.0 - -# Extra Images for Mongo -# ------------------------------------------------------------------------------ -mongo_extras_version_default: 7.0.12 -mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}" - -# Variables used to mirror additional mongo image versions -mongo_extras_version_4: 4.4.21 -mongo_extras_version_5: 5.0.23 -mongo_extras_version_6: 6.0.12 -mongo_extras_version_7: 7.0.12 - -# Extra Images for Db2u -# ------------------------------------------------------------------------------ -db2u_extras_version: 1.0.6 - -# Extra Images for IBM Watson Discovery -# ------------------------------------------------------------------------------ -#wd_extras_version: 1.0.4 - -# Extra Images for Amlen -# ------------------------------------------------------------------------------ -amlen_extras_version: 1.1.1 - -# Default Cloud Pak for Data version -# ------------------------------------------------------------------------------ -cpd_product_version_default: 4.8.0 -cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}" diff --git a/ibm/mas_devops/common_vars/default_storage_classes.yml b/ibm/mas_devops/common_vars/default_storage_classes.yml deleted file mode 100644 index 4d93750422..0000000000 --- a/ibm/mas_devops/common_vars/default_storage_classes.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -default_storage_classes_rwx: - - ibmc-file-gold-gid - - ocs-storagecluster-cephfs - - ocs-external-storagecluster-cephfs - - efs - - azurefiles-premium - - nfs-client - -default_storage_classes_rwx_nogid: - - ibmc-file-gold - - ocs-storagecluster-cephfs - - efs - - azurefiles-premium - - nfs-client - -default_storage_classes_rwo: - - ibmc-block-gold - - ocs-storagecluster-ceph-rbd - - ocs-external-storagecluster-ceph-rbd - - gp3-csi - - managed-premium - - nfs-client diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml index 8f4fcf757b..2e51a0ccc3 100644 --- a/ibm/mas_devops/playbooks/br_core.yml +++ b/ibm/mas_devops/playbooks/br_core.yml @@ -31,7 +31,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" @@ -40,34 +39,13 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" + - name: "MongoDB: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.mongodb vars: - _rt_playbook_name: "br_core" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "core" + mongodb_action: "{{ masbr_action }}" + mas_app_id: "core" - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore + - name: "MAS Core namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_backup_restore diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index 5d7c30a88d..94d7b9b9b1 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -47,33 +47,10 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" - vars: - _rt_playbook_name: "br_db2" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "DB2_INSTANCE_NAME" - value: "{{ db2_instance_id }}" - - name: "DB2_NAMESPACE" - value: "{{ db2_namespace }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - # Run backup/restore tasks locally # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "Db2 {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" + - name: "Db2 {{ masbr_action }}" + include_role: + name: ibm.mas_devops.db2 + vars: + db2_action: "{{ masbr_action }}" diff --git a/ibm/mas_devops/playbooks/br_health.yml b/ibm/mas_devops/playbooks/br_health.yml index 5c4ae8610b..5c440bc50d 100644 --- a/ibm/mas_devops/playbooks/br_health.yml +++ b/ibm/mas_devops/playbooks/br_health.yml @@ -53,57 +53,31 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" + - name: "MongoDB: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.mongodb vars: - _rt_playbook_name: "br_health" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_WORKSPACE_ID" - value: "{{ mas_workspace_id }}" - - name: "DB2_INSTANCE_NAME" - value: "{{ db2_instance_id }}" - - name: "DB2_NAMESPACE" - value: "{{ db2_namespace }}" - - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "health" + mongodb_action: "{{ masbr_action }}" + mas_app_id: "health" - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" + - name: "Db2: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.db2 + vars: + db2_action: "{{ masbr_action }}" - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore + - name: "MAS Core namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_backup_restore - - name: "Manage namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "manage" + - name: "Manage namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "manage" - - name: "Health namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "health" + - name: "Health namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "health" diff --git a/ibm/mas_devops/playbooks/br_iot.yml b/ibm/mas_devops/playbooks/br_iot.yml index 03845f9cac..cb5d2d2a56 100644 --- a/ibm/mas_devops/playbooks/br_iot.yml +++ b/ibm/mas_devops/playbooks/br_iot.yml @@ -53,51 +53,25 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" + - name: "MongoDB: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.mongodb vars: - _rt_playbook_name: "br_iot" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_WORKSPACE_ID" - value: "{{ mas_workspace_id }}" - - name: "DB2_INSTANCE_NAME" - value: "{{ db2_instance_id }}" - - name: "DB2_NAMESPACE" - value: "{{ db2_namespace }}" - - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "iot" + mongodb_action: "{{ masbr_action }}" + mas_app_id: "iot" - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" + - name: "Db2: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.db2 + vars: + db2_action: "{{ masbr_action }}" - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore + - name: "MAS Core namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_backup_restore - - name: "IoT namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "iot" + - name: "IoT namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "iot" diff --git a/ibm/mas_devops/playbooks/br_manage.yml b/ibm/mas_devops/playbooks/br_manage.yml index c4fa4c0123..a14e6f80d3 100644 --- a/ibm/mas_devops/playbooks/br_manage.yml +++ b/ibm/mas_devops/playbooks/br_manage.yml @@ -34,11 +34,6 @@ that: mas_workspace_id is defined and mas_workspace_id != "" fail_msg: "mas_workspace_id is required" - - name: "Fail if db2_instance_id is not provided" - assert: - that: db2_instance_id is defined and db2_instance_id != "" - fail_msg: "db2_instance_id is required" - - name: "Fail if masbr_action is not set to backup|restore" assert: that: masbr_action in ["backup", "restore"] @@ -53,51 +48,26 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" + - name: "MongoDB: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.mongodb vars: - _rt_playbook_name: "br_manage" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_WORKSPACE_ID" - value: "{{ mas_workspace_id }}" - - name: "DB2_INSTANCE_NAME" - value: "{{ db2_instance_id }}" - - name: "DB2_NAMESPACE" - value: "{{ db2_namespace }}" + mongodb_action: "{{ masbr_action }}" + mas_app_id: "manage" - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "manage" - - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" + - name: "Db2: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.db2 + vars: + db2_action: "{{ masbr_action }}" + when: db2_instance_id is defined and db2_instance_id != "" - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore + - name: "MAS Core namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_backup_restore - - name: "Manage namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "manage" + - name: "Manage namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "manage" diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml index b2389d7ed9..c15a65af83 100644 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -43,35 +43,8 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" + - name: "MongoDB: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.mongodb vars: - _rt_playbook_name: "br_mongodb" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_APP_ID" - value: "{{ mas_app_id }}" - - name: "MONGODB_PROVIDER" - value: "{{ mongodb_provider }}" - - name: "MONGODB_NAMESPACE" - value: "{{ mongodb_namespace }}" - - - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" + mongodb_action: "{{ masbr_action }}" diff --git a/ibm/mas_devops/playbooks/br_monitor.yml b/ibm/mas_devops/playbooks/br_monitor.yml index 0ffdc36ede..ef2c68f88a 100644 --- a/ibm/mas_devops/playbooks/br_monitor.yml +++ b/ibm/mas_devops/playbooks/br_monitor.yml @@ -53,57 +53,31 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" + - name: "MongoDB: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.mongodb vars: - _rt_playbook_name: "br_monitor" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_WORKSPACE_ID" - value: "{{ mas_workspace_id }}" - - name: "DB2_INSTANCE_NAME" - value: "{{ db2_instance_id }}" - - name: "DB2_NAMESPACE" - value: "{{ db2_namespace }}" - - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "monitor" + mongodb_action: "{{ masbr_action }}" + mas_app_id: "monitor" - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" + - name: "Db2: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.db2 + vars: + db2_action: "{{ masbr_action }}" - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore + - name: "MAS Core namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_backup_restore - - name: "IoT namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "iot" + - name: "IoT namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "iot" - - name: "Monitor namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "monitor" + - name: "Monitor namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "monitor" diff --git a/ibm/mas_devops/playbooks/br_optimizer.yml b/ibm/mas_devops/playbooks/br_optimizer.yml index 475161b2ff..041d209d97 100644 --- a/ibm/mas_devops/playbooks/br_optimizer.yml +++ b/ibm/mas_devops/playbooks/br_optimizer.yml @@ -53,57 +53,31 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" + - name: "MongoDB: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.mongodb vars: - _rt_playbook_name: "br_optimizer" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_WORKSPACE_ID" - value: "{{ mas_workspace_id }}" - - name: "DB2_INSTANCE_NAME" - value: "{{ db2_instance_id }}" - - name: "DB2_NAMESPACE" - value: "{{ db2_namespace }}" - - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "optimizer" + mongodb_action: "{{ masbr_action }}" + mas_app_id: "optimizer" - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" + - name: "Db2: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.db2 + vars: + db2_action: "{{ masbr_action }}" - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore + - name: "MAS Core namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_backup_restore - - name: "Manage namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "manage" + - name: "Manage namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "manage" - - name: "Optimizer namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "optimizer" + - name: "Optimizer namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "optimizer" diff --git a/ibm/mas_devops/playbooks/br_visualinspection.yml b/ibm/mas_devops/playbooks/br_visualinspection.yml index 3c62c0465e..feeddc1ca1 100644 --- a/ibm/mas_devops/playbooks/br_visualinspection.yml +++ b/ibm/mas_devops/playbooks/br_visualinspection.yml @@ -46,42 +46,19 @@ _job_type: "{{ masbr_action }}" tasks: - # Create k8s Job to run backup/restore tasks - # ------------------------------------------------------------------------- - - name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" + - name: "MongoDB: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.mongodb vars: - _rt_playbook_name: "br_visualinspection" - _rt_env: - - name: "MASBR_ACTION" - value: "{{ masbr_action }}" - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_WORKSPACE_ID" - value: "{{ mas_workspace_id }}" - + mongodb_action: "{{ masbr_action }}" + mas_app_id: "visualinspection" - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job - block: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "visualinspection" + - name: "MAS Core namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_backup_restore - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore - - - name: "Visual Inspection namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "visualinspection" + - name: "Visual Inspection namespace: {{ masbr_action }}" + include_role: + name: ibm.mas_devops.suite_app_backup_restore + vars: + mas_app_id: "visualinspection" diff --git a/ibm/mas_devops/playbooks/mirror_add_assist.yml b/ibm/mas_devops/playbooks/mirror_add_assist.yml index b03dcdbe0e..d5a16cd9ac 100644 --- a/ibm/mas_devops/playbooks/mirror_add_assist.yml +++ b/ibm/mas_devops/playbooks/mirror_add_assist.yml @@ -8,49 +8,45 @@ mirror_mode: "{{ lookup('env', 'MIRROR_MODE') | default ('direct', True) }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false roles: # 1. IBM Maximo Assist # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-assist - case_version: "{{ lookup('env', 'MAS_ASSIST_VERSION') | default (mas_assist_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_ASSIST_VERSION') | default (mas_catalog_metadata.mas_assist_version[mas_channel], True) }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-assist - manifest_version: "{{ lookup('env', 'MAS_ASSIST_VERSION') | default (mas_assist_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_ASSIST_VERSION') | default (mas_catalog_metadata.mas_assist_version[mas_channel], True) }}" # 2. IBM MAS Assist dependency from 9.0 - ibm-couchdb # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - couchdb_version is defined - mirror_mode != "from-filesystem" vars: case_name: ibm-couchdb - case_version: "{{ couchdb_version }}" + case_version: "{{ mas_catalog_metadata.couchdb_version }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: couchdb_version is defined vars: manifest_name: ibm-couchdb - manifest_version: "{{ couchdb_version }}" + manifest_version: "{{ mas_catalog_metadata.couchdb_version }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_hputilities.yml b/ibm/mas_devops/playbooks/mirror_add_hputilities.yml deleted file mode 100644 index d5973f8966..0000000000 --- a/ibm/mas_devops/playbooks/mirror_add_hputilities.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- hosts: localhost - any_errors_fatal: true - - vars: - catalog_tag: "{{ lookup('env', 'MAS_CATALOG_VERSION') | default ('@@MAS_LATEST_CATALOG@@', True) }}" - mas_channel: "{{ lookup('env', 'MAS_CHANNEL') | default ('8.10.x', True) }}" - mirror_mode: "{{ lookup('env', 'MIRROR_MODE') | default ('direct', True) }}" - - pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats - - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" - - roles: - # 1. IBM Maximo Health and Predict Utilities - # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare - when: mirror_mode != "from-filesystem" - vars: - case_name: ibm-mas-hputilities - case_version: "{{ lookup('env', 'MAS_HPUTILITIES_VERSION') | default (mas_hputilities_version[mas_channel], True) }}" - exclude_images: [] - ibmpak_skip_dependencies: false - - - name: ibm.mas_devops.mirror_images - vars: - manifest_name: ibm-mas-hputilities - manifest_version: "{{ lookup('env', 'MAS_HPUTILITIES_VERSION') | default (mas_hputilities_version[mas_channel], True) }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_iot.yml b/ibm/mas_devops/playbooks/mirror_add_iot.yml index 62b5568038..11459a790e 100644 --- a/ibm/mas_devops/playbooks/mirror_add_iot.yml +++ b/ibm/mas_devops/playbooks/mirror_add_iot.yml @@ -8,47 +8,43 @@ mirror_mode: "{{ lookup('env', 'MIRROR_MODE') | default ('direct', True) }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false roles: # 1. IBM Maximo IoT # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-iot - case_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_iot_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_catalog_metadata.mas_iot_version[mas_channel], True) }}" exclude_images: [] - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-iot - manifest_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_iot_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_catalog_metadata.mas_iot_version[mas_channel], True) }}" # 2. Eclipse Amlen # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - amlen_extras_version is defined - mirror_mode != "from-filesystem" vars: extras_name: amlen - extras_version: "{{ amlen_extras_version }}" + extras_version: "{{ mas_catalog_metadata.amlen_extras_version }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: amlen_extras_version is defined vars: manifest_name: extras_amlen - manifest_version: "{{ amlen_extras_version }}" + manifest_version: "{{ mas_catalog_metadata.amlen_extras_version }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_manage.yml b/ibm/mas_devops/playbooks/mirror_add_manage.yml index 8ed7cde88b..cb307155ef 100644 --- a/ibm/mas_devops/playbooks/mirror_add_manage.yml +++ b/ibm/mas_devops/playbooks/mirror_add_manage.yml @@ -9,31 +9,27 @@ mirror_icd: "{{ lookup('env', 'MIRROR_MAS_ICD') }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false roles: # 1. IBM Maximo Manage # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-manage - case_version: "{{ lookup('env', 'MAS_MANAGE_VERSION') | default (mas_manage_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_MANAGE_VERSION') | default (mas_catalog_metadata.mas_manage_version[mas_channel], True) }}" exclude_images: [] image_group_filter: "{{ (mirror_icd == 'true') | ternary ('ibmmasManage,ibmMasManage,ibmmasMaximoIT', 'ibmmasManage,ibmMasManage') }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-manage - manifest_version: "{{ lookup('env', 'MAS_MANAGE_VERSION') | default (mas_manage_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_MANAGE_VERSION') | default (mas_catalog_metadata.mas_manage_version[mas_channel], True) }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_monitor.yml b/ibm/mas_devops/playbooks/mirror_add_monitor.yml index 7b203a09f9..c2655022d4 100644 --- a/ibm/mas_devops/playbooks/mirror_add_monitor.yml +++ b/ibm/mas_devops/playbooks/mirror_add_monitor.yml @@ -8,45 +8,41 @@ mirror_mode: "{{ lookup('env', 'MIRROR_MODE') | default ('direct', True) }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false roles: # 1. IBM Asset Data Dictionary # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-data-dictionary - case_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (dd_version, True) }}" + case_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (mas_catalog_metadata.dd_version, True) }}" exclude_images: [] - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-data-dictionary - manifest_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (dd_version, True) }}" + manifest_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (mas_catalog_metadata.dd_version, True) }}" # 2. IBM Maximo Monitor # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-monitor - case_version: "{{ lookup('env', 'MAS_MONITOR_VERSION') | default (mas_monitor_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_MONITOR_VERSION') | default (mas_catalog_metadata.mas_monitor_version[mas_channel], True) }}" exclude_images: [] - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-monitor - manifest_version: "{{ lookup('env', 'MAS_MONITOR_VERSION') | default (mas_monitor_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_MONITOR_VERSION') | default (mas_catalog_metadata.mas_monitor_version[mas_channel], True) }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_optimizer.yml b/ibm/mas_devops/playbooks/mirror_add_optimizer.yml index 5fafede1c1..eaa51b0436 100644 --- a/ibm/mas_devops/playbooks/mirror_add_optimizer.yml +++ b/ibm/mas_devops/playbooks/mirror_add_optimizer.yml @@ -8,30 +8,26 @@ mirror_mode: "{{ lookup('env', 'MIRROR_MODE') | default ('direct', True) }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false roles: # 1. IBM Maximo Optimizer # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-optimizer - case_version: "{{ lookup('env', 'MAS_OPTIMIZER_VERSION') | default (mas_optimizer_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_OPTIMIZER_VERSION') | default (mas_catalog_metadata.mas_optimizer_version[mas_channel], True) }}" exclude_images: [] - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-optimizer - manifest_version: "{{ lookup('env', 'MAS_OPTIMIZER_VERSION') | default (mas_optimizer_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_OPTIMIZER_VERSION') | default (mas_catalog_metadata.mas_optimizer_version[mas_channel], True) }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_predict.yml b/ibm/mas_devops/playbooks/mirror_add_predict.yml index 24ef9e43fd..d9d7a53af3 100644 --- a/ibm/mas_devops/playbooks/mirror_add_predict.yml +++ b/ibm/mas_devops/playbooks/mirror_add_predict.yml @@ -8,31 +8,27 @@ mirror_mode: "{{ lookup('env', 'MIRROR_MODE') | default ('direct', True) }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false roles: # 1. IBM Maximo Predict # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-predict - case_version: "{{ lookup('env', 'MAS_PREDICT_VERSION') | default (mas_predict_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_PREDICT_VERSION') | default (mas_catalog_metadata.mas_predict_version[mas_channel], True) }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-predict - manifest_version: "{{ lookup('env', 'MAS_PREDICT_VERSION') | default (mas_predict_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_PREDICT_VERSION') | default (mas_catalog_metadata.mas_predict_version[mas_channel], True) }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_visualinspection.yml b/ibm/mas_devops/playbooks/mirror_add_visualinspection.yml index 05134ac4f9..6fadf429eb 100644 --- a/ibm/mas_devops/playbooks/mirror_add_visualinspection.yml +++ b/ibm/mas_devops/playbooks/mirror_add_visualinspection.yml @@ -14,30 +14,26 @@ mirror_mode: "{{ lookup('env', 'MIRROR_MODE') | default ('direct', True) }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false roles: # 1. IBM Maximo Visual Inspection # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-visualinspection - case_version: "{{ lookup('env', 'MAS_VISUALINSPECTION_VERSION') | default (mas_visualinspection_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_VISUALINSPECTION_VERSION') | default (mas_catalog_metadata.mas_visualinspection_version[mas_channel], True) }}" exclude_images: [] - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-visualinspection - manifest_version: "{{ lookup('env', 'MAS_VISUALINSPECTION_VERSION') | default (mas_visualinspection_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_VISUALINSPECTION_VERSION') | default (mas_catalog_metadata.mas_visualinspection_version[mas_channel], True) }}" diff --git a/ibm/mas_devops/playbooks/mirror_core.yml b/ibm/mas_devops/playbooks/mirror_core.yml index 8953f12d92..d2e3c6722b 100644 --- a/ibm/mas_devops/playbooks/mirror_core.yml +++ b/ibm/mas_devops/playbooks/mirror_core.yml @@ -8,31 +8,27 @@ mirror_mode: "{{ lookup('env', 'MIRROR_MODE') | default ('direct', True) }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" - - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false roles: # 1. IBM Maximo Application Suite Core # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas - case_version: "{{ lookup('env', 'MAS_CORE_VERSION') | default (mas_core_version[mas_channel], True) | replace('_', '.') }}" + case_version: "{{ lookup('env', 'MAS_CORE_VERSION') | default (mas_catalog_metadata.mas_core_version[mas_channel], True) | replace('_', '.') }}" exclude_images: [] ibmpak_skip_dependencies: true - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas - manifest_version: "{{ lookup('env', 'MAS_CORE_VERSION') | default (mas_core_version[mas_channel], True) | replace('_', '.') }}" + manifest_version: "{{ lookup('env', 'MAS_CORE_VERSION') | default (mas_catalog_metadata.mas_core_version[mas_channel], True) | replace('_', '.') }}" diff --git a/ibm/mas_devops/playbooks/mirror_dependencies.yml b/ibm/mas_devops/playbooks/mirror_dependencies.yml index a1c0b98508..3d009b7ef9 100644 --- a/ibm/mas_devops/playbooks/mirror_dependencies.yml +++ b/ibm/mas_devops/playbooks/mirror_dependencies.yml @@ -66,30 +66,33 @@ mirror_odf: "{{ lookup('env', 'MIRROR_ODF') | default ('False', True) | bool }}" pre_tasks: - - name: "Get stats for the catalog file" - ansible.builtin.stat: - path: "{{ playbook_dir }}/../common_vars/casebundles/{{ catalog_tag }}.yml" - register: catalog_file_stats + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: mas_catalog_metadata - - name: "Debug stats if the digest image map file exists" - ansible.builtin.set_fact: - file_catalog_tag: "{{ (catalog_file_stats.stat.exists|bool) | ternary(catalog_tag, '@@MAS_LATEST_CATALOG@@') }}" + - name: "Check that the catalog is a know catalog" + assert: + that: mas_catalog_metadata.failed == false - - name: Load CASE bundle versions - include_vars: - file: "{{ playbook_dir }}/../common_vars/casebundles/{{ file_catalog_tag }}.yml" + # If the CPD_PRODUCT_VERSION environment variable wasn't set, then use the default from the catalog metadata + - name: "Set default cpd_product_version (if necessary)" + when: cpd_product_version == "" + set_fact: + cpd_product_version: "{{ mas_catalog_metadata.cpd_product_version_default }}" roles: # 1. IBM Maximo Operator Catalog # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_catalog - mirror_mode != "from-filesystem" vars: extras_name: catalog + catalog_digest: "{{ mas_catalog_metadata.catalog_digest }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_catalog vars: manifest_name: catalog @@ -98,91 +101,91 @@ # 2. MongoDb Community Edition # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_mongoce - mirror_mode != "from-filesystem" vars: extras_name: mongoce - extras_version: "{{ mongo_extras_version }}" + extras_version: "{{ mas_catalog_metadata.mongo_extras_version_default }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_mongoce vars: manifest_name: extras_mongoce - manifest_version: "{{ mongo_extras_version }}" + manifest_version: "{{ mas_catalog_metadata.mongo_extras_version_default }}" # Mirror Mongo v4 specifically when requested - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_mode != "from-filesystem" - mirror_mongoce_v4 vars: extras_name: mongoce - extras_version: "{{ mongo_extras_version_4 }}" + extras_version: "{{ mas_catalog_metadata.mongo_extras_version_4 }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_mongoce_v4 vars: manifest_name: extras_mongoce - manifest_version: "{{ mongo_extras_version_4 }}" + manifest_version: "{{ mas_catalog_metadata.mongo_extras_version_4 }}" # Mirror Mongo v5 specifically when requested - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_mode != "from-filesystem" - mirror_mongoce_v5 vars: extras_name: mongoce - extras_version: "{{ mongo_extras_version_5 }}" + extras_version: "{{ mas_catalog_metadata.mongo_extras_version_5 }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_mongoce_v5 vars: manifest_name: extras_mongoce - manifest_version: "{{ mongo_extras_version_5 }}" + manifest_version: "{{ mas_catalog_metadata.mongo_extras_version_5 }}" # Mirror Mongo v6 specifically when requested - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_mode != "from-filesystem" - mirror_mongoce_v6 vars: extras_name: mongoce - extras_version: "{{ mongo_extras_version_6 }}" + extras_version: "{{ mas_catalog_metadata.mongo_extras_version_6 }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_mongoce_v6 vars: manifest_name: extras_mongoce - manifest_version: "{{ mongo_extras_version_6 }}" + manifest_version: "{{ mas_catalog_metadata.mongo_extras_version_6 }}" # Mirror Mongo v7 specifically when requested - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_mode != "from-filesystem" - mirror_mongoce_v7 vars: extras_name: mongoce - extras_version: "{{ mongo_extras_version_7 }}" + extras_version: "{{ mas_catalog_metadata.mongo_extras_version_7 }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_mongoce_v7 vars: manifest_name: extras_mongoce - manifest_version: "{{ mongo_extras_version_7 }}" + manifest_version: "{{mas_catalog_metadata.mongo_extras_version_7 }}" # 3. IBM Cloud Pak Foundation Services # ------------------------------------------------------------------------- # Just mirror IBM Cloud Pak Foundation Services images 'mirror_common_svcs' is set, or if CPD 4.8+ is being mirrored - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_common_svcs or (cpd_48_or_higher and mirror_cp4d) - mirror_mode != "from-filesystem" vars: case_name: ibm-cp-common-services - case_version: "{{ common_svcs_version }}" + case_version: "{{ mas_catalog_metadata.common_svcs_version }}" exclude_images: - ibm-auditlogging - ibm-cpp @@ -191,22 +194,46 @@ - ibm-events-operator ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_common_svcs or (cpd_48_or_higher and mirror_cp4d) vars: manifest_name: ibm-cp-common-services - manifest_version: "{{ common_svcs_version }}" + manifest_version: "{{ mas_catalog_metadata.common_svcs_version }}" + + - role: ibm.mas_devops.mirror_case_prepare + when: + - mirror_cp4d + - mas_catalog_metadata.ibm_zen_version is defined + - mirror_mode != "from-filesystem" + vars: + case_name: ibm-zen + case_version: "{{ mas_catalog_metadata.ibm_zen_version }}" + exclude_images: + - ibm-auditlogging + - ibm-cpp + - ibm-cs-healthcheck + - ibm-cs-monitoring + - ibm-events-operator + ibmpak_skip_dependencies: false + + - role: ibm.mas_devops.mirror_images + when: + - mirror_cp4d + - mas_catalog_metadata.ibm_zen_version is defined + vars: + manifest_name: ibm-zen + manifest_version: "{{ mas_catalog_metadata.ibm_zen_version }}" # Mirroring another version for Cloud Pak Foundation Services - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_common_svcs - mirror_mode != "from-filesystem" - - common_svcs_version_1 is defined - - common_svcs_version_1 != "" + - mas_catalog_metadata.common_svcs_version_1 is defined + - mas_catalog_metadata.common_svcs_version_1 != "" vars: case_name: ibm-cp-common-services - case_version: "{{ common_svcs_version_1 }}" + case_version: "{{ mas_catalog_metadata.common_svcs_version_1 }}" exclude_images: - ibm-auditlogging - ibm-cpp @@ -215,67 +242,67 @@ - ibm-events-operator ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: - mirror_common_svcs - - common_svcs_version_1 is defined - - common_svcs_version_1 != "" + - mas_catalog_metadata.common_svcs_version_1 is defined + - mas_catalog_metadata.common_svcs_version_1 != "" vars: manifest_name: ibm-cp-common-services - manifest_version: "{{ common_svcs_version_1 }}" + manifest_version: "{{ mas_catalog_metadata.common_svcs_version_1 }}" # 4. IBM Suite License Service # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_sls - mirror_mode != "from-filesystem" vars: case_name: ibm-sls - case_version: "{{ lookup('env', 'SLS_VERSION') | default (sls_version, True) }}" + case_version: "{{ lookup('env', 'SLS_VERSION') | default (mas_catalog_metadata.sls_version, True) }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_sls vars: manifest_name: ibm-sls - manifest_version: "{{ lookup('env', 'SLS_VERSION') | default (sls_version, True) }}" + manifest_version: "{{ lookup('env', 'SLS_VERSION') | default (mas_catalog_metadata.sls_version, True) }}" # 5. IBM Truststore Manager # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_tsm - mirror_mode != "from-filesystem" vars: case_name: ibm-truststore-mgr - case_version: "{{ lookup('env', 'TSM_VERSION') | default (tsm_version, True) }}" + case_version: "{{ lookup('env', 'TSM_VERSION') | default (mas_catalog_metadata.tsm_version, True) }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_tsm vars: manifest_name: ibm-truststore-mgr - manifest_version: "{{ lookup('env', 'TSM_VERSION') | default (tsm_version, True) }}" + manifest_version: "{{ lookup('env', 'TSM_VERSION') | default (mas_catalog_metadata.tsm_version, True) }}" # 6.1 CP4D Platform # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_cp4d - mirror_mode != "from-filesystem" vars: case_name: ibm-cp-datacore - case_version: "{{ cp4d_platform_version }}" + case_version: "{{ mas_catalog_metadata.cp4d_platform_version }}" exclude_images: [] ibmpak_skip_dependencies: true - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_cp4d - cpd_48 @@ -284,7 +311,7 @@ extras_name: cp4d extras_version: "4.8.0" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: - mirror_cp4d - cpd_48 @@ -292,239 +319,239 @@ manifest_name: extras_cp4d manifest_version: "4.8.0" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_cp4d vars: manifest_name: ibm-cp-datacore - manifest_version: "{{ cp4d_platform_version }}" - + manifest_version: "{{ mas_catalog_metadata.cp4d_platform_version }}" # 6.2 CP4D Platform - IBM Licensing dependency # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - cpd_48_or_higher - mirror_cp4d - mirror_mode != "from-filesystem" vars: case_name: ibm-licensing - case_version: "{{ ibm_licensing_version }}" + case_version: "{{ mas_catalog_metadata.ibm_licensing_version }}" exclude_images: [] ibmpak_skip_dependencies: true - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: - cpd_48_or_higher - mirror_cp4d - mirror_mode != "from-filesystem" vars: manifest_name: ibm-licensing - manifest_version: "{{ ibm_licensing_version }}" + manifest_version: "{{ mas_catalog_metadata.ibm_licensing_version }}" # 6.3 IBM Watson Studio and Watson Machine Learning dependency - ibm-ccs # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_wsl or mirror_wml - mirror_mode != "from-filesystem" vars: case_name: ibm-ccs - case_version: "{{ wsl_version }}" + case_version: "{{ mas_catalog_metadata.wsl_version }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_wsl or mirror_wml vars: manifest_name: ibm-ccs - manifest_version: "{{ wsl_version }}" + manifest_version: "{{ mas_catalog_metadata.wsl_version }}" # 7.1 IBM Watson Studio dependency - ibm-datarefinery # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_wsl - mirror_mode != "from-filesystem" vars: case_name: ibm-datarefinery - case_version: "{{ wsl_version }}" + case_version: "{{ mas_catalog_metadata.wsl_version }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_wsl vars: manifest_name: ibm-datarefinery - manifest_version: "{{ wsl_version }}" + manifest_version: "{{ mas_catalog_metadata.wsl_version }}" # 7.2 IBM Watson Studio dependency - ibm-wsl-runtimes # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_wsl - mirror_mode != "from-filesystem" vars: case_name: ibm-wsl-runtimes - case_version: "{{ wsl_version }}" + case_version: "{{ mas_catalog_metadata.wsl_version }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_wsl vars: manifest_name: ibm-wsl-runtimes - manifest_version: "{{ wsl_version }}" + manifest_version: "{{ mas_catalog_metadata.wsl_version }}" # 7.3 IBM Watson Studio dependency - ibm-elasticsearch-operator # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_wsl - mirror_mode != "from-filesystem" vars: case_name: ibm-elasticsearch-operator - case_version: "{{ elasticsearch_version }}" + case_version: "{{ mas_catalog_metadata.elasticsearch_version }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_wsl vars: manifest_name: ibm-elasticsearch-operator - manifest_version: "{{ elasticsearch_version }}" + manifest_version: "{{ mas_catalog_metadata.elasticsearch_version }}" # 7.4 IBM Watson Studio # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_wsl - mirror_mode != "from-filesystem" vars: case_name: ibm-wsl - case_version: "{{ wsl_version }}" + case_version: "{{ mas_catalog_metadata.wsl_version }}" exclude_images: [] ibmpak_skip_dependencies: true - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_wsl vars: manifest_name: ibm-wsl - manifest_version: "{{ wsl_version }}" + manifest_version: "{{ mas_catalog_metadata.wsl_version }}" # 8. IBM Watson Machine Learning # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_wml - mirror_mode != "from-filesystem" vars: case_name: ibm-wml-cpd - case_version: "{{ wml_version }}" + case_version: "{{ mas_catalog_metadata.wml_version }}" exclude_images: [] ibmpak_skip_dependencies: true - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_wml vars: manifest_name: ibm-wml-cpd - manifest_version: "{{ wml_version }}" + manifest_version: "{{ mas_catalog_metadata.wml_version }}" # 9. Analytics Engine (Spark) # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_spark - mirror_mode != "from-filesystem" vars: case_name: ibm-analyticsengine - case_version: "{{ spark_version }}" + case_version: "{{ mas_catalog_metadata.spark_version }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_spark vars: manifest_name: ibm-analyticsengine - manifest_version: "{{ spark_version }}" + manifest_version: "{{ mas_catalog_metadata.spark_version }}" # 10. IBM Db2u # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_db2u - mirror_mode != "from-filesystem" vars: case_name: ibm-db2uoperator - case_version: "{{ db2u_version }}" + case_version: "{{ mas_catalog_metadata.db2u_version }}" exclude_images: [] - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_db2u vars: manifest_name: "ibm-db2uoperator" - manifest_version: "{{ db2u_version }}" + manifest_version: "{{ mas_catalog_metadata.db2u_version }}" mirror_single_arch: amd64 - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_db2u - mirror_mode != "from-filesystem" vars: extras_name: db2u - extras_version: "{{ db2u_extras_version }}" + extras_version: "{{ mas_catalog_metadata.db2u_extras_version }}" - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_db2u vars: manifest_name: extras_db2u - manifest_version: "{{ db2u_extras_version }}" + manifest_version: "{{ mas_catalog_metadata.db2u_extras_version }}" # 11. Cognos Analytics # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_cognos - mirror_mode != "from-filesystem" vars: case_name: ibm-cognos-analytics-prod - case_version: "{{ cognos_version }}" + case_version: "{{ mas_catalog_metadata.cognos_version }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_cognos vars: manifest_name: ibm-cognos-analytics-prod - manifest_version: "{{ cognos_version }}" + manifest_version: "{{ mas_catalog_metadata.cognos_version }}" # 12. App Connect # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_case_prepare + - role: ibm.mas_devops.mirror_case_prepare when: - mirror_appconnect - mirror_mode != "from-filesystem" vars: case_name: ibm-appconnect - case_version: "{{ appconnect_version }}" + case_version: "{{ mas_catalog_metadata.appconnect_version }}" exclude_images: [] ibmpak_skip_dependencies: false - - name: ibm.mas_devops.mirror_images + - role: ibm.mas_devops.mirror_images when: mirror_appconnect vars: manifest_name: ibm-appconnect - manifest_version: "{{ appconnect_version }}" + manifest_version: "{{ mas_catalog_metadata.appconnect_version }}" + - # 14. odf - some extras needed + # 13. ODF - some extras needed # ------------------------------------------------------------------------- - - name: ibm.mas_devops.mirror_extras_prepare + - role: ibm.mas_devops.mirror_extras_prepare when: - mirror_odf - mirror_mode != "from-filesystem" diff --git a/ibm/mas_devops/plugins/action/apply_subscription.py b/ibm/mas_devops/plugins/action/apply_subscription.py new file mode 100644 index 0000000000..f76ef3d0af --- /dev/null +++ b/ibm/mas_devops/plugins/action/apply_subscription.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.common import get_api_client +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase + +from mas.devops.olm import applySubscription, OLMException + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Apply Subscription" + ibm.mas_devops.apply_subscription: + namespace: test-namespace + package_name: ibm-sls + package_channel: 3.x + register: subscription + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Target subscription + namespace = self._task.args.get('namespace', None) + config = self._task.args.get('config', None) + + # From which package? + packageName = self._task.args.get('package_name', None) + packageChannel = self._task.args.get('package_channel', None) + + # From which catalog? + catalogSource = self._task.args.get('catalog_source', None) + catalogSourceNamespace = self._task.args.get('catalog_source_namespace', None) + + if namespace is None: + raise AnsibleError(f"Error: namespace argument was not provided") + if not isinstance(packageName, str): + raise AnsibleError(f"Error: packageName argument is not a string") + + # Initialize DynamicClient and apply the Subscription + dynClient = get_api_client() + try: + subscription = applySubscription(dynClient, namespace, packageName, packageChannel, catalogSource, catalogSourceNamespace, config) + except OLMException as e: + raise AnsibleError(f"Error applying subscription: {e}") + + if subscription is None: + return dict( + message=f"Failed to apply subscription for {packageName} in {namespace}", + success=False, + failed=True, + changed=False + ) + + return dict( + message=f"Successfully applied subscription for {packageName} in {namespace}: {subscription.metadata.name}", + success=True, + failed=False, + changed=False, + **subscription.to_dict() + ) diff --git a/ibm/mas_devops/plugins/action/get_catalog_info.py b/ibm/mas_devops/plugins/action/get_catalog_info.py new file mode 100644 index 0000000000..0d3c3c303d --- /dev/null +++ b/ibm/mas_devops/plugins/action/get_catalog_info.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase + +from mas.devops.data import getCatalog + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Load catalog metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + fail_if_catalog_does_not_exist: true + register: mas_catalog_metadata + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + catalogId = self._task.args.get('mas_catalog_version', None) + failIfCatalogDoesNotExist = self._task.args.get('fail_if_catalog_does_not_exist', False) + + if catalogId is None: + raise AnsibleError(f"Error: mas_catalog_version argument was not provided") + if not isinstance(catalogId, str): + raise AnsibleError(f"Error: mas_catalog_version argument is not a string") + if not isinstance(failIfCatalogDoesNotExist, bool): + raise AnsibleError(f"Error: fail_if_catalog_does_not_exist argument is not a boolean") + + catalogData = getCatalog(catalogId) + + if catalogData is None: + return dict( + message=f"Failed to load catalog information for {catalogId}", + success=False, + failed=failIfCatalogDoesNotExist, + changed=False, + id=catalogId + ) + + return dict( + message=f"Successfully loaded catalog information for {catalogId}", + success=True, + failed=False, + changed=False, + id=catalogId, + **catalogData + ) diff --git a/ibm/mas_devops/plugins/action/get_default_storage_classes.py b/ibm/mas_devops/plugins/action/get_default_storage_classes.py new file mode 100644 index 0000000000..c87b8d1dae --- /dev/null +++ b/ibm/mas_devops/plugins/action/get_default_storage_classes.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.common import get_api_client +from ansible.plugins.action import ActionBase + +from mas.devops.mas import getDefaultStorageClasses + + +# Disabling warnings will prevent InsecureRequestWarnings from dynClient +urllib3.disable_warnings() + + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Lookup default storage classes" + ibm.mas_devops.get_default_storage_classes: + register: defaultStorageClasses + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + dynClient = get_api_client() + storageClasses = getDefaultStorageClasses(dynClient) + + # We don't want to fail if we can't find any default storage classes, doing so will + # result in roles/playbooks failing in environments where none of the default + # storage classes are available. We use the success=false to track when we couldn't + # find a default storage class, which does not trigger Ansible treating the action as + # failed. + if storageClasses.provider is None: + return dict( + message=f"Failed to find any default supported storage classes", + success=False, + failed=False, + changed=False, + **vars(storageClasses) + ) + + return dict( + message=f"Successfully found default storage classes ({storageClasses.provider})", + success=True, + failed=False, + changed=False, + **vars(storageClasses) + ) diff --git a/ibm/mas_devops/plugins/action/get_newest_catalog_tag.py b/ibm/mas_devops/plugins/action/get_newest_catalog_tag.py new file mode 100644 index 0000000000..2c70cd5112 --- /dev/null +++ b/ibm/mas_devops/plugins/action/get_newest_catalog_tag.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError + +from mas.devops.data import getNewestCatalogTag + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + arch = self._task.args['arch'] + catalogTag = getNewestCatalogTag(arch) + + if catalogTag is None: + raise AnsibleError(f"Error: No catalogs available for {arch}") + + return dict( + message=f"Successfully found newest catalog for {arch}", + failed=False, + changed=False, + arch=arch, + result=catalogTag + ) diff --git a/ibm/mas_devops/plugins/action/update_ibm_entitlement.py b/ibm/mas_devops/plugins/action/update_ibm_entitlement.py new file mode 100644 index 0000000000..7bd98b146c --- /dev/null +++ b/ibm/mas_devops/plugins/action/update_ibm_entitlement.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.common import get_api_client +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase + +from mas.devops.ocp import createNamespace +from mas.devops.mas import updateIBMEntitlementKey + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Update IBM Entitlement Key" + ibm.mas_devops.update_ibm_entitlement: + namespace: "{{ mas_namespace }}" + icr_username: "{{ icr_username }}" + icr_password: "{{ icr_password }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + register: secret + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + namespace = self._task.args.get('namespace', None) + icrUsername = self._task.args.get('icr_username', None) + icrPassword = self._task.args.get('icr_password', None) + artifactoryUsername = self._task.args.get('artifactory_username', None) + artifactoryPassword = self._task.args.get('artifactory_password', None) + secretName = self._task.args.get('secret_name', None) + + if namespace is None: + raise AnsibleError(f"Error: namespace argument was not provided") + + # Initialize DynamicClient, ensure the namespace exists, and create/update the entitlement secret + dynClient = get_api_client() + createNamespace(dynClient, namespace) + secret = updateIBMEntitlementKey(dynClient, namespace, icrUsername, icrPassword, artifactoryUsername, artifactoryPassword, secretName) + + return dict( + message=f"Successfully updated IBM entitlement in {namespace}: {secret.metadata.name}", + success=True, + failed=False, + changed=False, + **secret.to_dict() + ) diff --git a/ibm/mas_devops/plugins/filter/filters.py b/ibm/mas_devops/plugins/filter/filters.py index d4cf386aa0..bac70ec613 100644 --- a/ibm/mas_devops/plugins/filter/filters.py +++ b/ibm/mas_devops/plugins/filter/filters.py @@ -8,6 +8,7 @@ import yaml import re + def private_vlan(vlans): """ filter: private_vlan @@ -49,6 +50,7 @@ def public_vlan(vlans): public_vlan = [x['id'] for x in vlans if x['type'] == 'public'][0] return public_vlan + def appws_components(components): """ filter: appws_components @@ -75,6 +77,7 @@ def appws_components(components): return components_yaml + def db2_overwrite_config(components): """ filter: db2_overwrite_config @@ -102,6 +105,7 @@ def db2_overwrite_config(components): return components_yaml + def string2dict(_string = None): """ filter: string2dict @@ -130,6 +134,7 @@ def string2dict(_string = None): _dict = {} return _dict + def getAnnotations(annotations = None): """ filter: getAnnotations @@ -158,6 +163,7 @@ def getAnnotations(annotations = None): annotation_dict = {} return annotation_dict + def addAnnotationBlock(cr_definition,annotation_block = None): """ filter: addAnnotationBlock @@ -195,6 +201,7 @@ def addAnnotationBlock(cr_definition,annotation_block = None): return cr_definition + def getResourceNames(resourceList): """ filter: getResourceNames @@ -215,30 +222,6 @@ def getResourceNames(resourceList): resourceNames.append(resource['metadata']['name']) return resourceNames -def defaultStorageClass(storageClassLookup, storageClassOptions): - """ - filter: defaultStorageClass - author: David Parker - version_added: 10.0 - short_description: Return a dict of known storage classes - description: - - This filter returns the name of an available storage class from the list of options provided - options: - _storageClassLookup: - description: list of storageclass resources - required: True - _storageClassOptions: - description: list of storageclasses that are supported, the first one found in the results will be used - required: True - notes: - - limited error handling, will not handle unexpected data currently - """ - for classOptionName in storageClassOptions: - for storageClass in storageClassLookup["resources"]: - if storageClass['metadata']['name'] == classOptionName: - return classOptionName - # We couldn't find a suitable storage class - return "" def getWSLProjectId(wslProjectLookup, wslProjectName): """ @@ -264,6 +247,7 @@ def getWSLProjectId(wslProjectLookup, wslProjectName): # Project not found return "" + def setManagePVC(data, mountPath, pvcName, pvcSize, storageClassName, accessMode, volumeName = None): """ filter: setManagePVC @@ -293,7 +277,7 @@ def setManagePVC(data, mountPath, pvcName, pvcSize, storageClassName, accessMode """ pvc_list = [] - persistentVolumes = { + persistentVolumes = { "accessModes": [accessMode], "mountPath": mountPath, "pvcName": pvcName, @@ -308,6 +292,7 @@ def setManagePVC(data, mountPath, pvcName, pvcSize, storageClassName, accessMode pvc_list.append(pvc) return pvc_list + def setManageBirtProperties(data, rptRoute, rptServerBundleName): sb_list = [] rpt_bundle = { @@ -345,6 +330,7 @@ def setManageDoclinksProperties(data, doclinkPath01, bucketName, accessKey, secr sb_list.append(sb) return sb_list + def setManageFsDoclinksProperties(data, manage_url): sb_list = [] for sb in data: @@ -356,6 +342,7 @@ def setManageFsDoclinksProperties(data, manage_url): sb_list.append(sb) return sb_list + def _setSystemProperties(data, meaweb_value, oslc_rest_value, webapp_value, rest_webapp_value): sb_list = [] @@ -369,6 +356,7 @@ def _setSystemProperties(data, meaweb_value, oslc_rest_value, webapp_value, rest sb_list.append(sb) return sb_list + def format_pre_version_with_plus(data): """ Versions in format 9.0.0-pre.stable-3757 cannot be used to compare with the version @@ -379,6 +367,7 @@ def format_pre_version_with_plus(data): return data return data[::-1].replace("-", "+", 1)[::-1] + def format_pre_version_without_buildid(data): """ Versions in format 9.0.0-pre.stable-3757 cannot be used to compare with the version @@ -389,6 +378,7 @@ def format_pre_version_without_buildid(data): return data return data[:data.rfind("-")] + def get_db2_instance_name(binding_scope, mas_instance_id, mas_workspace_id, mas_application_id): if binding_scope == "": return "" @@ -399,6 +389,7 @@ def get_db2_instance_name(binding_scope, mas_instance_id, mas_workspace_id, mas_ } return jdbc_instance_names[binding_scope] + class FilterModule(object): def filters(self): return { @@ -410,7 +401,6 @@ def filters(self): 'string2dict': string2dict, 'getAnnotations': getAnnotations, 'getResourceNames': getResourceNames, - 'defaultStorageClass': defaultStorageClass, 'getWSLProjectId': getWSLProjectId, 'setManagePVC': setManagePVC, 'setManageBirtProperties': setManageBirtProperties, diff --git a/ibm/mas_devops/requirements.txt b/ibm/mas_devops/requirements.txt new file mode 100644 index 0000000000..6974c01c5a --- /dev/null +++ b/ibm/mas_devops/requirements.txt @@ -0,0 +1 @@ +mas-devops >= 1.11.1 diff --git a/ibm/mas_devops/roles/aibroker/defaults/main.yml b/ibm/mas_devops/roles/aibroker/defaults/main.yml index 49d6eb6674..c3b5aee6c6 100644 --- a/ibm/mas_devops/roles/aibroker/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker/defaults/main.yml @@ -17,6 +17,10 @@ mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | defaul ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" +# Development Registry Entitlement +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + # MAS Annotation block # ----------------------------------------------------------------------------- mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 882666cfab..dd93accf78 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -17,18 +17,25 @@ kind: Namespace name: "{{ aibroker_namespace }}" + # Install the operator & create entitlement secret # ----------------------------------------------------------------------------- -- name: "Install IBM Maximo AI Broker Operator" - include_role: - name: ibm.mas_devops.install_operator - vars: +- name: "Create IBM Entitlement Key" + ibm.mas_devops.update_ibm_entitlement: namespace: "{{ aibroker_namespace }}" icr_username: "{{ mas_entitlement_username }}" icr_password: "{{ mas_entitlement_key }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + +- name: "Create ibm-mas-aibroker Subscription" + ibm.mas_devops.apply_subscription: + namespace: "{{ aibroker_namespace }}" + package_name: ibm-mas-aibroker + package_channel: "{{ aibroker_channel }}" catalog_source: "{{ mas_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/aibroker/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/aibroker/subscription.yml.j2') }}" + register: subscription + # Wait until the IBM Maximo AI Broker CRD is available # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/appconnect/README.md b/ibm/mas_devops/roles/appconnect/README.md index b0128900aa..43ce9da604 100644 --- a/ibm/mas_devops/roles/appconnect/README.md +++ b/ibm/mas_devops/roles/appconnect/README.md @@ -60,6 +60,7 @@ Storage class where AppConnect will be installed - for IBM Cloud clusters, `ibmc - **Required** - Environment Variable: `APPCONNECT_STORAGE_CLASS` - Default Value: None +- **Note**: The App Connect Dashboard requires a file-based storage class with ReadWriteMany (RWX) capability. ### appconnect_dashboard_name AppConnect dashboard instance name. Defaults to `dashboard-12040r2` as a reference to AppConnect Dashboard version `12.0.4.0-r2` that is compatible with the default subscription channel and license ID. diff --git a/ibm/mas_devops/roles/appconnect/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/appconnect/tasks/determine-storage-classes.yml index 3e0645b2a1..112f885650 100644 --- a/ibm/mas_devops/roles/appconnect/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/appconnect/tasks/determine-storage-classes.yml @@ -1,38 +1,25 @@ --- -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" -# 2. Set Storage (Required) +# 2. Set storage class # ----------------------------------------------------------------------------- -- name: Default Storage if not set by user - when: appconnect_storage_class is not defined or appconnect_storage_class == "" +- name: "Use default storage class (RWX)" + when: + - appconnect_storage_class is not defined or appconnect_storage_class == "" + - defaultStorageClasses.success set_fact: - appconnect_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" + appconnect_storage_class: "{{ defaultStorageClasses.rwx }}" - name: Assert that storage class has been defined assert: that: appconnect_storage_class is defined and appconnect_storage_class != "" fail_msg: "appconnect_storage_class must be defined" - -# 3. Debug storage class configuration -# ----------------------------------------------------------------------------- - name: "Debug AppConnect storage class configuration" debug: msg: - - "Storage class (appconnect) ...... {{ appconnect_storage_class }}" + - "Storage class (appconnect) ............. {{ appconnect_storage_class }}" diff --git a/ibm/mas_devops/roles/arcgis/defaults/main.yml b/ibm/mas_devops/roles/arcgis/defaults/main.yml index 1a36c08c93..7061a4ebf9 100644 --- a/ibm/mas_devops/roles/arcgis/defaults/main.yml +++ b/ibm/mas_devops/roles/arcgis/defaults/main.yml @@ -29,6 +29,10 @@ mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | defaul ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" +# Development Registry Entitlement +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + # MAS Annotation block # ----------------------------------------------------------------------------- mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" diff --git a/ibm/mas_devops/roles/arcgis/tasks/main.yml b/ibm/mas_devops/roles/arcgis/tasks/main.yml index 51f1b196e7..f5c5e78d31 100644 --- a/ibm/mas_devops/roles/arcgis/tasks/main.yml +++ b/ibm/mas_devops/roles/arcgis/tasks/main.yml @@ -43,18 +43,25 @@ state: present template: templates/namespace.yml.j2 + # 5. Install the operator & create entitlement secret # ----------------------------------------------------------------------------- -- name: "Install IBM Maximo Location Services for Esri Operator" - include_role: - name: ibm.mas_devops.install_operator - vars: +- name: "Create IBM Entitlement Key" + ibm.mas_devops.update_ibm_entitlement: namespace: "{{ mas_arcgis_namespace }}" icr_username: "{{ mas_entitlement_username }}" icr_password: "{{ mas_entitlement_key }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + +- name: "Create ibm-mas-arcgis Subscription" + ibm.mas_devops.apply_subscription: + namespace: "{{ mas_arcgis_namespace }}" + package_name: ibm-mas-arcgis + package_channel: "{{ mas_arcgis_channel }}" catalog_source: "{{ mas_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/subscription.yml.j2') }}" + register: subscription + # 6. Wait until the IBM Maximo Location Services CRD is available # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/arcgis/templates/operator-group.yml.j2 b/ibm/mas_devops/roles/arcgis/templates/operator-group.yml.j2 deleted file mode 100644 index 94d83a5905..0000000000 --- a/ibm/mas_devops/roles/arcgis/templates/operator-group.yml.j2 +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: ibm-mas-arcgis-operator-group - namespace: "{{ mas_arcgis_namespace }}" -{% if custom_labels is defined and custom_labels.items() %} - labels: -{% for key, value in custom_labels.items() %} - "{{ key }}": "{{ value }}" -{% endfor %} -{% endif %} -spec: - targetNamespaces: - - "{{ mas_arcgis_namespace }}" diff --git a/ibm/mas_devops/roles/arcgis/templates/subscription.yml.j2 b/ibm/mas_devops/roles/arcgis/templates/subscription.yml.j2 deleted file mode 100644 index 1e13237ee6..0000000000 --- a/ibm/mas_devops/roles/arcgis/templates/subscription.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: ibm-mas-arcgis-operator - namespace: "{{ mas_arcgis_namespace }}" -spec: - channel: "{{ mas_arcgis_channel }}" - installPlanApproval: Automatic - name: ibm-mas-arcgis - source: "{{ mas_catalog_source }}" - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/install.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/install.yml index fbdbea160c..5188054b1a 100644 --- a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/install.yml +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/install.yml @@ -70,11 +70,11 @@ # 4. Create RHCM Subscription # ----------------------------------------------------------------------------- - name: "install : Create Red Hat Certificate Manager Subscription" - include_tasks: "{{ role_path }}/../../common_tasks/create_subscription.yml" - vars: - subscription_namespace: "{{ cert_manager_operator_namespace }}" + ibm.mas_devops.apply_subscription: + namespace: "{{ cert_manager_operator_namespace }}" package_name: openshift-cert-manager-operator - channel_name: stable-v1 + package_channel: stable-v1 + register: subscription # 5. Wait for Subscription to be processed diff --git a/ibm/mas_devops/roles/cos/README.md b/ibm/mas_devops/roles/cos/README.md index ad9f907149..1bb0fdb1e2 100644 --- a/ibm/mas_devops/roles/cos/README.md +++ b/ibm/mas_devops/roles/cos/README.md @@ -1,23 +1,18 @@ cos -=== - +=============================================================================== This role provides support for: -- Provisioning and Configuring Cloud Object Storage in MAS. It currently supports two providers: - - - In-cluster Ceph Object Storage leveraging OpenShift Container Storage - - IBM Cloud Object Storage +- Provisioning and Configuring Cloud Object Storage in MAS. It currently supports two providers: + - In-cluster Ceph Object Storage leveraging OpenShift Container Storage + - IBM Cloud Object Storage - Deprovision Cloud Object Store. It currently supports one provider: - IBM Cloud Object Storage Currently this role only supports generating a system-scoped ObjectStorageCfg resource, but the generated file can be modified if you wish to use other scopes. -Role Variables --------------- - -# IBM Cloud Object Storage (ibm) -# --------------------------------------------------------------------------------------------------------------------- +Role Variables - General +------------------------------------------------------------------------------- ### cos_type Required. Which COS provider to use; can be set to either `ibm` for IBM Cloud Object Storage or `ocs` for OpenShift Container Storage @@ -30,6 +25,29 @@ Required. Which action you want to run for the COS instance. You can either `pr - Environment Variable: `COS_ACTION` - Default Value: `provision` +### cos_use_hmac +Supported values are true and false, this is used when ibm cloud-cos to be setup with hmac encrypted credentials. + +- Environment Variable: COS_USE_HMAC +- Default: false + +### cluster ingres tls secret name +Specify the name of the cluster's ingres tls secret which contains the default router certificate. + +- Optional +- Environment Variable: `OCP_INGRESS_TLS_SECRET_NAME` +- Default Value: router-certs-default + +### custom_labels +List of comma separated key=value pairs for setting custom labels on instance specific resources. + +- Optional +- Environment Variable: `CUSTOM_LABELS` +- Default Value: None + + +Role Variables - IBM COS +------------------------------------------------------------------------------- ### cos_instance_name Provide an optional name for the Object Storage instance. This is only used when cos_type is set to `ibm` for IBM Cloud Object Storage. @@ -54,20 +72,35 @@ Required (For Provisioning). The COS region location url endpoint. Needed to gen ### cos_resource_key_iam_role Provide an optional role when cos service credential is getting created during COS provisioning. - Environment Variable: `COS_RESOURCE_KEY_IAM_ROLE` - - Default Value: `Manager` + - Default Value: `Manager` + +### cos_apikey +Required if cos_type is set to `ibm`. Provide your less priviledged IBM Cloud API Key for COS only + +- Environment Variable: `COS_APIKEY` +- Default Value: `ibmcloud_apikey` ### ibmcloud_apikey -Required if cos_type is set to `ibm`. Provide your IBM Cloud API Key. +Required if cos_type is set to `ibm`. Provide your IBM Cloud API Key that will be used as the default API Key across multiple roles in this collection. - Environment Variable: `IBMCLOUD_APIKEY` - Default Value: None -### ibmcloud_resourcegroup +### cos_resourcegroup Only used when cos_type is set to `ibm`. Provide the name of the resource group which will own the COS instance. +- Environment Variable: `COS_RESOURCEGROUP` +- Default Value: `ibmcloud_resourcegroup` + +### ibmcloud_resourcegroup +Only used when cos_type is set to `ibm`. Provide the name of the default resource group used across multiple roles in this collection. + - Environment Variable: `IBMCLOUD_RESOURCEGROUP` - Default Value: `Default` + +Role Variables - MAS Configuration +------------------------------------------------------------------------------- ### mas_instance_id The instance ID of Maximo Application Suite that the ObjectStorageCfg configuration will target. If this or `mas_config_dir` are not set then the role will not generate a ObjectStorageCfg template. @@ -80,26 +113,6 @@ Local directory to save the generated ObjectStorageCfg resource definition. Thi - Environment Variable: `MAS_CONFIG_DIR` - Default Value: None -### cos_use_hmac -Supported values are true and false, this is used when ibm cloud-cos to be setup with hmac encrypted credentials. - -- Environment Variable: COS_USE_HMAC -- Default: false - -### cluster ingres tls secret name -Specify the name of the cluster's ingres tls secret which contains the default router certificate. - -- Optional -- Environment Variable: `OCP_INGRESS_TLS_SECRET_NAME` -- Default Value: router-certs-default - -### custom_labels -List of comma separated key=value pairs for setting custom labels on instance specific resources. - -- Optional -- Environment Variable: `CUSTOM_LABELS` -- Default Value: None - ### include_cluster_ingress_cert_chain Optional. When set to `True`, includes the complete certificates chain in the generated MAS configuration, when a trusted certificate authority is found in your cluster's ingress. @@ -107,10 +120,12 @@ Optional. When set to `True`, includes the complete certificates chain in the ge - Environment Variable: `INCLUDE_CLUSTER_INGRESS_CERT_CHAIN` - Default: `False` + Example Playbook ----------------- +------------------------------------------------------------------------------- Create the Ceph Object store on the existing OCS cluster and prepare the objectstorageCfg yaml to mas_config_dir. + ```yaml - hosts: localhost any_errors_fatal: true @@ -121,7 +136,9 @@ Create the Ceph Object store on the existing OCS cluster and prepare the objects roles: - ibm.mas_devops.cos ``` + Create the IBM Cloud Object storage Instance and prepare the objectstorageCfg yaml to mas_config_dir. + ```yaml - hosts: localhost any_errors_fatal: true @@ -134,6 +151,5 @@ Create the IBM Cloud Object storage Instance and prepare the objectstorageCfg ya - ibm.mas_devops.cos ``` License -------- - +------------------------------------------------------------------------------- EPL-2.0 diff --git a/ibm/mas_devops/roles/cos/defaults/main.yml b/ibm/mas_devops/roles/cos/defaults/main.yml index bac614fc2e..74c7e019d7 100644 --- a/ibm/mas_devops/roles/cos/defaults/main.yml +++ b/ibm/mas_devops/roles/cos/defaults/main.yml @@ -19,6 +19,7 @@ cos_include_isrg_root_cert: "{{ lookup('env', 'COS_INCLUDE_ISRG_ROOT_CERT') | de ibmcloud_apikey: "{{ lookup('env', 'IBMCLOUD_APIKEY') }}" ibmcloud_resourcegroup: "{{ lookup('env', 'IBMCLOUD_RESOURCEGROUP') | default('Default', true) }}" cos_resourcegroup: "{{ lookup('env', 'COS_RESOURCEGROUP') | default(ibmcloud_resourcegroup, true) }}" +cos_apikey: "{{ lookup('env', 'COS_APIKEY') | default(ibmcloud_apikey, true) }}" cos_instance_name: "{{ lookup('env', 'COS_INSTANCE_NAME') }}" cos_location_info: "{{ lookup('env', 'COS_LOCATION') | default('global', True) }}" cos_plan_type: "{{ lookup('env', 'COS_PLAN') | default('standard', True) }}" diff --git a/ibm/mas_devops/roles/cos/tasks/main.yml b/ibm/mas_devops/roles/cos/tasks/main.yml index 4ab71c491f..46743bc218 100644 --- a/ibm/mas_devops/roles/cos/tasks/main.yml +++ b/ibm/mas_devops/roles/cos/tasks/main.yml @@ -9,7 +9,7 @@ that: cos_action is defined and cos_action != "" fail_msg: "cos_action property is required" -# 1. Configure COS using IBM Cloud Object Storage (ibmcos) or OpenShift Container Storage (ocs) +# 1. Configure COS using IBM Cloud Object Storage (COS) or OpenShift Container Storage (ocs) # --------------------------------------------------------------------------------------------------------------------- - name: "Run the task for the appropriate provider" include_tasks: tasks/providers/{{ cos_type }}/{{ cos_action }}.yml diff --git a/ibm/mas_devops/roles/cos/tasks/providers/ibm/deprovision.yml b/ibm/mas_devops/roles/cos/tasks/providers/ibm/deprovision.yml index 393b575d7b..d4427e1aec 100644 --- a/ibm/mas_devops/roles/cos/tasks/providers/ibm/deprovision.yml +++ b/ibm/mas_devops/roles/cos/tasks/providers/ibm/deprovision.yml @@ -1,8 +1,8 @@ --- -- name: "Fail if ibmcloud_apikey is not provided" +- name: "Fail if cos_apikey is not provided" assert: - that: ibmcloud_apikey is defined and ibmcloud_apikey != "" - fail_msg: "ibmcloud_apikey property is required" + that: cos_apikey is defined and cos_apikey != "" + fail_msg: "cos_apikey property is required" # 1. Determine the instance name # --------------------------------------------------------------------------------------------------------------------- @@ -39,7 +39,7 @@ - name: Fetch IBM Cloud Resource Group Id ibm.cloudcollection.ibm_resource_group_info: name: "{{ cos_resourcegroup }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: rg_info - name: Fetch IBM COS Information @@ -48,7 +48,7 @@ resource_group_id: "{{ rg_info.resource.id }}" service: "{{ cos_service }}" location: "{{ cos_location_info }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: cos_info failed_when: cos_info.rc != 0 and "No resource instance found" not in cos_info.stderr @@ -77,7 +77,7 @@ service: "cloud-object-storage" plan: "{{ cos_plan_type }}" location: "{{ cos_location_info }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" state: absent register: cos_deprovision_output diff --git a/ibm/mas_devops/roles/cos/tasks/providers/ibm/provision.yml b/ibm/mas_devops/roles/cos/tasks/providers/ibm/provision.yml index 5a4e3456bc..b490948dca 100644 --- a/ibm/mas_devops/roles/cos/tasks/providers/ibm/provision.yml +++ b/ibm/mas_devops/roles/cos/tasks/providers/ibm/provision.yml @@ -1,8 +1,8 @@ --- -- name: "Fail if ibmcloud_apikey is not provided" +- name: "Fail if cos_apikey is not provided" assert: - that: ibmcloud_apikey is defined and ibmcloud_apikey != "" - fail_msg: "ibmcloud_apikey property is required" + that: cos_apikey is defined and cos_apikey != "" + fail_msg: "cos_apikey property is required" # 1. Determine the instance name # --------------------------------------------------------------------------------------------------------------------- @@ -37,7 +37,7 @@ - name: "ibm : Retrieve resource group guid" ibm.cloudcollection.ibm_resource_group_info: name: "{{ cos_resourcegroup }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: rg_info - name: "ibm : Debug group GUID" @@ -61,10 +61,10 @@ service: "{{ cos_service }}" plan: "{{ cos_plan_type }}" location: "{{ cos_location_info }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: cos_output -- name: "Fail if IBMCOS resource ID not found" +- name: "Fail if COS resource ID not found" assert: that: - not cos_output.failed @@ -76,14 +76,14 @@ resource_group_id: "{{ resourceGID }}" service: "{{ cos_service }}" location: "{{ cos_location_info }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: cos_info -- name: "ibm : Debug IBMCOS resource ID" +- name: "ibm : Debug COS resource ID" debug: msg: "COS Instance resource ID ... {{ cos_info.resource.id }}" -- name: "ibmcos: Set resource instance id variable" +- name: "cos: Set resource instance id variable" when: cos_info.resource.id is defined set_fact: cos_resource_id: "{{ cos_info.resource.id }}" @@ -96,7 +96,7 @@ name: "{{ mas_instance_id }}-key" role: "{{ cos_resource_key_iam_role }}" resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" parameters: HMAC: "{{ cos_use_hmac }}" @@ -104,7 +104,7 @@ ibm.cloudcollection.ibm_resource_key_info: name: "{{ mas_instance_id }}-key" resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: cos_key_info - name: "ibm : Debug COS Key Resource" diff --git a/ibm/mas_devops/roles/cos_bucket/defaults/main.yml b/ibm/mas_devops/roles/cos_bucket/defaults/main.yml index 7ed2501c41..89e4cb8b9c 100644 --- a/ibm/mas_devops/roles/cos_bucket/defaults/main.yml +++ b/ibm/mas_devops/roles/cos_bucket/defaults/main.yml @@ -12,6 +12,8 @@ cos_bucket_action: "{{ lookup('env', 'COS_BUCKET_ACTION') | default('create', tr # --------------------------------------------------------------------------------------------------------------------- ibmcloud_apikey: "{{ lookup('env', 'IBMCLOUD_APIKEY') }}" ibmcloud_resourcegroup: "{{ lookup('env', 'IBMCLOUD_RESOURCEGROUP') | default('Default', true) }}" +cos_resourcegroup: "{{ lookup('env', 'COS_RESOURCEGROUP') | default(ibmcloud_resourcegroup, true) }}" +cos_apikey: "{{ lookup('env', 'COS_APIKEY') | default(ibmcloud_apikey, true) }}" cos_service: cloud-object-storage cos_instance_name: "{{ lookup('env', 'COS_INSTANCE_NAME') }}" cos_location_info: global diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create.yml index 5e00a002a2..1a71800ed6 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create.yml @@ -1,9 +1,9 @@ --- -# ibmcloud_apikey is required in order to be able to lookup cos resources -- name: "Fail if ibmcloud_apikey is not provided" +# cos_apikey is required in order to be able to lookup cos resources +- name: "Fail if cos_apikey is not provided" assert: - that: ibmcloud_apikey is defined and ibmcloud_apikey != "" - fail_msg: "ibmcloud_apikey property is required" + that: cos_apikey is defined and cos_apikey != "" + fail_msg: "cos_apikey property is required" # cos_instance_name is required in order to be able to create cos resources i.e credentials and buckets - name: "Fail if cos_instance_name is not provided" @@ -53,7 +53,7 @@ - name: "ibm : Retrieve resource group guid" ibm.cloudcollection.ibm_resource_group_info: name: "{{ ibmcloud_resourcegroup }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: rg_info - name: "ibm : Debug group GUID" @@ -75,7 +75,7 @@ resource_group_id: "{{ resourceGID }}" service: "{{ cos_service }}" location: "{{ cos_location_info }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: cos_info - name: "Debug IBM COS resource ID" diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_cross_region_location_bucket.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_cross_region_location_bucket.yml index 7d8acc7707..33609f922b 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_cross_region_location_bucket.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_cross_region_location_bucket.yml @@ -2,7 +2,7 @@ - name: "Lookup IBM COS bucket named '{{ cos_bucket_name }}' in COS instance '{{ cos_instance_name }}'" ibm.cloudcollection.ibm_cos_bucket_info: resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" bucket_name: "{{ cos_bucket_name }}" bucket_type: "cross_region_location" bucket_region: "{{ cos_bucket_region_location }}" @@ -23,7 +23,7 @@ - name: "Create cos bucket if it does not exist" ibm.cloudcollection.ibm_cos_bucket: resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" bucket_name: "{{ cos_bucket_name }}" storage_class: "{{ cos_bucket_storage_class }}" cross_region_location: "{{ cos_bucket_region_location }}" diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_region_location_bucket.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_region_location_bucket.yml index 3666c6ebc5..79bcd211be 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_region_location_bucket.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_region_location_bucket.yml @@ -2,7 +2,7 @@ - name: "Lookup IBM COS bucket named '{{ cos_bucket_name }}' in COS instance '{{ cos_instance_name }}'" ibm.cloudcollection.ibm_cos_bucket_info: resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" bucket_name: "{{ cos_bucket_name }}" bucket_type: "region_location" bucket_region: "{{ cos_bucket_region_location }}" @@ -23,7 +23,7 @@ - name: "Create cos bucket if it does not exist" ibm.cloudcollection.ibm_cos_bucket: resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" bucket_name: "{{ cos_bucket_name }}" storage_class: "{{ cos_bucket_storage_class }}" region_location: "{{ cos_bucket_region_location }}" diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_service_credentials.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_service_credentials.yml index cb016c917b..00eea0709f 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_service_credentials.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_service_credentials.yml @@ -6,14 +6,14 @@ name: "{{ cos_service_credentials_name }}" role: "{{ cos_resource_key_iam_role }}" resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" parameters: {'HMAC': true } - name: "Retrieve IBM COS service credentials {{ cos_service_credentials_name }}" ibm.cloudcollection.ibm_resource_key_info: name: "{{ cos_service_credentials_name }}" resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: cos_key_info - name: "IBM COS: Set access key id & secret access key variables" diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/delete.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/delete.yml index b3332151d9..1144bcbccb 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/delete.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/delete.yml @@ -1,9 +1,9 @@ --- -# ibmcloud_apikey is required in order to be able to lookup cos resources -- name: "Fail if ibmcloud_apikey is not provided" +# cos_apikey is required in order to be able to lookup cos resources +- name: "Fail if cos_apikey is not provided" assert: - that: ibmcloud_apikey is defined and ibmcloud_apikey != "" - fail_msg: "ibmcloud_apikey property is required" + that: cos_apikey is defined and cos_apikey != "" + fail_msg: "cos_apikey property is required" # cos_instance_name is required in order to be able to create cos resources i.e credentials and buckets - name: "Fail if cos_instance_name is not provided" @@ -29,7 +29,7 @@ - name: "ibm : Retrieve resource group guid" ibm.cloudcollection.ibm_resource_group_info: name: "{{ ibmcloud_resourcegroup }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: rg_info - name: "ibm : Debug group GUID" @@ -51,7 +51,7 @@ resource_group_id: "{{ resourceGID }}" service: "cloud-object-storage" location: "{{ cos_location_info }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" register: cos_info - name: "Set resource instance id variable" @@ -64,7 +64,7 @@ - name: "Lookup IBM COS bucket named '{{ cos_bucket_name }}' in COS instance '{{ cos_instance_name }}'" ibm.cloudcollection.ibm_cos_bucket_info: resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" bucket_name: "{{ cos_bucket_name }}" bucket_type: "{{ cos_bucket_region_location_type }}" bucket_region: "{{ cos_bucket_region_location }}" @@ -94,7 +94,7 @@ ibm.cloudcollection.ibm_cos_bucket: resource_instance_id: "{{ cos_bucket_resource_instance_id }}" id: "{{ cos_bucket_resource_id }}" - ibmcloud_api_key: "{{ ibmcloud_apikey }}" + ibmcloud_api_key: "{{ cos_apikey }}" bucket_name: "{{ cos_bucket_name }}" state: absent register: cos_bucket_delete_output diff --git a/ibm/mas_devops/roles/cp4d/README.md b/ibm/mas_devops/roles/cp4d/README.md index 4eed1845c8..2b9b48aa8b 100644 --- a/ibm/mas_devops/roles/cp4d/README.md +++ b/ibm/mas_devops/roles/cp4d/README.md @@ -10,7 +10,7 @@ Currently supported Cloud Pak for Data release versions are: - 4.6.4 - 4.6.6 - 4.8.0 - - 5.0.0 (Install only, currently upgrade from 4.8 is not supported) + - 5.0.0 The role will automatically install or upgrade (if targeted to an existing CPD deployment) the corresponding Zen version associated to the chosen Cloud Pak for Data release, for example: @@ -23,6 +23,12 @@ The role will automatically install or upgrade (if targeted to an existing CPD d For more information about CPD versioning, see [IBM Cloud Pak for Data Operator and operand versions 4.9.x](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=planning-operator-operand-versions) or [IBM Cloud Pak for Data Operator and operand versions 5.0.x](https://www.ibm.com/docs/en/cloud-paks/cp-data/5.0.x?topic=planning-operator-operand-versions) +!!! info + - Install CP4D ControlPlane (~1 hour) + - Install CP4D Services (~30 Minutes - 1 hour for each service) + + All timings are estimates. + Cloud Pak for Data version mapping to MAS Catalog ==== @@ -31,7 +37,7 @@ If `CPD_PRODUCT_VERSION` variable is not defined, then the automation will try t Upgrade ------------------ -This role also supports seamlessly CPD control plane (or also called `Zen` service) minor version upgrades (CPD 4.6.x > CPD 4.8.x), and patch version upgrades (i.e CPD 4.8.0 -> CPD 4.8.1). +This role also supports seamlessly CPD control plane (or also called `Zen` service) minor version upgrades (CPD 4.6.x > CPD 4.8.0 or CPD 4.8.0 > CPD 5.0.0), and patch version upgrades (i.e CPD 4.6.0 -> CPD 4.6.6). All you need to do is to define `cpd_product_version` variable to the version you target to upgrade and run this role against an existing CPD instance. For more information about IBM Cloud Pak for Data upgrade process, refer to the [Cloud Pak for Data official documentation](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=upgrading). @@ -53,6 +59,63 @@ Cloud Pak for Data will be configured as a [specialized installation](https://ww Cloud Pak for Data deployment details ------------------ +### 5.0.x version: + +Cloud Pak for Data 5.0.x leverages Cloud Pak Foundational Services v4, which runs its deployments in isolated/dedicated scope model, that means that its dependencies will be grouped and installed within the Cloud Pak for Data related projects/namespaces. There are only two namespaces that will be used: CPD instance namespace (e.g `ibm-cpd`) and CPD operators namespace (e.g `ibm-cpd-operators`). + +In the **ibm-cpd-operators** namespace: +```bash +oc -n ibm-cpd-operators get deployments + +NAME READY UP-TO-DATE AVAILABLE AGE +cpd-platform-operator-manager 1/1 1 1 17h +ibm-common-service-operator 1/1 1 1 17h +ibm-namespace-scope-operator 1/1 1 1 17h +ibm-zen-operator 1/1 1 1 17h +meta-api-deploy 1/1 1 1 17h +operand-deployment-lifecycle-manager 1/1 1 1 17h +postgresql-operator-controller-manager-1-18-7 1/1 1 1 17h +``` + +In the **ibm-cpd** namespace: +```bash +oc -n ibm-cpd get zenservice,ibmcpd,deployments,sts,pvc + +NAME VERSION STATUS AGE +zenservice.zen.cpd.ibm.com/lite-cr 6.0.1 Completed 17h + +NAME AGE +ibmcpd.cpd.ibm.com/ibmcpd-cr 17h + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/ibm-mcs-hubwork 1/1 1 1 17h +deployment.apps/ibm-mcs-placement 1/1 1 1 17h +deployment.apps/ibm-mcs-storage 1/1 1 1 17h +deployment.apps/ibm-nginx 3/3 3 3 16h +deployment.apps/ibm-nginx-tester 1/1 1 1 16h +deployment.apps/usermgmt 3/3 3 3 16h +deployment.apps/zen-audit 2/2 2 2 16h +deployment.apps/zen-core 3/3 3 3 16h +deployment.apps/zen-core-api 3/3 3 3 16h +deployment.apps/zen-watchdog 2/2 2 2 16h +deployment.apps/zen-watcher 1/1 1 1 16h + +NAME READY AGE +statefulset.apps/zen-minio 3/3 17h + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +persistentvolumeclaim/export-zen-minio-0 Bound pvc-b2a2a729-13c1-4e7f-b672-0b5efc6aa40a 20Gi RWO ibmc-block-gold 17h +persistentvolumeclaim/export-zen-minio-1 Bound pvc-7e772a3a-8849-4291-8e14-501f49e79182 20Gi RWO ibmc-block-gold 17h +persistentvolumeclaim/export-zen-minio-2 Bound pvc-e0dd31dc-916d-4b15-9d9c-351db0a2b47f 20Gi RWO ibmc-block-gold 17h +persistentvolumeclaim/ibm-cs-postgres-backup Bound pvc-ef788b99-784f-4531-a1b3-12611f112551 20Gi RWO ibmc-block-gold 16h +persistentvolumeclaim/ibm-zen-objectstore-backup-pvc Bound pvc-d5e61dcf-65a3-4930-9cbf-ab80d04dda00 20Gi RWO ibmc-block-gold 16h +persistentvolumeclaim/zen-metastore-edb-1 Bound pvc-19d44f17-05ab-4dc0-bb5d-1b5f15ffd201 20Gi RWO ibmc-block-gold 17h +persistentvolumeclaim/zen-metastore-edb-2 Bound pvc-741ea444-b6f0-44ff-a123-bb4615d97381 20Gi RWO ibmc-block-gold 17h +``` + +!!! tip + You can retrieve the Initial Cloud Pak for Data password from the **admin-user-details** secret: `oc -n ibm-cpd get secret admin-user-details -o jsonpath="{.data.initial_admin_password}" | base64 -d` + ### 4.8.x version: Cloud Pak for Data 4.8.x leverages Cloud Pak Foundational Services v4, which runs its deployments in isolated/dedicated scope model, that means that its dependencies will be grouped and installed within the Cloud Pak for Data related projects/namespaces. Differently from CPD 4.6.x, there are only two namespaces that will be used: CPD instance namespace (e.g `ibm-cpd`) and CPD operators namespace (e.g `ibm-cpd-operators`). @@ -174,7 +237,6 @@ statefulset.apps/zen-metastoredb 3/3 68m - Role Variables -------------- ### cpd_product_version @@ -199,14 +261,17 @@ An IBM entitlement key specific for Cloud Pak for Data installation, primarily u - Default: None ### cpd_primary_storage_class -Primary storage class for Cloud Pak for Data. +Primary storage class for Cloud Pak for Data. For more details please read the [Storage Considerations for IBM Cloud Pak for Data](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.6.x?topic=planning-storage-considerations). +According to the mentioned documentation, Cloud Pak for Data uses the following access modes for storage classes: + - RWX file storage: ocs-storagecluster-cephfs + - RWX file storage: ibmc-file-gold-gid - **Required** if one of the known supported storage classes is not installed in the cluster. - Environment Variable: `CPD_PRIMARY_STORAGE_CLASS` - Default Value: `ibmc-file-gold-gid`, `ocs-storagecluster-cephfs`, `azurefiles-premium` (if available) ### cpd_metadata_storage_class -Storage class for the Cloud Pak for Data Zen meta database. +Storage class for the Cloud Pak for Data Zen meta database. This must support ReadWriteOnce (RWO access) access mode. - **Required** if one of the known supported storage classes is not installed in the cluster. - Environment Variable: `CPD_METADATA_STORAGE_CLASS` @@ -239,16 +304,16 @@ The CP4D Admin username to authenticate with CP4D APIs. If you didn't change the - Optional - Environment Variable: `CPD_ADMIN_USERNAME` -- Default Value: +- Default Value: - `admin` (CPD 4.6) - - `cpadmin` (CPD 4.8) + - `cpadmin` (CPD 4.8) ### cpd_admin_password The CP4D Admin User password to call CP4D API to provision Discovery Instance. If you didn't change the initial admin password after CP4D install, you don't need to provide it. The initial admin user password for `admin` or `cpdamin` will be used. - Optional - Environment Variable: `CPD_ADMIN_PASSWORD` -- Default Value: +- Default Value: - CPD 4.6: Looked up from the `admin-user-details` secret in the `cpd_instance_namespace` namespace - CPD 4.8: Looked up from the `ibm-iam-bindinfo-platform-auth-idp-credentials` secret in the `cpd_instance_namespace` namespace diff --git a/ibm/mas_devops/roles/cp4d/tasks/determine-ibmcatalog-tag.yml b/ibm/mas_devops/roles/cp4d/tasks/determine-ibmcatalog-tag.yml index 10a7851ffa..c1cfec8bb2 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/determine-ibmcatalog-tag.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/determine-ibmcatalog-tag.yml @@ -2,21 +2,20 @@ # The dependency version is based on the chosen ibm-operator-catalog. However the # ibm-operator-catalog does not explicitly include information about the cp4d version. # -# The configuration files in common_vars/casebundles do specify the dependency -# version for each of the MAS published ibm-operator-catalog catalogs. -# -# To determine which common_vars/casebundles configuration file to use, this role +# To determine which catalog metadata to use, this role # extracts the catalog_tag from the ibm-operator-catalog's displayName. # # For example: # displayName: IBM Maximo Operators (v9-240625-amd64) # # catalog_tag is v9-240625-amd64 -# dependency version will be determined by loading common_vars/casebundles/v9-240625-amd64.yml +# dependency version will be determined from the catalog metadata (get_catalog_info.py) + # 1. Get the IBM Catalog if available # ----------------------------------------------------------------------------- - name: "Lookup ibm-operator-catalog" + when: cpd_product_version is not defined or cpd_product_version == "" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 name: ibm-operator-catalog @@ -25,40 +24,42 @@ register: catalog_lookup - name: "Determine catalog version from catalog displayName" - block: - # extract the catalog tag from displayName - - set_fact: - catalog_tag: "{{ catalog_lookup.resources[0].spec.displayName.split('(')[1].split(')')[0].split(' ')[0] }}" - when: - - catalog_lookup is defined - - catalog_lookup.resources is defined - - catalog_lookup.resources | length > 0 - - catalog_lookup.resources[0].spec is defined - - catalog_lookup.resources[0].spec.displayName is defined - - '"(" in catalog_lookup.resources[0].spec.displayName' + when: + - catalog_lookup is defined + - catalog_lookup.resources is defined + - catalog_lookup.resources | length > 0 + - catalog_lookup.resources[0].spec is defined + - catalog_lookup.resources[0].spec.displayName is defined + - '"(" in catalog_lookup.resources[0].spec.displayName' + set_fact: + catalog_tag: "{{ catalog_lookup.resources[0].spec.displayName.split('(')[1].split(')')[0].split(' ')[0] }}" -- name: "Catalog Version" - debug: - msg: - - "Catalog Version ............................ {{ (catalog_tag is defined and catalog_tag != '') | ternary(catalog_tag, 'Undefined') }}" -# 2. Load default settings +# 2. Load catalog metadata # ----------------------------------------------------------------------------- -- name: Check if file exists in casebundles - stat: - path: "{{ role_path }}/../../common_vars/casebundles/{{ catalog_tag }}.yml" - register: file_exists_result +- name: "Load catalog metadata to determine default CP4D version" when: - catalog_tag is defined and catalog_tag != "" - - catalog_tag.split('-') | length == 3 # only pull casebundle if it's a curated catalog, skip if it's v8-amd64 + - cpd_product_version is not defined or cpd_product_version == "" + block: + - name: "Lookup catalog metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: catalog_metadata -- debug: - var: file_exists_result + - name: "Debug Catalog Metadata Lookup" + when: not catalog_metadata.failed + debug: + msg: + - "Catalog Version ................................ {{ catalog_tag }}" + - "Default CP4D Version ........................... {{ catalog_metadata.cpd_product_version_default }}" -- name: Load CASE bundle versions - include_vars: - file: "{{ role_path }}/../../common_vars/casebundles/{{ catalog_tag }}.yml" - when: - - file_exists_result is defined - - file_exists_result.stat is defined - - file_exists_result.stat.exists + - name: "Set CP4D Product Version (with default)" + when: not catalog_metadata.failed + set_fact: + cpd_product_version: "{{ catalog_metadata.cpd_product_version_default }}" + +- name: "Debug final CP4D version" + debug: + msg: + - "CP4D Version ................................... {{ cpd_product_version }}" diff --git a/ibm/mas_devops/roles/cp4d/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/cp4d/tasks/determine-storage-classes.yml index 4ff4a73ff4..deac82d3ef 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/determine-storage-classes.yml @@ -1,53 +1,38 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" -- name: "determine-storage-classes : Lookup storage classes" - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" - -# 2. Set Primary Storage (Required) +# 2. Set default storage classes # ----------------------------------------------------------------------------- -- name: "determine-storage-classes : Default Primary Storage if not set by user" - when: cpd_primary_storage_class is not defined or cpd_primary_storage_class == "" +- name: "determine-storage-classes : Set default File Storage if not set by user" + when: + - defaultStorageClasses.success + - cpd_primary_storage_class is not defined or cpd_primary_storage_class == "" + set_fact: + cpd_primary_storage_class: "{{ defaultStorageClasses.rwx }}" + +- name: "determine-storage-classes : Set default Block Storage if not set by user" + when: + - defaultStorageClasses.success + - cpd_metadata_storage_class is not defined or cpd_metadata_storage_class == "" set_fact: - cpd_primary_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" + cpd_metadata_storage_class: "{{ defaultStorageClasses.rwo }}" -- name: "determine-storage-classes : Assert that primary storage class has been defined" +- name: "determine-storage-classes : Assert that primary storage class has been defined (RWX)" assert: that: cpd_primary_storage_class is defined and cpd_primary_storage_class != "" fail_msg: "cpd_primary_storage_class must be defined" - -# 2. Set Metadata Storage (Required) -# ----------------------------------------------------------------------------- -- name: "determine-storage-classes : Default Metadata Storage if not set by user" - when: cpd_metadata_storage_class is not defined or cpd_metadata_storage_class == "" - set_fact: - cpd_metadata_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" - -- name: "determine-storage-classes : Assert that metadata storage class has been defined" +- name: "determine-storage-classes : Assert that metadata storage class has been defined (RWO)" assert: that: cpd_metadata_storage_class is defined and cpd_metadata_storage_class != "" fail_msg: "cpd_metadata_storage_class must be defined" - -# 3. Debug storage class configuration -# ----------------------------------------------------------------------------- - name: "determine-storage-classes : Debug CP4D storage class configuration" debug: msg: - - "Storage class (primary) .......... {{ cpd_primary_storage_class }}" - - "Storage class (metadata) ......... {{ cpd_metadata_storage_class }}" + - "Primary Storage class (RWX) ............ {{ cpd_primary_storage_class }}" + - "Metadata Storage class (RWO) ........... {{ cpd_metadata_storage_class }}" diff --git a/ibm/mas_devops/roles/cp4d/tasks/install-cp4d.yml b/ibm/mas_devops/roles/cp4d/tasks/install-cp4d.yml index 144b262a67..44e4fa78e7 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/install-cp4d.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/install-cp4d.yml @@ -160,15 +160,6 @@ retries: 20 # Approximately 20 minutes before we give up delay: 60 # 1 minute -# 3-pre. Wait for Zen Metastore Cluster to be ready -# ----------------------------------------------------------------------------- -# There have been issues with Zen Metastore Cluster not starting due to Persistent Storage, -# This task restarts any failing pods -- name: "install-cp4d : Wait for zen-metadata to be ready again (60s delay)" - include_tasks: "tasks/wait/wait-zenmetastore-edb.yml" - when: - - cpd_48_or_higher - # 3. Wait for zenStatus # ----------------------------------------------------------------------------- # oc get ZenService lite-cr -o jsonpath="{.status.zenStatus}{'\n'}" diff --git a/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml b/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml deleted file mode 100644 index 22432104f4..0000000000 --- a/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml +++ /dev/null @@ -1,112 +0,0 @@ ---- -# 1. Wait for zen metastore cluster to start -# ----------------------------------------------------------------------------- -- name: "wait-zenmetastore-edb : Wait for Zen Metastore EDB Cluster to be created" - k8s_info: - kind: Cluster - namespace: "{{ cpd_instance_namespace }}" - name: "zen-metastore-edb" - register: zenmetastoreCluster - retries: 120 # Give 60 minutes for the zenService to start Zen Metastore Pods (Logs show this taking ~20 minutes in a good run) - delay: 30 - until: zenmetastoreCluster.resources[0].status is defined - -# 2. For V4.8, We need to patch the postgres licensing job acccording to https://www.ibm.com/support/pages/node/7158524 -- name: "wait-zenmetastore-edb : Check and display the create-postgres-license-config license expiry date" - when: cpd_48 - block: - - name: "wait-zenmetastore-edb : Fetch the license expiry date" - kubernetes.core.k8s_info: - kind: Cluster - namespace: "{{ cpd_instance_namespace }}" - name: "zen-metastore-edb" - register: zenmetastoreCluster - until: - zenmetastoreCluster.resources[0].spec.instances is defined - and zenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus is defined - retries: 40 # Give 20 minutes - delay: 30 - - debug: - msg: - - "License Expiration .. {{ zenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration | default ('') }}" - - "License Status ...... {{ zenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus}}" - -- name: "wait-zenmetastore-edb : Update create-postgres-license-config job" - when: - - cpd_48 - - zenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration is not defined - block: - - name: "wait-zenmetastore-edb : Wait for create-postgres-license-config Job to be Completed or Failed (10s delay)" - kubernetes.core.k8s_info: - api_version: batch/v1 - kind: Job - name: "create-postgres-license-config" - namespace: "{{ cpd_operators_namespace }}" - register: _job_info - until: - - _job_info.resources is defined - - _job_info.resources | length > 0 - - (_job_info.resources | json_query('[*].status.conditions[?type==`Complete`][].status') | select ('match','True') | list | length == 1) or - (_job_info.resources | json_query('[*].status.conditions[?type==`Failed`][].status') | select ('match','True') | list | length == 1) - retries: 30 - delay: 10 - - - name: "wait-zenmetastore-edb : Recreate the job with up to date license image" - shell: >- - oc get job create-postgres-license-config -n {{ cpd_operators_namespace }} -o yaml | \ - sed -e 's/operator.ibm.com\/opreq-control: "true"/operator.ibm.com\/opreq-control: "false"/' \ - -e 's|\(image: \).*|\1"cp.icr.io/cp/cpd/edb-postgres-license-provider@sha256:c1670e7dd93c1e65a6659ece644e44aa5c2150809ac1089e2fd6be37dceae4ce"|' \ - -e '/controller-uid:/d' | \ - oc replace --force -f - - register: _job_recreate_output - - - debug: - msg: "Recreate the job: {{ _job_recreate_output }}" - - - name: "wait-zenmetastore-edb : Wait for create-postgres-license-config Job to be Completed or Failed (10s delay)" - kubernetes.core.k8s_info: - api_version: batch/v1 - kind: Job - name: "create-postgres-license-config" - namespace: "{{ cpd_operators_namespace }}" - register: _job_info - until: - - _job_info.resources is defined - - _job_info.resources | length > 0 - - (_job_info.resources | json_query('[*].status.conditions[?type==`Complete`][].status') | select ('match','True') | list | length == 1) or - (_job_info.resources | json_query('[*].status.conditions[?type==`Failed`][].status') | select ('match','True') | list | length == 1) - retries: 30 - delay: 10 - - - name: "wait-zenmetastore-edb : Check and display the license expiry date" - kubernetes.core.k8s_info: - kind: Cluster - namespace: "{{ cpd_instance_namespace }}" - name: "zen-metastore-edb" - register: newzenmetastoreCluster - until: - newzenmetastoreCluster.resources[0].spec.instances is defined - and newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus is defined - and "Valid license" in newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus - retries: 40 # Give 20 minutes - delay: 30 - - - debug: - msg: - - "License Expiration .. {{ newzenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration | default ('') }}" - - "License Status ...... {{ newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus | default ('')}}" - -# 3. Wait for zen metastore replica pods to become ready -# ----------------------------------------------------------------------------- -- name: "wait-zenmetastore-edb : Wait for ZenMetastore pods to be become ready" - k8s_info: - kind: Cluster - namespace: "{{ cpd_instance_namespace }}" - name: "zen-metastore-edb" - register: zenmetastoreCluster - retries: 40 # Give 20 minutes for the pods to become ready - delay: 30 - until: >- - zenmetastoreCluster.resources[0].spec.instances is defined - and zenmetastoreCluster.resources[0].status.readyInstances is defined - and zenmetastoreCluster.resources[0].spec.instances == zenmetastoreCluster.resources[0].status.readyInstances diff --git a/ibm/mas_devops/roles/cp4d_service/README.md b/ibm/mas_devops/roles/cp4d_service/README.md index bf245cf732..9c54c4f5ba 100644 --- a/ibm/mas_devops/roles/cp4d_service/README.md +++ b/ibm/mas_devops/roles/cp4d_service/README.md @@ -10,6 +10,7 @@ Currently supported Cloud Pak for Data release versions supported are: - 4.6.4 - 4.6.6 - 4.8.0 + - 5.0.0 The role will automatically install the corresponding CPD service operator channel and custom resource version associated to the chosen Cloud Pak for Data release version. @@ -24,12 +25,18 @@ These services can be deployed and configured using this role: - [Analytics Services (Apache Spark)](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=services-analytics) required by [Predict](https://www.ibm.com/docs/en/mas-cd/mhmpmh-and-p-u/continuous-delivery) - [Watson OpenScale](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=services-watson-openscale) an optional dependency for [Predict](https://www.ibm.com/docs/en/mas-cd/mhmpmh-and-p-u/continuous-delivery) - [SPSS Modeler](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=modeler-installing) optional dependency for [Predict](https://www.ibm.com/docs/en/mas-cd/mhmpmh-and-p-u/continuous-delivery) -- [Watson Discovery](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=services-watson-discovery) required by [Assist](https://www.ibm.com/docs/en/mas-cd/maximo-assist) +- [Watson Discovery](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=services-watson-discovery) required by [Assist](https://www.ibm.com/docs/en/mas-cd/maximo-assist) - Not supported with CPD 5.0 - [Cognos Analytics](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=analytics-installing) optional dependency for [Manage application](https://www.ibm.com/docs/en/mas-cd/maximo-manage) +!!! info + - Install CP4D Services (~30 Minutes - 1 hour for each service) + + All timings are estimates. + + Upgrade ------------------ -This role also supports seamlessly CPD services minor version upgrades (CPD 4.6.x -> CPD 4.8.x), as well as patch version upgrades (CPD 4.8.0 -> CPD 4.8.1), with the exception of [Watson Discovery](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=u-upgrading-from-version-46-2). +This role also supports seamlessly CPD services minor version upgrades (CPD 4.6.x > CPD 4.8.0 or CPD 4.8.0 > CPD 5.0.0), as well as patch version upgrades (e.g. CPD 4.6.0 -> CPD 4.6.6), with the exception of [Watson Discovery](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=u-upgrading-from-version-46-2). All you need to do is to define `cpd_product_version` variable to the version you target to upgrade and run this role for a particular CPD service. It's important that before you upgrade CPD services, the CPD Control Plane/Zen is also upgraded to the same release version. @@ -56,124 +63,78 @@ Subscriptions related to Watson Studio: Watson Studio is made up of many moving parts across multiple namespaces. -In the **ibm-common-services** namespace: - -- 15 workloads / 12 pods -- 0.126 CPU usage / 1.11 CPU requests / 3.57 CPU limit (11% utilization) -- 773.8 MiB memory usage, 2.27 GiB memory requests / 5.72 GiB memory limit (33% utilization) - -``` -oc -n ibm-common-services get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cert-manager-cainjector 1/1 1 1 4h57m -cert-manager-controller 1/1 1 1 4h57m -cert-manager-webhook 1/1 1 1 4h57m -configmap-watcher 1/1 1 1 4h57m -ibm-cert-manager-operator 1/1 1 1 4h58m -ibm-common-service-operator 1/1 1 1 5h3m -ibm-common-service-webhook 1/1 1 1 5h2m -ibm-namespace-scope-operator 1/1 1 1 5h3m -ibm-zen-operator 1/1 1 1 4h58m -meta-api-deploy 1/1 1 1 4h57m -operand-deployment-lifecycle-manager 1/1 1 1 5h2m -secretshare 1/1 1 1 5h2m -``` - In the **ibm-cpd-operators** namespace: -- 7 workloads / 7 pods -- 0.007 CPU usage / 0.7 CPU requests / 3.75 CPU limit (1% utilization) -- 263.4 MiB memory usage, 1.64 GiB memory requests /6.5 GiB memory limit (15% utilization) - ```bash oc -n ibm-cpd-operators get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cpd-platform-operator-manager 1/1 1 1 5h -ibm-common-service-operator 1/1 1 1 5h -ibm-cpd-ccs-operator 1/1 1 1 3h43m -ibm-cpd-datarefinery-operator 1/1 1 1 134m -ibm-cpd-ws-operator 1/1 1 1 3h45m -ibm-cpd-ws-runtimes-operator 1/1 1 1 118m -ibm-namespace-scope-operator 1/1 1 1 5h +NAME READY UP-TO-DATE AVAILABLE AGE +ibm-cpd-ccs-operator 1/1 1 1 83m +ibm-cpd-datarefinery-operator 1/1 1 1 83m +ibm-cpd-ws-operator 1/1 1 1 83m +ibm-cpd-ws-runtimes-operator 1/1 1 1 83m +ibm-elasticsearch-operator-ibm-es-controller-manager 1/1 1 1 83m ``` In the **ibm-cpd** namespace: -- 51 workloads / 101 pods -- 1.1 CPU usage / 18.72 CPU requests / 86.7 CPU limit (5% utilization) -- 16.46 GiB memory usage, 33.04 GiB memory requests / 175.7 GiB memory limit (50% utilization) - ```bash oc -n ibm-cpd get ccs,ws,datarefinery,notebookruntimes,deployments,sts NAME VERSION RECONCILED STATUS AGE -ccs.ccs.cpd.ibm.com/ccs-cr 4.0.9 4.0.9 Completed 3h42m +ccs.ccs.cpd.ibm.com/ccs-cr 9.0.0 9.0.0 Completed 82m NAME VERSION RECONCILED STATUS AGE -ws.ws.cpd.ibm.com/ws-cr 4.0.9 4.0.9 Completed 3h45m +ws.ws.cpd.ibm.com/ws-cr 9.0.0 9.0.0 Completed 83m -NAME VERSION STATUS AGE -datarefinery.datarefinery.cpd.ibm.com/datarefinery-sample 4.0.9 Completed 133m +NAME VERSION RECONCILED STATUS AGE +datarefinery.datarefinery.cpd.ibm.com/datarefinery-cr 9.0.0 9.0.0 Completed 36m -NAME VERSION RECONCILED STATUS AGE -notebookruntime.ws.cpd.ibm.com/ibm-cpd-ws-runtime-py39 4.0.9 Completed 116m +NAME NLP MODELS VERSION RECONCILED STATUS AGE +notebookruntime.ws.cpd.ibm.com/ibm-cpd-ws-runtime-241-py 9.0.0 9.0.0 Completed 22m NAME READY UP-TO-DATE AVAILABLE AGE -deployment.apps/asset-files-api 1/1 1 1 159m -deployment.apps/ax-cdsx-jupyter-notebooks-converter-deploy 1/1 1 1 111m -deployment.apps/ax-cdsx-notebooks-job-manager-deploy 1/1 1 1 111m -deployment.apps/ax-environments-api-deploy 1/1 1 1 144m -deployment.apps/ax-environments-ui-deploy 1/1 1 1 144m -deployment.apps/ax-wdp-notebooks-api-deploy 1/1 1 1 111m -deployment.apps/ax-ws-notebooks-ui-deploy 1/1 1 1 111m -deployment.apps/catalog-api 2/2 2 2 167m -deployment.apps/dap-dashboards-api 1/1 1 1 159m -deployment.apps/dataview-api-service 1/1 1 1 138m -deployment.apps/dc-main 1/1 1 1 167m -deployment.apps/event-logger-api 1/1 1 1 159m -deployment.apps/ibm-0100-model-viewer-prod 1/1 1 1 110m -deployment.apps/ibm-nginx 3/3 3 3 4h37m -deployment.apps/jobs-api 1/1 1 1 155m -deployment.apps/jobs-ui 1/1 1 1 155m -deployment.apps/ngp-projects-api 1/1 1 1 159m -deployment.apps/portal-catalog 1/1 1 1 166m -deployment.apps/portal-common-api 1/1 1 1 159m -deployment.apps/portal-dashboards 1/1 1 1 159m -deployment.apps/portal-job-manager 1/1 1 1 159m -deployment.apps/portal-main 1/1 1 1 159m -deployment.apps/portal-ml-dl 1/1 1 1 111m -deployment.apps/portal-notifications 1/1 1 1 159m -deployment.apps/portal-projects 1/1 1 1 159m -deployment.apps/redis-ha-haproxy 1/1 1 1 174m -deployment.apps/runtime-assemblies-operator 1/1 1 1 151m -deployment.apps/runtime-manager-api 1/1 1 1 147m -deployment.apps/spaces 1/1 1 1 142m -deployment.apps/spawner-api 1/1 1 1 146m -deployment.apps/usermgmt 3/3 3 3 4h39m -deployment.apps/wdp-connect-connection 1/1 1 1 167m -deployment.apps/wdp-connect-connector 1/1 1 1 167m -deployment.apps/wdp-connect-flight 1/1 1 1 167m -deployment.apps/wdp-dataprep 1/1 1 1 126m -deployment.apps/wdp-dataview 1/1 1 1 138m -deployment.apps/wdp-shaper 1/1 1 1 128m -deployment.apps/wkc-search 1/1 1 1 167m -deployment.apps/wml-main 1/1 1 1 143m -deployment.apps/zen-audit 1/1 1 1 4h33m -deployment.apps/zen-core 3/3 3 3 4h32m -deployment.apps/zen-core-api 3/3 3 3 4h32m -deployment.apps/zen-data-sorcerer 2/2 2 2 4h25m -deployment.apps/zen-watchdog 1/1 1 1 4h25m -deployment.apps/zen-watcher 1/1 1 1 4h32m - -NAME READY AGE -statefulset.apps/dsx-influxdb 1/1 4h28m -statefulset.apps/elasticsearch-master 3/3 171m -statefulset.apps/rabbitmq-ha 3/3 3h35m -statefulset.apps/redis-ha-server 3/3 3h1m -statefulset.apps/wdp-couchdb 3/3 3h38m -statefulset.apps/zen-metastoredb 3/3 4h43m +deployment.apps/asset-files-api 1/1 1 1 60m +deployment.apps/ax-cdsx-jupyter-notebooks-converter-deploy 1/1 1 1 15m +deployment.apps/ax-cdsx-notebooks-job-manager-deploy 1/1 1 1 15m +deployment.apps/ax-environments-api-deploy 1/1 1 1 53m +deployment.apps/ax-environments-ui-deploy 1/1 1 1 53m +deployment.apps/ax-wdp-notebooks-api-deploy 1/1 1 1 15m +deployment.apps/ax-ws-notebooks-ui-deploy 1/1 1 1 15m +deployment.apps/catalog-api 2/2 2 2 69m +deployment.apps/dataview-api-service 1/1 1 1 48m +deployment.apps/dc-main 1/1 1 1 65m +deployment.apps/event-logger-api 1/1 1 1 59m +deployment.apps/ibm-0100-model-viewer-prod 1/1 1 1 14m +deployment.apps/jobs-api 1/1 1 1 49m +deployment.apps/jobs-ui 1/1 1 1 49m +deployment.apps/ngp-projects-api 1/1 1 1 60m +deployment.apps/portal-catalog 1/1 1 1 65m +deployment.apps/portal-common-api 1/1 1 1 60m +deployment.apps/portal-job-manager 1/1 1 1 60m +deployment.apps/portal-main 1/1 1 1 60m +deployment.apps/portal-ml-dl 1/1 1 1 14m +deployment.apps/portal-notifications 1/1 1 1 60m +deployment.apps/portal-projects 1/1 1 1 60m +deployment.apps/redis-ha-haproxy 1/1 1 1 78m +deployment.apps/runtime-assemblies-operator 1/1 1 1 59m +deployment.apps/runtime-manager-api 1/1 1 1 59m +deployment.apps/spaces 1/1 1 1 48m +deployment.apps/task-credentials 1/1 1 1 48m +deployment.apps/wdp-connect-connection 1/1 1 1 66m +deployment.apps/wdp-connect-connector 1/1 1 1 66m +deployment.apps/wdp-connect-flight 1/1 1 1 66m +deployment.apps/wdp-dataprep 1/1 1 1 29m +deployment.apps/wdp-dataview 1/1 1 1 48m +deployment.apps/wdp-shaper 1/1 1 1 29m +deployment.apps/wkc-search 1/1 1 1 66m +deployment.apps/wml-main 1/1 1 1 48m + +NAME READY AGE +statefulset.apps/elasticsea-0ac3-ib-6fb9-es-server-esnodes 3/3 74m +statefulset.apps/rabbitmq-ha 3/3 79m +statefulset.apps/redis-ha-server 3/3 79m +statefulset.apps/wdp-couchdb 3/3 79m ``` - ### Watson Machine Learning Subscriptions related to Watson Machine Learning: @@ -181,117 +142,41 @@ Subscriptions related to Watson Machine Learning: - **ibm-cpd-wml** - **ibm-cpd-ccs** - Watson Machine Learning is made up of many moving parts across multiple namespaces. -In the **ibm-common-services** namespace: - -- 15 workloads / 12 pods -- 0.126 CPU usage / 1.11 CPU requests / 3.57 CPU limit (11% utilization) -- 773.8 MiB memory usage, 2.27 GiB memory requests / 5.72 GiB memory limit (33% utilization) - -```bash -oc -n ibm-common-services get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cert-manager-cainjector 1/1 1 1 4h7m -cert-manager-controller 1/1 1 1 4h7m -cert-manager-webhook 1/1 1 1 4h7m -configmap-watcher 1/1 1 1 4h7m -ibm-cert-manager-operator 1/1 1 1 4h8m -ibm-common-service-operator 1/1 1 1 4h14m -ibm-common-service-webhook 1/1 1 1 4h12m -ibm-namespace-scope-operator 1/1 1 1 4h13m -ibm-zen-operator 1/1 1 1 4h8m -meta-api-deploy 1/1 1 1 4h8m -operand-deployment-lifecycle-manager 1/1 1 1 4h12m -secretshare 1/1 1 1 4h12m -``` - In the **ibm-cpd-operators** namespace: -- 5 workloads / 5 pods -- 0.011 CPU usage / 0.5 CPU requests / 2.75 CPU limit (1% utilization) -- 177.4 MiB memory usage, 1.14 GiB memory requests / 4.5 GiB memory limit (15% utilization) - ```bash oc -n ibm-cpd-operators get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cpd-platform-operator-manager 1/1 1 1 3h57m -ibm-common-service-operator 1/1 1 1 3h57m -ibm-cpd-ccs-operator 1/1 1 1 150m -ibm-cpd-wml-operator 1/1 1 1 152m -ibm-namespace-scope-operator 1/1 1 1 3h57m +NAME READY UP-TO-DATE AVAILABLE AGE +ibm-cpd-ccs-operator 1/1 1 1 134m +ibm-cpd-datarefinery-operator 1/1 1 1 134m +ibm-cpd-wml-operator 1/1 1 1 49m +ibm-elasticsearch-operator-ibm-es-controller-manager 1/1 1 1 134m ``` In the **ibm-cpd** namespace: -- 50 workloads / 103 pods -- 1.28 CPU usage / 18.07 CPU requests / 86.15 CPU limit (7% utilization) -- 16.96 GiB memory usage, 41.29 GiB memory requests / 184.5 GiB memory limit (41% utilization) - -``` +```bash oc -n ibm-cpd get ccs,wmlbase,deployments,sts NAME VERSION RECONCILED STATUS AGE -ccs.ccs.cpd.ibm.com/ccs-cr 4.0.9 4.0.9 Completed 149m - -NAME VERSION BUILD STATUS AGE -wmlbase.wml.cpd.ibm.com/wml-cr 4.0.9 4.0.10-3220 Completed 151m - -NAME READY UP-TO-DATE AVAILABLE AGE -deployment.apps/asset-files-api 1/1 1 1 80m -deployment.apps/ax-environments-api-deploy 1/1 1 1 66m -deployment.apps/ax-environments-ui-deploy 1/1 1 1 66m -deployment.apps/catalog-api 2/2 2 2 89m -deployment.apps/dap-dashboards-api 1/1 1 1 80m -deployment.apps/dataview-api-service 1/1 1 1 62m -deployment.apps/dc-main 1/1 1 1 88m -deployment.apps/event-logger-api 1/1 1 1 80m -deployment.apps/ibm-nginx 3/3 3 3 3h34m -deployment.apps/jobs-api 1/1 1 1 76m -deployment.apps/jobs-ui 1/1 1 1 76m -deployment.apps/ngp-projects-api 1/1 1 1 80m -deployment.apps/portal-catalog 1/1 1 1 88m -deployment.apps/portal-common-api 1/1 1 1 80m -deployment.apps/portal-dashboards 1/1 1 1 80m -deployment.apps/portal-job-manager 1/1 1 1 80m -deployment.apps/portal-main 1/1 1 1 80m -deployment.apps/portal-notifications 1/1 1 1 80m -deployment.apps/portal-projects 1/1 1 1 80m -deployment.apps/redis-ha-haproxy 1/1 1 1 97m -deployment.apps/runtime-assemblies-operator 1/1 1 1 72m -deployment.apps/runtime-manager-api 1/1 1 1 70m -deployment.apps/spaces 1/1 1 1 64m -deployment.apps/spawner-api 1/1 1 1 69m -deployment.apps/usermgmt 3/3 3 3 3h36m -deployment.apps/wdp-connect-connection 1/1 1 1 88m -deployment.apps/wdp-connect-connector 1/1 1 1 88m -deployment.apps/wdp-connect-flight 1/1 1 1 88m -deployment.apps/wdp-dataview 1/1 1 1 62m -deployment.apps/wkc-search 1/1 1 1 88m -deployment.apps/wml-deployment-envoy 1/1 1 1 37m -deployment.apps/wml-main 1/1 1 1 65m -deployment.apps/wml-repositoryv4 1/1 1 1 18m -deployment.apps/wmltraining 1/1 1 1 13m -deployment.apps/wmltrainingorchestrator 1/1 1 1 8m11s -deployment.apps/zen-audit 1/1 1 1 3h28m -deployment.apps/zen-core 3/3 3 3 3h27m -deployment.apps/zen-core-api 3/3 3 3 3h27m -deployment.apps/zen-data-sorcerer 2/2 2 2 3h20m -deployment.apps/zen-watchdog 1/1 1 1 3h20m -deployment.apps/zen-watcher 1/1 1 1 3h27m - -NAME READY AGE -statefulset.apps/dsx-influxdb 1/1 3h23m -statefulset.apps/elasticsearch-master 3/3 94m -statefulset.apps/rabbitmq-ha 3/3 142m -statefulset.apps/redis-ha-server 3/3 106m -statefulset.apps/wdp-couchdb 3/3 145m -statefulset.apps/wml-deployment-agent 1/1 32m -statefulset.apps/wml-deployment-manager 1/1 24m -statefulset.apps/wml-deployments-etcd 3/3 43m -statefulset.apps/zen-metastoredb 3/3 3h40m -``` +ccs.ccs.cpd.ibm.com/ccs-cr 9.0.0 9.0.0 Completed 133m +NAME VERSION BUILD STATUS RECONCILED AGE +wmlbase.wml.cpd.ibm.com/wml-cr 5.0.0 5.0.0-918 Completed 5.0.0 50m + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/wml-deployment-envoy 1/1 1 1 23m +deployment.apps/wml-deployment-manager 1/1 1 1 19m +deployment.apps/wml-main 1/1 1 1 99m +deployment.apps/wml-repositoryv4 1/1 1 1 16m +deployment.apps/wmltraining 1/1 1 1 15m +deployment.apps/wmltrainingorchestrator 1/1 1 1 14m + +NAME READY AGE +statefulset.apps/wml-cpd-etcd 3/3 26m +statefulset.apps/wml-deployment-agent 1/1 21m +``` ### Analytics Engine Subscriptions related to Analytics Engine: @@ -301,74 +186,30 @@ Subscriptions related to Analytics Engine: Analytics Engine is made up of many moving parts across multiple namespaces. -In the **ibm-common-services** namespace: - -- 15 workloads / 12 pods -- 0.051 CPU usage / 1.11 CPU requests / 3.57 CPU limit (5% utilization) -- 774.7 MiB memory usage, 2.27 GiB memory requests / 5.72 GiB memory limit (33% utilization) - -``` -oc -n ibm-common-services get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cert-manager-cainjector 1/1 1 1 126m -cert-manager-controller 1/1 1 1 126m -cert-manager-webhook 1/1 1 1 126m -configmap-watcher 1/1 1 1 126m -ibm-cert-manager-operator 1/1 1 1 126m -ibm-common-service-operator 1/1 1 1 131m -ibm-common-service-webhook 1/1 1 1 130m -ibm-namespace-scope-operator 1/1 1 1 131m -ibm-zen-operator 1/1 1 1 126m -meta-api-deploy 1/1 1 1 126m -operand-deployment-lifecycle-manager 1/1 1 1 130m -secretshare 1/1 1 1 130m -``` - In the **ibm-cpd-operators** namespace: -- 4 workloads / 4 pods -- 0.003 CPU usage / 0.4 CPU requests / 2 CPU limit (1% utilization) -- 131.5 MiB memory usage, 912 MiB memory requests / 3 GiB memory limit (15% utilization) - -``` +```bash oc -n ibm-cpd-operators get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cpd-platform-operator-manager 1/1 1 1 128m -ibm-common-service-operator 1/1 1 1 128m -ibm-cpd-ae-operator 1/1 1 1 65m -ibm-namespace-scope-operator 1/1 1 1 128m +NAME READY UP-TO-DATE AVAILABLE AGE +ibm-cpd-ae-operator 1/1 1 1 31m ``` In the **ibm-cpd** namespace: -- 16 workloads / 60 pods -- 0.449 CPU usage / 8.06 CPU requests / 21.15 CPU limit (5% utilization) -- 3.15 GiB memory usage, 14.31 GiB memory requests / 32.71 GiB memory limit (22% utilization) - -``` -oc -n ibm-cpd get analyticsengine,deployments,sts +```bash +oc -n ibm-cpd get analyticsengine,deployments NAME VERSION RECONCILED STATUS AGE -analyticsengine.ae.cpd.ibm.com/analyticsengine-sample 4.0.9 4.0.9 Completed 66m - -NAME READY UP-TO-DATE AVAILABLE AGE -deployment.apps/ibm-nginx 3/3 3 3 93m -deployment.apps/spark-hb-control-plane 1/1 1 1 62m -deployment.apps/spark-hb-create-trust-store 1/1 1 1 64m -deployment.apps/spark-hb-deployer-agent 1/1 1 1 62m -deployment.apps/spark-hb-helm-repo 1/1 1 1 62m -deployment.apps/spark-hb-nginx 1/1 1 1 62m -deployment.apps/spark-hb-register-hb-dataplane 1/1 1 1 55m -deployment.apps/usermgmt 3/3 3 3 94m -deployment.apps/zen-audit 1/1 1 1 89m -deployment.apps/zen-core 3/3 3 3 89m -deployment.apps/zen-core-api 3/3 3 3 89m -deployment.apps/zen-data-sorcerer 2/2 2 2 83m -deployment.apps/zen-watchdog 1/1 1 1 83m -deployment.apps/zen-watcher 1/1 1 1 89m - -NAME READY AGE -statefulset.apps/dsx-influxdb 1/1 85m -statefulset.apps/zen-metastoredb 3/3 118m +analyticsengine.ae.cpd.ibm.com/analyticsengine-sample 5.0.0 5.0.0 Completed 31m + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/spark-hb-br-recovery 1/1 1 1 11m +deployment.apps/spark-hb-control-plane 1/1 1 1 19m +deployment.apps/spark-hb-create-trust-store 1/1 1 1 25m +deployment.apps/spark-hb-deployer-agent 1/1 1 1 19m +deployment.apps/spark-hb-nginx 1/1 1 1 19m +deployment.apps/spark-hb-register-hb-dataplane 1/1 1 1 10m +deployment.apps/spark-hb-ui 1/1 1 1 19m + ``` @@ -376,231 +217,118 @@ statefulset.apps/zen-metastoredb 3/3 118m Subscriptions related to Watson OpenScale (in the **ibm-cpd-operators** namespace): - **cpd-platform-operator** -- **ibm-cpd-wos** +- **ibm-cpd-wos-operator** Analytics Engine is made up of many moving parts across multiple namespaces. -In the **ibm-common-services** namespace: - -- 15 workloads / 12 pods -- 0.051 CPU usage / 1.11 CPU requests / 3.57 CPU limit (5% utilization) -- 774.7 MiB memory usage, 2.27 GiB memory requests / 5.72 GiB memory limit (33% utilization) - -``` -oc -n ibm-common-services get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cert-manager-cainjector 1/1 1 1 126m -cert-manager-controller 1/1 1 1 126m -cert-manager-webhook 1/1 1 1 126m -configmap-watcher 1/1 1 1 126m -ibm-cert-manager-operator 1/1 1 1 126m -ibm-common-service-operator 1/1 1 1 131m -ibm-common-service-webhook 1/1 1 1 130m -ibm-namespace-scope-operator 1/1 1 1 131m -ibm-zen-operator 1/1 1 1 126m -meta-api-deploy 1/1 1 1 126m -operand-deployment-lifecycle-manager 1/1 1 1 130m -secretshare 1/1 1 1 130m -``` - In the **ibm-cpd-operators** namespace: -- 4 workloads / 4 pods -- 0.005 CPU usage / 0.4 CPU requests / 2 CPU limit (1% utilization) -- 148.1 MiB memory usage, 912 MiB memory requests / 3 GiB memory limit (16% utilization) - -``` +```bash oc -n ibm-cpd-operators get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cpd-platform-operator-manager 1/1 1 1 145m -ibm-common-service-operator 1/1 1 1 145m -ibm-cpd-wos-operator 1/1 1 1 76m -ibm-namespace-scope-operator 1/1 1 1 145m +NAME READY UP-TO-DATE AVAILABLE AGE +ibm-cpd-wos-operator 1/1 1 1 30m ``` In the **ibm-cpd** namespace: -- 32 workloads / 63 pods -- 0.591 CPU usage / 16.76 CPU requests / 31.9 CPU limit (4% utilization) -- 7.67 GiB memory usage, 37.32 GiB memory requests / 97.89 GiB memory limit (20% utilization) -``` +```bash oc -n ibm-cpd get woservice,deployments,sts -NAME TYPE STORAGE SCALECONFIG PHASE RECONCILED STATUS -woservice.wos.cpd.ibm.com/aiopenscale service ibmc-file-gold-gid small Ready 4.0.9 Completed - -NAME READY UP-TO-DATE AVAILABLE AGE -deployment.apps/aiopenscale-ibm-aios-bias 1/1 1 1 61m -deployment.apps/aiopenscale-ibm-aios-bkpicombined 1/1 1 1 61m -deployment.apps/aiopenscale-ibm-aios-common-api 1/1 1 1 61m -deployment.apps/aiopenscale-ibm-aios-configuration 1/1 1 1 61m -deployment.apps/aiopenscale-ibm-aios-dashboard 1/1 1 1 60m -deployment.apps/aiopenscale-ibm-aios-datamart 1/1 1 1 60m -deployment.apps/aiopenscale-ibm-aios-drift 1/1 1 1 60m -deployment.apps/aiopenscale-ibm-aios-explainability 1/1 1 1 60m -deployment.apps/aiopenscale-ibm-aios-fast-path 1/1 1 1 60m -deployment.apps/aiopenscale-ibm-aios-feedback 1/1 1 1 59m -deployment.apps/aiopenscale-ibm-aios-ml-gateway-discovery 1/1 1 1 59m -deployment.apps/aiopenscale-ibm-aios-ml-gateway-service 1/1 1 1 59m -deployment.apps/aiopenscale-ibm-aios-mrm 1/1 1 1 59m -deployment.apps/aiopenscale-ibm-aios-nginx 1/1 1 1 58m -deployment.apps/aiopenscale-ibm-aios-notification 1/1 1 1 59m -deployment.apps/aiopenscale-ibm-aios-payload-logging 1/1 1 1 58m -deployment.apps/aiopenscale-ibm-aios-payload-logging-api 1/1 1 1 59m -deployment.apps/aiopenscale-ibm-aios-scheduling 1/1 1 1 58m -deployment.apps/ibm-nginx 3/3 3 3 125m -deployment.apps/usermgmt 3/3 3 3 127m -deployment.apps/zen-audit 1/1 1 1 120m -deployment.apps/zen-core 3/3 3 3 120m -deployment.apps/zen-core-api 3/3 3 3 120m -deployment.apps/zen-data-sorcerer 2/2 2 2 114m -deployment.apps/zen-watchdog 1/1 1 1 114m -deployment.apps/zen-watcher 1/1 1 1 120m - -NAME READY AGE -statefulset.apps/aiopenscale-ibm-aios-etcd 3/3 62m -statefulset.apps/aiopenscale-ibm-aios-kafka 3/3 62m -statefulset.apps/aiopenscale-ibm-aios-redis 3/3 62m -statefulset.apps/aiopenscale-ibm-aios-zookeeper 3/3 70m -statefulset.apps/dsx-influxdb 1/1 116m -statefulset.apps/zen-metastoredb 3/3 130m -``` +NAME TYPE STORAGE SCALECONFIG PHASE RECONCILED STATUS +woservice.wos.cpd.ibm.com/openscale-defaultinstance service small Ready 5.0.0 Completed + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/openscale-defaultinstance-ibm-aios-bias 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-bkpicombined 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-common-api 1/1 1 1 22m +deployment.apps/openscale-defaultinstance-ibm-aios-configuration 1/1 1 1 22m +deployment.apps/openscale-defaultinstance-ibm-aios-dashboard 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-datamart 1/1 1 1 22m +deployment.apps/openscale-defaultinstance-ibm-aios-drift 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-explainability 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-fast-path 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-feedback 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-metrics-compute-manager 0/0 0 0 21m +deployment.apps/openscale-defaultinstance-ibm-aios-ml-gateway-discovery 1/1 1 1 22m +deployment.apps/openscale-defaultinstance-ibm-aios-ml-gateway-service 1/1 1 1 22m +deployment.apps/openscale-defaultinstance-ibm-aios-mrm 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-nginx 1/1 1 1 25m +deployment.apps/openscale-defaultinstance-ibm-aios-notification 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-payload-logging 1/1 1 1 21m +deployment.apps/openscale-defaultinstance-ibm-aios-payload-logging-api 1/1 1 1 22m +deployment.apps/openscale-defaultinstance-ibm-aios-redis 1/1 1 1 28m +deployment.apps/openscale-defaultinstance-ibm-aios-scheduling 1/1 1 1 21m + +NAME READY AGE +statefulset.apps/openscale-defaultinstance-ibm-aios-etcd 3/3 27m +statefulset.apps/openscale-defaultinstance-ibm-aios-kafka 3/3 27m +``` -### Watson Discovery -Subscriptions related to Watson Discovery (in the **ibm-cpd-operators** namespace): +### Cognos Analytics +Subscriptions related to Cognos Analytics (in the **ibm-cpd-operators** namespace): - **cpd-platform-operator** -- **ibm-watson-discovery-operator** -- **ibm-elasticsearch-operator** -- **ibm-etcd-operator** -- **ibm-minio-operator** -- **ibm-model-train-classic-operator** -- **ibm-rabbitmq-operator** -- **ibm-watson-gateway-operator** +- **ibm-ca-operator-controller-manager** -Subscriptions related to Watson Discovery (in the **ibm-common-services** namespace): +Cognos Analytics is made up of many moving parts across multiple namespaces. -- **cloud-native-postgresql** +In the **ibm-cpd-operators** namespace: -Watson Discovery is made up of many moving parts across multiple namespaces. +```bash +oc -n ibm-cpd-operators get deployments +NAME READY UP-TO-DATE AVAILABLE AGE +ibm-ca-operator-controller-manager 1/1 1 1 19m +``` -In the **ibm-common-services** namespace: +In the **ibm-cpd** namespace: -- 13 workloads / 16 pods -- 0.126 CPU usage / 1.11 CPU requests / 3.57 CPU limit (8% utilization) -- 921.9 MiB memory usage, 2.27 GiB memory requests / 5.72 GiB memory limit (40% utilization) -``` -oc -n ibm-common-services get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -cert-manager-cainjector 1/1 1 1 3h9m -cert-manager-controller 1/1 1 1 3h9m -cert-manager-webhook 1/1 1 1 3h9m -configmap-watcher 1/1 1 1 3h9m -ibm-cert-manager-operator 1/1 1 1 3h10m -ibm-common-service-operator 1/1 1 1 3h16m -ibm-common-service-webhook 1/1 1 1 3h14m -ibm-namespace-scope-operator 1/1 1 1 3h15m -ibm-zen-operator 1/1 1 1 3h10m -meta-api-deploy 1/1 1 1 3h9m -operand-deployment-lifecycle-manager 1/1 1 1 3h14m -postgresql-operator-controller-manager-1-15-0 1/1 1 1 134m -secretshare 1/1 1 1 3h14m +```bash +oc -n ibm-cpd get caservice,deployments +NAME AGE +caservice.ca.cpd.ibm.com/ca-addon-cr 19m + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/cognos-analytics-cognos-analytics-addon 1/1 1 1 9m17s + ``` -In the **ibm-cpd-operators** namespace: +### SPSS +Subscriptions related to SPSS (in the **ibm-cpd-operators** namespace): -- 10 workloads / 10 pods -- 0.984 CPU usage / 1.65 CPU requests / 7.3 CPU limit (60% utilization) -- 671.6 MiB memory usage, 2.56 GiB memory requests / 9.46 GiB memory limit (25% utilization) +- **cpd-platform-operator** +- **ibm-cpd-spss-operator** -``` -oc -n ibm-cpd-operators get deployments +SPSS is made up of many moving parts across multiple namespaces. + +In the **ibm-cpd-operators** namespace: + +```bash NAME READY UP-TO-DATE AVAILABLE AGE -cpd-platform-operator-manager 1/1 1 1 3h12m -gateway-operator 1/1 1 1 131m -ibm-common-service-operator 1/1 1 1 3h12m -ibm-elasticsearch-operator-ibm-es-controller-manager 1/1 1 1 131m -ibm-etcd-operator 1/1 1 1 131m -ibm-minio-operator 1/1 1 1 131m -ibm-model-train-classic-operator 1/1 1 1 131m -ibm-namespace-scope-operator 1/1 1 1 3h12m -ibm-rabbitmq-operator 1/1 1 1 131m -wd-discovery-operator 1/1 1 1 131m +ibm-cpd-canvasbase-operator 1/1 1 1 38m +ibm-cpd-spss-operator 1/1 1 1 38m ``` In the **ibm-cpd** namespace: -- 49 workloads / 83 pods -- 0.994 CPU usage / 20.06 CPU requests / 112.3 CPU limit (5% utilization) -- 12.2 GiB memory usage, 96.1 GiB memory requests / 195.5 GiB memory limit (12% utilization) -``` -oc -n ibm-cpd get watsondiscoveries,deployments,sts -NAME VERSION READY READYREASON UPDATING UPDATINGREASON DEPLOYED VERIFIED QUIESCE DATASTOREQUIESCE AGE -watsondiscovery.discovery.watson.ibm.com/wd 4.0.9 True Stable False Stable 23/23 23/23 NOT_QUIESCED NOT_QUIESCED 130m - -NAME READY UP-TO-DATE AVAILABLE AGE -deployment.apps/ibm-nginx 3/3 3 3 174m -deployment.apps/usermgmt 3/3 3 3 176m -deployment.apps/wd-discovery-cnm-api 1/1 1 1 39m -deployment.apps/wd-discovery-converter 1/1 1 1 39m -deployment.apps/wd-discovery-crawler 1/1 1 1 39m -deployment.apps/wd-discovery-gateway 1/1 1 1 19m -deployment.apps/wd-discovery-glimpse-builder 1/1 1 1 37m -deployment.apps/wd-discovery-glimpse-query 1/1 1 1 39m -deployment.apps/wd-discovery-haywire 1/1 1 1 39m -deployment.apps/wd-discovery-hdp-rm 1/1 1 1 39m -deployment.apps/wd-discovery-ingestion-api 1/1 1 1 39m -deployment.apps/wd-discovery-inlet 1/1 1 1 39m -deployment.apps/wd-discovery-management 1/1 1 1 39m -deployment.apps/wd-discovery-minerapp 1/1 1 1 24m -deployment.apps/wd-discovery-orchestrator 1/1 1 1 39m -deployment.apps/wd-discovery-outlet 1/1 1 1 39m -deployment.apps/wd-discovery-po-box 1/1 1 1 122m -deployment.apps/wd-discovery-project-data-prep-agent 1/1 1 1 39m -deployment.apps/wd-discovery-ranker-master 1/1 1 1 37m -deployment.apps/wd-discovery-ranker-monitor-agent 1/1 1 1 39m -deployment.apps/wd-discovery-ranker-rest 1/1 1 1 37m -deployment.apps/wd-discovery-rapi 1/1 1 1 120m -deployment.apps/wd-discovery-rcm 1/1 1 1 122m -deployment.apps/wd-discovery-serve-ranker 1/1 1 1 37m -deployment.apps/wd-discovery-stateless-api-model-runtime 1/1 1 1 39m -deployment.apps/wd-discovery-stateless-api-rest-proxy 1/1 1 1 39m -deployment.apps/wd-discovery-support 0/0 0 0 39m -deployment.apps/wd-discovery-tooling 1/1 1 1 24m -deployment.apps/wd-discovery-training-agents 1/1 1 1 39m -deployment.apps/wd-discovery-training-crud 1/1 1 1 39m -deployment.apps/wd-discovery-training-rest 1/1 1 1 39m -deployment.apps/wd-discovery-watson-gateway-gw-instance 1/1 1 1 15m -deployment.apps/wd-discovery-wd-indexer 1/1 1 1 122m -deployment.apps/wd-discovery-wksml 1/1 1 1 39m -deployment.apps/zen-audit 1/1 1 1 171m -deployment.apps/zen-core 3/3 3 3 171m -deployment.apps/zen-core-api 3/3 3 3 171m -deployment.apps/zen-data-sorcerer 2/2 2 2 165m -deployment.apps/zen-watchdog 1/1 1 1 165m -deployment.apps/zen-watcher 1/1 1 1 170m - -NAME READY AGE -statefulset.apps/dsx-influxdb 1/1 167m -statefulset.apps/wd-discovery-etcd 3/3 124m -statefulset.apps/wd-discovery-hdp-worker 2/2 39m -statefulset.apps/wd-discovery-sdu 1/1 24m -statefulset.apps/wd-ibm-elasticsearch-es-server-client 1/1 121m -statefulset.apps/wd-ibm-elasticsearch-es-server-data 1/1 121m -statefulset.apps/wd-ibm-elasticsearch-es-server-master 1/1 121m -statefulset.apps/wd-minio-discovery 4/4 125m -statefulset.apps/wd-rabbitmq-discovery 1/1 125m -statefulset.apps/zen-metastoredb 3/3 179m -``` +```bash +oc -n ibm-cpd get spss,deployments +NAME VERSION STATUS AGE +spss.spssmodeler.cpd.ibm.com/spssmodeler 9.0.0 Completed 38m +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/canvasbase-flow-api 1/1 1 1 35m +deployment.apps/canvasbase-flow-ui 1/1 1 1 30m +deployment.apps/spss-modeler-modeler-flow-api 1/1 1 1 22m + +``` Role Variables - Installation ----------------------------- ### cpd_service_name -Name of the service to install, supported values are: `wsl`, `wml`, `wd`, `aiopenscale` and `spark` +Name of the service to install, supported values are: `wsl`, `wml`, `wd`, `aiopenscale`, `spark`, `ca` and `spss` - **Required** - Environment Variable: `CPD_SERVICE_NAME` @@ -614,14 +342,14 @@ The product version (also known as operand version) of this service to install. - Default Value: Defined by the installed MAS catalog version ### cpd_service_storage_class -This is used to set `spec.storageClass` in all CPD services that uses file storage class (read-write-many). +This is used to set `spec.storageClass` in all CPD services that uses file storage class (read-write-many RWX). - **Required**, unless IBMCloud storage classes are available. - Environment Variable: `CPD_SERVICE_STORAGE_CLASS` - Default Value: Auto determined if default storage classes are provided and available by your cloud provider. i.e `ibmc-file` for IBM Cloud, `efs` for AWS. ### cpd_service_block_storage_class -This is used to set `spec.blockStorageClass` in all CPD services that uses block storage class (read-write-only). +This is used to set `spec.blockStorageClass` in all CPD services that uses block storage class (read-write-only RWO). - **Required**, unless IBMCloud storage classes are available. - Environment Variable: `CPD_SERVICE_BLOCK_STORAGE_CLASS` @@ -646,18 +374,18 @@ The CP4D Admin username to authenticate with CP4D APIs. If you didn't change the - Optional - Environment Variable: `CPD_ADMIN_USERNAME` -- Default Value: +- Default Value: - `admin` (CPD 4.6) - - `cpadmin` (CPD 4.8) + - `cpadmin` (CPD 4.8 and 5.0) ### cpd_admin_password The CP4D Admin User password to call CP4D API to provision Discovery Instance. If you didn't change the initial admin password after CP4D install, you don't need to provide it. The initial admin user password for `admin` or `cpdamin` will be used. - Optional - Environment Variable: `CPD_ADMIN_PASSWORD` -- Default Value: +- Default Value: - CPD 4.6: Looked up from the `admin-user-details` secret in the `cpd_instance_namespace` namespace - - CPD 4.8: Looked up from the `ibm-iam-bindinfo-platform-auth-idp-credentials` secret in the `cpd_instance_namespace` namespace + - CPD 4.8 and 5.0: Looked up from the `ibm-iam-bindinfo-platform-auth-idp-credentials` secret in the `cpd_instance_namespace` namespace ### cpd_service_scale_config Adjust and scale the resources for your Cloud Pak for Data services to increase processing capacity. @@ -694,7 +422,7 @@ Stores the name of the CP4D Watson Discovery Instance that can be used to config - Default Value: `wd-mas-${mas_instance_id}-assist` ### cpd_wd_deployment_type -Defines the CP4D Watson Discovery deployment type: +Defines the CP4D Watson Discovery deployment type: - `Starter`: One replica pod for each wd service/component, uses fewer resources in your cluster. - `Production`: Multiple replica pods for each Watson Discovery service/component, recommended for production deployments to increase workload capacity however consumes more cluster resources. @@ -730,7 +458,7 @@ Example Playbook - hosts: localhost any_errors_fatal: true vars: - cpd_product_version: 4.5.0 + cpd_product_version: 5.0.0 cpd_service_storage_class: ibmc-file-gold-gid cpd_service_name: wsl roles: diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/cp4d_service/tasks/determine-storage-classes.yml index 63e9666221..7d193bdc5d 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/determine-storage-classes.yml @@ -1,48 +1,38 @@ --- -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" -# 2. Set Storage (Required) +# 2. Debug storage class configuration # ----------------------------------------------------------------------------- -# - name: Default Metadata Storage if not set by user -# when: -# # - cpd_service_name not in cpd_services_supporting_block_storage_class # if not installing one of the services, then we just need a file storage -# - cpd_service_storage_class is not defined or cpd_service_storage_class == "" -# set_fact: -# cpd_service_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" - -- name: Set default File and Block Storage if not set by user +- name: "determine-storage-classes : Lookup & use default file storage class (if needed)" when: - # - cpd_service_name in cpd_services_supporting_block_storage_class # if installing one of the services, then we need both block and file storage + - defaultStorageClasses.success - cpd_service_storage_class is not defined or cpd_service_storage_class == "" + set_fact: + cpd_service_storage_class: "{{ defaultStorageClasses.rwx }}" + +- name: "determine-storage-classes : Lookup & use default block storage class (if needed)" + when: + - defaultStorageClasses.success - cpd_service_block_storage_class is not defined or cpd_service_block_storage_class == "" set_fact: - cpd_service_storage_class: "{{ (cpd_minor_version is version_compare('4.6', '>=')) | ternary(lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx), lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo)) }}" # if cpd 4.6+ then we need RWX as SC, otherwise it will be RWO. - cpd_service_block_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + cpd_service_block_storage_class: "{{ defaultStorageClasses.rwo }}" -- name: Assert that service storage class has been defined +- name: Assert that service storage class has been defined (RWX) assert: that: cpd_service_storage_class is defined and cpd_service_storage_class != "" fail_msg: "cpd_service_storage_class must be defined" -# 3. Debug storage class configuration -# ----------------------------------------------------------------------------- +- name: Assert that service storage class has been defined (RWO) + assert: + that: cpd_service_block_storage_class is defined and cpd_service_block_storage_class != "" + fail_msg: "cpd_service_block_storage_class must be defined" + - name: "Debug CP4D service {{ cpd_service_name }} storage class configuration" debug: msg: - - "Storage class (CP4D service {{ cpd_service_name }}) ............................ {{ cpd_service_storage_class }}" - - "Block Storage class (CP4D service {{ cpd_service_name }}) ...................... {{ cpd_service_block_storage_class | default('N/A',true) }}" + - "Storage Class (RWX) .................... {{ cpd_service_storage_class }}" + - "Block Storage Class (RWO) .............. {{ cpd_service_block_storage_class }}" diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml index cd71465703..f304fa5bef 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml @@ -109,16 +109,7 @@ - cpd_48_or_higher # elastic search operator was just introduced with cpd 4.8 - not skip_ibm_entitlement_injection # eventually we hope to be able to skip patching the elastic search cr with image pull secret, but not for now -# 5. Wait for CouchDB Stateful Set to be ready -# ----------------------------------------------------------------------------- -# There have been issues with CouchDB not starting due to Persistent Storage, -# This task restarts any failing pods -- include_tasks: "tasks/wait/wait-couchdb.yml" - when: - - cpd_48_or_higher - - -# 6. Wait for CCS CR to be ready +# 5. Wait for CCS CR to be ready # ----------------------------------------------------------------------------- # Note: We can't fail early when we see Failed status, as the operator will # report failed multiple times during initial reconcile. diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml deleted file mode 100644 index 8b7cf0939f..0000000000 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml +++ /dev/null @@ -1,108 +0,0 @@ ---- -# 1. Wait for couch-db stateful set to start all the replica pods -# ----------------------------------------------------------------------------- -- name: "wait-couchdb: Wait for CouchDB pods to be created" - k8s_info: - kind: StatefulSet - namespace: "{{ cpd_instance_namespace }}" - name: "wdp-couchdb" - register: couchdbStatefulSet - retries: 40 # Give 20 minutes for the ccs Operator to start CouchDB Pods (Logs show this taking ~7 minutes in a good run) - delay: 30 - until: >- - (( couchdbStatefulSet.resources[0].status is defined - and couchdbStatefulSet.resources[0].status.replicas is defined - and couchdbStatefulSet.resources[0].status.replicas == 0 ) - or ( couchdbStatefulSet.resources[0].status is defined - and couchdbStatefulSet.resources[0].status.updatedReplicas is defined - and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.updatedReplicas )) - - -# 2. Wait for couchdb replica pods to become ready -# ----------------------------------------------------------------------------- -- name: "wait-couchdb: Wait for CouchDB pods to be become ready" - k8s_info: - kind: StatefulSet - namespace: "{{ cpd_instance_namespace }}" - name: "wdp-couchdb" - register: couchdbStatefulSet - retries: 10 # Give 5 minutes for the pods to become ready - delay: 30 - until: >- - couchdbStatefulSet.resources[0].status.readyReplicas is defined - and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.readyReplicas - #ignore-errors: true # If this fails then we restart pending pods below - failed_when: false - -# 2. Restart any couchDB pods that are still Pending -# ----------------------------------------------------------------------------- -- set_fact: - is_couchdb_ready: true - when: - couchdbStatefulSet.resources[0].status.readyReplicas is defined - and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.readyReplicas - -- name: "wait-couchdb: Detecting and restarting pending CouchDB Pods" - block: - - name: "install-cp4d : Get pending CouchDB Pods" - kubernetes.core.k8s_info: - api_version: v1 - kind: Pod - label_selectors: - - "app=couchdb" - field_selectors: - - "status.phase=Pending" - namespace: "{{ cpd_instance_namespace }}" - register: pending_pod_lookup - - - set_fact: - pending_pod_names: "{{ pending_pod_lookup.resources | map(attribute='metadata.name') }}" - - - debug: - msg: "Restarting pending CouchDB Pods: {{ pending_pod_names }}" - - - name: "wait-couchdb: Restarting pending CouchDB Pods" - kubernetes.core.k8s: - state: absent - api_version: v1 - kind: Pod - namespace: "{{ cpd_instance_namespace }}" - name: "{{ item }}" - loop: "{{ pending_pod_names }}" - - # 3. Wait again couchdb replica pods to become ready - # ----------------------------------------------------------------------------- - - name: "wait-couchdb: Wait for CouchDB pods to be become ready" - k8s_info: - kind: StatefulSet - namespace: "{{ cpd_instance_namespace }}" - name: "wdp-couchdb" - register: couchdbStatefulSet - retries: 10 # Give another 5 minutes for the pods to become ready - delay: 30 - until: >- - couchdbStatefulSet.resources[0].status.readyReplicas is defined - and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.readyReplicas - failed_when: false # We handle and log the failure below. - - - name: "wait-couchdb: Fail if CouchDB pods are not ready" - block: - - name: "install-cp4d : Get Pending CouchDB Pods" - kubernetes.core.k8s_info: - api_version: v1 - kind: Pod - label_selectors: - - "app=couchdb" - field_selectors: - - "status.phase=Pending" - namespace: "{{ cpd_instance_namespace }}" - register: pending_pod_lookup - - - fail: - msg: - - "CouchDB pods are not ready {{ couchdbStatefulSet.resources[0].status }}" - - "Pending CouchDB Pods: {{ pending_pod_lookup.resources | map(attribute='metadata.name') }}" - when: - couchdbStatefulSet.resources[0].status.replicas != couchdbStatefulSet.resources[0].status.readyReplicas - - when: is_couchdb_ready is not defined diff --git a/ibm/mas_devops/roles/db2/README.md b/ibm/mas_devops/roles/db2/README.md index a959e4f2b2..7691c626e8 100644 --- a/ibm/mas_devops/roles/db2/README.md +++ b/ibm/mas_devops/roles/db2/README.md @@ -139,7 +139,7 @@ Role Variables - Storage We recommend reviewing the Db2 documentation about the certified storage options for Db2 on Red Hat OpenShift. Please ensure your storage class meets the specified deployment requirements for Db2. [https://www.ibm.com/docs/en/db2/11.5?topic=storage-certified-options](https://www.ibm.com/docs/en/db2/11.5?topic=storage-certified-options) ### db2_meta_storage_class -Storage class used for metadata. This must support ReadWriteMany +Storage class used for metadata. This must support ReadWriteMany(RWX) access mode. - **Required** - Environment Variable: `DB2_META_STORAGE_CLASS` @@ -160,7 +160,7 @@ The access mode for the storage. - Default: `ReadWriteMany` ### db2_data_storage_class -Storage class used for user data. This must support ReadWriteOnce +Storage class used for user data. This must support ReadWriteMany(RWX) access mode. - **Required** - Environment Variable: `DB2_DATA_STORAGE_CLASS` @@ -181,7 +181,7 @@ The access mode for the storage. - Default: `ReadWriteOnce` ### db2_backup_storage_class -Storage class used for backup. This must support ReadWriteMany +Storage class used for backup. This must support ReadWriteMany(RWX) access mode. - Optional - Environment Variable: `DB2_BACKUP_STORAGE_CLASS` @@ -202,7 +202,7 @@ The access mode for the storage. - Default: `ReadWriteMany` ### db2_logs_storage_class -Storage class used for transaction logs. This must support ReadWriteOnce +Storage class used for transaction logs. This must support ReadWriteMany(RWX) access mode. - Optional - Environment Variable: `DB2_LOGS_STORAGE_CLASS` @@ -223,7 +223,7 @@ The access mode for the storage. - Default: `ReadWriteOnce` ### db2_temp_storage_class -Storage class used for temporary data. This must support ReadWriteOnce +Storage class used for temporary data. This must support ReadWriteMany(RWX) access mode. - Optional - Environment Variable: `DB2_TEMP_STORAGE_CLASS` @@ -237,7 +237,7 @@ Size of temporary persistent volume. - Default: `100Gi` ### db2_temp_storage_accessmode -The access mode for the storage. +The access mode for the storage. This must support ReadWriteOnce(RWO) access mode. - Optional - Environment Variable: `DB2_TEMP_STORAGE_ACCESSMODE` @@ -249,9 +249,9 @@ Role Variables - Resource Requests These variables allow you to customize the resources available to the Db2 pod in your cluster. In most circumstances you will want to set these properties because it's impossible for us to provide a default value that will be appropriate for all users. We have set defaults that are suitable for deploying Db2 onto a dedicated worker node with 4cpu and 16gb memory. !!! tip - Note that you must take into account the system overhead on any given node when setting these parameters, if you set the requests equal to the number of CPU or amount of memory on yournode then the scheduler will not be able to schedule the Db2 pod because not 100% of the worker nodes' resource will be available to pod on that node, even if there's only a single pod on it. + Note that you must take into account the system overhead on any given node when setting these parameters, if you set the requests equal to the number of CPU or amount of memory on your node then the scheduler will not be able to schedule the Db2 pod because not 100% of the worker nodes' resource will be available to pod on that node, even if there's only a single pod on it. - Db2 is sensitive to both CPU and memory issues, particularly memory, we recommennd setting requests and limits to the same values, ensuring the scheduler always reserves the resources that Db2 expects to be available to it. + Db2 is sensitive to both CPU and memory issues, particularly memory, we recommend setting requests and limits to the same values, ensuring the scheduler always reserves the resources that Db2 expects to be available to it. ### db2_cpu_requests Define the Kubernetes CPU request for the Db2 pod. @@ -442,82 +442,13 @@ Set the [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - Environment Variable: `MASBR_JOB_TIMEZONE` - Default: None -### masbr_storage_type -Set `local` or `cloud` to indicate this role to save the backup files to local file system or cloud object storage. - -- **Required** -- Environment Variable: `MASBR_STORAGE_TYPE` -- Default: None - ### masbr_storage_local_folder Set local path to save the backup files. -- **Required** only when `MASBR_STORAGE_TYPE=local` +- **Required** - Environment Variable: `MASBR_STORAGE_LOCAL_FOLDER` - Default: None -### masbr_storage_cloud_rclone_file -Set the path of `rclone.conf` file. - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_RCLONE_FILE` -- Default: None - -### masbr_storage_cloud_rclone_name -Set the configuration name defined in `rclone.conf` file. - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_RCLONE_NAME` -- Default: None - -### masbr_storage_cloud_bucket -Set the object storage bucket name for saving the backup files - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_BUCKET` -- Default: None - -### masbr_slack_enabled -Set `true` or `false` to indicate whether this role will send Slack notification messages of the backup and restore progress. - -- Optional -- Environment Variable: `MASBR_SLACK_ENABLED` -- Default: `false` - -### masbr_slack_level -Set `failure`, `info` or `verbose` to indicate this role to send Slack notification messages in which backup and resore phases: - -| Slack level | Backup/Restore phases | -| ----------- | ------------------------------------------------------- | -| failure | `Failed`, `PartiallyFailed` | -| info | `Completed`, `Failed`, `PartiallyFailed` | -| verbose | `InProgress`, `Completed`, `Failed`, `PartiallyFailed` | - -- Optional -- Environment Variable: `MASBR_SLACK_LEVEL` -- Default: `info` - -### masbr_slack_token -The Slack integration token. - -- **Required** only when `MASBR_SLACK_ENABLED=true` -- Environment Variable: `MASBR_SLACK_TOKEN` -- Default: None - -### masbr_slack_channel -The Slack channel to send the notification messages to. - -- **Required** only when `MASBR_SLACK_ENABLED=true` -- Environment Variable: `MASBR_SLACK_CHANNEL` -- Default: None - -### masbr_slack_user -The sender of the Slack notification message. - -- Optional -- Environment Variable: `MASBR_SLACK_USER` -- Default: `MASBR` - ### masbr_backup_type Set `full` or `incr` to indicate the role to create a full backup or incremental backup. @@ -580,7 +511,6 @@ Example Playbook vars: db2_action: backup db2_instance_name: db2u-db01 - masbr_storage_type: local masbr_storage_local_folder: /tmp/masbr roles: - ibm.mas_devops.db2 @@ -594,7 +524,6 @@ Example Playbook db2_action: restore db2_instance_name: db2u-db01 masbr_restore_from_version: 20240621021316 - masbr_storage_type: local masbr_storage_local_folder: /tmp/masbr roles: - ibm.mas_devops.db2 diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index 5227678445..e25acad2e4 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -30,28 +30,7 @@ _component_before_task_path: "{{ role_path }}/tasks/before-backup-restore.yml" -# Create k8s Job to run backup tasks -# ----------------------------------------------------------------------------- -- name: "Create k8s Job to run backup tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" - vars: - _rt_role_name: "db2" - _rt_env: - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "DB2_ACTION" - value: "{{ db2_action }}" - - name: "DB2_INSTANCE_NAME" - value: "{{ db2_instance_name }}" - - name: "DB2_NAMESPACE" - value: "{{ db2_namespace }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - - name: "Run backup tasks" - when: not masbr_create_task_job block: # Update backup job status: New # ------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/tasks/install/determine-storage-classes.yml b/ibm/mas_devops/roles/db2/tasks/install/determine-storage-classes.yml index ba98fb9854..5f74a94bf0 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/determine-storage-classes.yml @@ -1,30 +1,19 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Data Storage (Required) # ----------------------------------------------------------------------------- - name: Default Data Storage for ROKS if not set by user (ReadWriteOnce) - when: db2_data_storage_class is not defined or db2_data_storage_class == "" + when: + - db2_data_storage_class is not defined or db2_data_storage_class == "" + - defaultStorageClasses.success vars: set_fact: - db2_data_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + db2_data_storage_class: "{{ defaultStorageClasses.rwo }}" - name: Assert that a data storage class has been defined assert: @@ -35,9 +24,11 @@ # 3. Meta Storage (Required) # ----------------------------------------------------------------------------- - name: Default Meta Storage for ROKS if not set by user (ReadWriteMany) - when: db2_meta_storage_class is not defined or db2_meta_storage_class == "" + when: + - db2_meta_storage_class is not defined or db2_meta_storage_class == "" + - defaultStorageClasses.success set_fact: - db2_meta_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" + db2_meta_storage_class: "{{ defaultStorageClasses.rwx }}" - name: Assert that a meta storage class has been defined assert: @@ -48,9 +39,11 @@ # 4. Logs Storage (Optional) # ----------------------------------------------------------------------------- - name: Default Logs Storage for ROKS if not set by user (ReadWriteOnce) - when: db2_logs_storage_class is not defined or db2_logs_storage_class == "" + when: + - db2_logs_storage_class is not defined or db2_logs_storage_class == "" + - defaultStorageClasses.success set_fact: - db2_logs_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + db2_logs_storage_class: "{{ defaultStorageClasses.rwo }}" # 5. Backup Storage (Optional) @@ -58,15 +51,17 @@ - name: Default Backup Storage for ROKS if not set by user (ReadWriteMany) when: db2_backup_storage_class is not defined or db2_backup_storage_class == "" set_fact: - db2_backup_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" + db2_backup_storage_class: "{{ defaultStorageClasses.rwx }}" # 6. Temp Storage (Optional) # ----------------------------------------------------------------------------- - name: Default Temp Storage for ROKS if not set by user (ReadWriteOnce) - when: db2_temp_storage_class is not defined or db2_temp_storage_class == "" + when: + - db2_temp_storage_class is not defined or db2_temp_storage_class == "" + - defaultStorageClasses.success set_fact: - db2_temp_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + db2_temp_storage_class: "{{ defaultStorageClasses.rwo }}" # 7. Debug storage class configuration @@ -74,8 +69,8 @@ - name: "Debug DB2U storage class configuration" debug: msg: - - "Storage class (meta) ......... {{ db2_meta_storage_class }}" - - "Storage class (data) ......... {{ db2_data_storage_class }}" - - "Storage class (backup) ....... {{ db2_backup_storage_class }}" - - "Storage class (logs) ......... {{ db2_logs_storage_class }}" - - "Storage class (temp) ......... {{ db2_temp_storage_class }}" + - "Storage class (meta) ................... {{ db2_meta_storage_class }}" + - "Storage class (data) ................... {{ db2_data_storage_class }}" + - "Storage class (backup) ................. {{ db2_backup_storage_class | default('', true) }}" + - "Storage class (logs) ................... {{ db2_logs_storage_class | default('', true) }}" + - "Storage class (temp) ................... {{ db2_temp_storage_class | default('', true) }}" diff --git a/ibm/mas_devops/roles/db2/tasks/install/main.yml b/ibm/mas_devops/roles/db2/tasks/install/main.yml index 25908b7b95..345d7bf2bd 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/main.yml @@ -157,12 +157,14 @@ - name: "Delete old db2 subscription, operand request and csv from {{ ibm_common_services_namespace }}" include_tasks: "tasks/delete_db2_operand_request.yml" + - name: "Create Db2 Universal Operator Subscription in {{ db2_namespace }} namespace" - include_tasks: "{{ role_path }}/../../common_tasks/create_subscription.yml" - vars: - subscription_namespace: "{{ db2_namespace }}" + ibm.mas_devops.apply_subscription: + namespace: "{{ db2_namespace }}" package_name: db2u-operator - channel_name: "{{ db2_channel }}" + package_channel: "{{ db2_channel }}" + register: subscription + # 7. Get the cluster subdomain to be used for the certificate and route creation # ----------------------------------------------------------------------------- @@ -317,7 +319,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: Secret - name: "db2u-certificate" + name: "db2u-certificate-{{db2_instance_name}}" namespace: "{{db2_namespace}}" register: _db2_instance_certificates diff --git a/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml b/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml index 97d724931e..20a7f46c72 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml @@ -19,7 +19,7 @@ {{ exec_in_pod_begin }} mkdir -p {{ db2_restore_folder }}/{{ _job_name }} && tar -xzf {{ db2_restore_folder }}/{{ _job_name }}.tar.gz - -C {{ db2_restore_folder }}/{{ _job_name }} . && + -C {{ db2_restore_folder }}/{{ _job_name }} && ls {{ db2_restore_folder }}/{{ _job_name }} {{ exec_in_pod_end }} register: _extract_output diff --git a/ibm/mas_devops/roles/db2/tasks/restore/main.yml b/ibm/mas_devops/roles/db2/tasks/restore/main.yml index a9f5263980..a6a1b18a25 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/main.yml @@ -29,28 +29,7 @@ _component_before_task_path: "{{ role_path }}/tasks/before-backup-restore.yml" -# Create k8s Job to run restore tasks -# ----------------------------------------------------------------------------- -- name: "Create k8s Job to run restore tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" - vars: - _rt_role_name: "db2" - _rt_env: - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "DB2_ACTION" - value: "{{ db2_action }}" - - name: "DB2_INSTANCE_NAME" - value: "{{ db2_instance_name }}" - - name: "DB2_NAMESPACE" - value: "{{ db2_namespace }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - - name: "Run restore tasks" - when: not masbr_create_task_job block: # Update restore job status: New # ------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/tasks/upgrade/run-db2-subscription-upgrade.yml b/ibm/mas_devops/roles/db2/tasks/upgrade/run-db2-subscription-upgrade.yml index 4160d3483d..3c25639e02 100644 --- a/ibm/mas_devops/roles/db2/tasks/upgrade/run-db2-subscription-upgrade.yml +++ b/ibm/mas_devops/roles/db2/tasks/upgrade/run-db2-subscription-upgrade.yml @@ -22,11 +22,11 @@ # Leverage that we're upgrading DB2 and check if need to migrate db2 operator from ibm-common-services namespace - name: "Update Db2 Universal Operator Subscription in {{ db2_namespace }} namespace" - include_tasks: "{{ role_path }}/../../common_tasks/create_subscription.yml" - vars: - subscription_namespace: "{{ db2_namespace }}" + ibm.mas_devops.apply_subscription: + namespace: "{{ db2_namespace }}" package_name: db2u-operator - channel_name: "{{ db2_channel }}" + package_channel: "{{ db2_channel }}" + register: subscription - name: "Pause for a minute before checking subscription status..." pause: diff --git a/ibm/mas_devops/roles/db2/templates/certs/certificate.yml.j2 b/ibm/mas_devops/roles/db2/templates/certs/certificate.yml.j2 index bf9676c4c1..d17754d2ae 100644 --- a/ibm/mas_devops/roles/db2/templates/certs/certificate.yml.j2 +++ b/ibm/mas_devops/roles/db2/templates/certs/certificate.yml.j2 @@ -1,7 +1,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: "db2u-certificate" + name: "db2u-certificate-{{db2_instance_name}}" namespace: "{{ db2_namespace }}" {% if custom_labels is defined and custom_labels.items() %} labels: @@ -10,7 +10,7 @@ metadata: {% endfor %} {% endif %} spec: - secretName: "db2u-certificate" + secretName: "db2u-certificate-{{db2_instance_name}}" duration: "175200h0m0s" renewBefore: "2160h0m0s" issuerRef: diff --git a/ibm/mas_devops/roles/db2/templates/db2ucluster.yml.j2 b/ibm/mas_devops/roles/db2/templates/db2ucluster.yml.j2 index 867c3838bc..563c846582 100644 --- a/ibm/mas_devops/roles/db2/templates/db2ucluster.yml.j2 +++ b/ibm/mas_devops/roles/db2/templates/db2ucluster.yml.j2 @@ -57,7 +57,7 @@ spec: settings: dftTableOrg: "{{db2_table_org }}" ssl: - secretName: "db2u-certificate" + secretName: "db2u-certificate-{{db2_instance_name}}" certLabel: "CN=db2u" instance: {% if db2_instance_registry is defined and db2_instance_registry != "" %} diff --git a/ibm/mas_devops/roles/dro/README.md b/ibm/mas_devops/roles/dro/README.md index a1b33bb28e..b2f980efc4 100644 --- a/ibm/mas_devops/roles/dro/README.md +++ b/ibm/mas_devops/roles/dro/README.md @@ -42,11 +42,12 @@ Provide your [IBM entitlement key](https://myibm.ibm.com/products-services/conta - Default: None ### dro_storage_class -Required. Storage class where DRO will be installed. MAS ansible playbooks will automatically try to determine a rwo (Read Write Once) storage class from a cluster if DRO_STORAGE_CLASS is not supplied. If a cluster is setup with a customize storage solution, please provide a valid rwo storage class name using DRO_STORAGE_CLASS +Required. Storage class where DRO will be installed. MAS ansible playbooks will automatically try to determine a RWO (Read Write Once) storage class from a cluster if DRO_STORAGE_CLASS is not supplied. If a cluster is setup with a customized storage solution, please provide a valid RWO storage class name using DRO_STORAGE_CLASS. - Optional - Environment Variable: `DRO_STORAGE_CLASS` - Default Value: None +- **Note**: The storage class must support the RWO(Read Write Once) access Mode Role Variables - BASCfg Generation ------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/dro/tasks/install-dro/determine-storage-classes.yml b/ibm/mas_devops/roles/dro/tasks/install-dro/determine-storage-classes.yml index 9338667106..2ca02915a6 100644 --- a/ibm/mas_devops/roles/dro/tasks/install-dro/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/dro/tasks/install-dro/determine-storage-classes.yml @@ -1,22 +1,8 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: - - "Available storage classes .................. {{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Set Storage (Required) @@ -24,7 +10,7 @@ - name: Default Storage for ROKS if not set by user when: dro_storage_class is not defined or dro_storage_class == "" set_fact: - dro_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + dro_storage_class: "{{ defaultStorageClasses.rwo }}" - name: Assert that storage class has been defined assert: @@ -37,4 +23,4 @@ - name: "Debug DRO storage class configuration" debug: msg: - - "Auto Detected Storage class for DRO .................... {{ dro_storage_class }}" + - "Auto Detected Storage class for DRO .... {{ dro_storage_class }}" diff --git a/ibm/mas_devops/roles/grafana/README.md b/ibm/mas_devops/roles/grafana/README.md index acd2fc9479..d82a071d6d 100644 --- a/ibm/mas_devops/roles/grafana/README.md +++ b/ibm/mas_devops/roles/grafana/README.md @@ -42,7 +42,7 @@ Sets the namespace to install the grafana operator V5 and grafana instance ### grafana_instance_storage_class Declare the storage class for Grafana Instance user data persistent volume. -- **Required** if one of the known supported storage classes is not installed in the cluster. +- **Required** if one of the known supported storage classes is not installed in the cluster. Storage classes must support ReadWriteOnce (RWO) access mode. - Environment Variable: `GRAFANA_INSTANCE_STORAGE_CLASS` - Default Value: `ibmc-file-gold-gid`, `ocs-storagecluster-cephfs`, `azurefiles-premium` (if available) diff --git a/ibm/mas_devops/roles/grafana/tasks/install/determine-storage-classes.yml b/ibm/mas_devops/roles/grafana/tasks/install/determine-storage-classes.yml index c74bfdd6b6..d429a9e209 100644 --- a/ibm/mas_devops/roles/grafana/tasks/install/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/grafana/tasks/install/determine-storage-classes.yml @@ -1,17 +1,8 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "install : determine-storage-classes : Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Grafana Instance Storage (Required) @@ -19,7 +10,7 @@ - name: "install : determine-storage-classes : Default Grafana Instance Storage if not set by user" when: grafana_instance_storage_class is not defined or grafana_instance_storage_class == "" set_fact: - grafana_instance_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + grafana_instance_storage_class: "{{ defaultStorageClasses.rwo }}" - name: "install : determine-storage-classes : Assert that Grafana Instance storage class has been defined" assert: diff --git a/ibm/mas_devops/roles/grafana/tasks/install/main.yml b/ibm/mas_devops/roles/grafana/tasks/install/main.yml index 1a36b46704..fbdf94f041 100644 --- a/ibm/mas_devops/roles/grafana/tasks/install/main.yml +++ b/ibm/mas_devops/roles/grafana/tasks/install/main.yml @@ -17,12 +17,30 @@ register: uwm_secret_lookup no_log: true +- name: "install : Check for existing user workload monitoring configmap" + kubernetes.core.k8s_info: + api: v1 + kind: ConfigMap + name: cluster-monitoring-config + namespace: openshift-monitoring + register: cluster_monitoring_configmap + +# If the monitoring config doesn't exist we just create a new configmap with just the enableUserWorkload added +- name: "install : Creating a new configmap for user workload monitoring" + when: cluster_monitoring_configmap.resources | length == 0 + set_fact: + cluster_monitoring_data: "{{ {'enableUserWorkload': True} }}" + +# If the monitoring config does exist, we can just grap the existing yaml and override the enableUserWorkload value +- name: "update : extracting existing data from configmap" + when: cluster_monitoring_configmap.resources | length > 0 + set_fact: + cluster_monitoring_data: "{{ cluster_monitoring_configmap.resources[0].data['config.yaml'] | from_yaml | combine({'enableUserWorkload': True}) }}" + - name: "install : Configure User Workload Monitoring" - when: uwm_secret_lookup.resources | length == 0 kubernetes.core.k8s: template: templates/cluster-monitoring-config.yml.j2 - - name: "Debug version" debug: var: grafana_major_version @@ -42,17 +60,18 @@ # 4. Create Grafana Subscription # ----------------------------------------------------------------------------- - name: "install : Create Grafana v{{grafana_major_version}} Subscription" - include_tasks: "{{ role_path }}/../../common_tasks/create_subscription.yml" - vars: + ibm.mas_devops.apply_subscription: + namespace: "{{ grafana_namespace }}" package_name: grafana-operator - channel_name: "v{{grafana_major_version}}" - subscription_namespace: "{{ grafana_namespace }}" - subscription_config: + package_channel: "v{{grafana_major_version}}" + config: env: - name: "WATCH_NAMESPACE" value: "" - name: "DASHBOARD_NAMESPACES_ALL" value: "true" + register: subscription + # 5. Configure Grafana Operator so it can scan all namespaces for dashboards # ------------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/grafana/tasks/update/determine-storage-config.yml b/ibm/mas_devops/roles/grafana/tasks/update/determine-storage-config.yml index d7c5457783..e8178d4502 100644 --- a/ibm/mas_devops/roles/grafana/tasks/update/determine-storage-config.yml +++ b/ibm/mas_devops/roles/grafana/tasks/update/determine-storage-config.yml @@ -3,7 +3,7 @@ # 1. Set Grafana Instance Storage Class and Size from existing V5 Operator # ----------------------------------------------------------------------------- -- name: "update : determine-storage-classes : Get V5 Grafana Instance" +- name: "update : determine-storage-config : Get V5 Grafana Instance" kubernetes.core.k8s_info: api_version: grafana.integreatly.org/v1beta1 name: mas-grafana @@ -11,7 +11,7 @@ kind: Grafana register: grafana_cr_result -- name: "update : determine-storage-classes : Set the Grafana storage class and size" +- name: "update : determine-storage-config : Set the Grafana storage class and size" when: grafana_cr_result.resources[0].spec.persistentVolumeClaim is defined set_fact: grafana_instance_storage_class: "{{ grafana_cr_result.resources[0].spec.persistentVolumeClaim.spec.storageClassName }}" @@ -20,7 +20,7 @@ # 2. Set Grafana Instance Storage Class and Size from existing V4 Operator # ----------------------------------------------------------------------------- -- name: "update : determine-storage-classes : Get V4 Grafana Instance" +- name: "update : determine-storage-config : Get V4 Grafana Instance" when: grafana_instance_storage_class is not defined or grafana_instance_storage_class == "" kubernetes.core.k8s_info: api_version: integreatly.org/v1alpha1 @@ -29,7 +29,7 @@ kind: Grafana register: grafana_cr_result -- name: "update : determine-storage-classes : Set the Grafana storage class and size" +- name: "update : determine-storage-config : Set the Grafana storage class and size" when: - grafana_instance_storage_class is not defined or grafana_instance_storage_class == "" - grafana_cr_result.resources[0].spec.dataStorage is defined @@ -40,26 +40,18 @@ # 3. Set Default Grafana Instance Storage Class if not already determined # ----------------------------------------------------------------------------- -- name: "update : determine-storage-classes : Load default storage class information" - when: grafana_instance_storage_class is not defined or grafana_instance_storage_class == "" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: "update : Lookup storage classes" - when: grafana_instance_storage_class is not defined or grafana_instance_storage_class == "" - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses +- name: "update : determine-storage-config : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" -- name: "update : determine-storage-classes : Default Grafana Instance Storage if still not set" +- name: "update : determine-storage-config : Default Grafana Instance Storage if still not set" when: grafana_instance_storage_class is not defined or grafana_instance_storage_class == "" set_fact: - grafana_instance_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + grafana_instance_storage_class: "{{ defaultStorageClasses.rwo }}" # 4. Fail if Grafana Storage Class has not been set # ----------------------------------------------------------------------------- -- name: "update : determine-storage-classes : Assert that Grafana Instance storage class has been defined" +- name: "update : determine-storage-config : Assert that Grafana Instance storage class has been defined" assert: that: grafana_instance_storage_class is defined and grafana_instance_storage_class != "" fail_msg: "grafana_instance_storage_class cannot be determined" @@ -67,7 +59,7 @@ # 5. Debug storage class configuration # ----------------------------------------------------------------------------- -- name: "update : determine-storage-classes : Debug grafana storage class configuration" +- name: "update : determine-storage-config : Debug grafana storage class configuration" debug: msg: - "Storage class (grafana instance) ....... {{ grafana_instance_storage_class }}" diff --git a/ibm/mas_devops/roles/grafana/templates/cluster-monitoring-config.yml.j2 b/ibm/mas_devops/roles/grafana/templates/cluster-monitoring-config.yml.j2 index 0d768886ce..016892a38b 100644 --- a/ibm/mas_devops/roles/grafana/templates/cluster-monitoring-config.yml.j2 +++ b/ibm/mas_devops/roles/grafana/templates/cluster-monitoring-config.yml.j2 @@ -1,3 +1,4 @@ +# We use to_yaml and trim to ensure formatting, booleans and numbers are represented correctly apiVersion: v1 kind: ConfigMap metadata: @@ -5,4 +6,4 @@ metadata: namespace: openshift-monitoring data: config.yaml: | - enableUserWorkload: true + {{ cluster_monitoring_data | to_nice_yaml | indent(4) }} \ No newline at end of file diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/install/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/install/main.yml index 9ded0007f3..6023105083 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/install/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/install/main.yml @@ -26,10 +26,10 @@ - artifactory_token is not defined or artifactory_token == "" - airgap_install block: - - name: "Load CASE bundle versions for the offline catalog" - include_vars: - file: "{{ role_path }}/../../common_vars/casebundles/{{ mas_catalog_version }}.yml" - + - name: "Load Catalog Metadata" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ mas_catalog_version }}" + register: mas_catalog_metadata - name: "Create IBM offline catalog" kubernetes.core.k8s: template: templates/offline-catalog.yml.j2 diff --git a/ibm/mas_devops/roles/ibm_catalogs/templates/offline-catalog.yml.j2 b/ibm/mas_devops/roles/ibm_catalogs/templates/offline-catalog.yml.j2 index 6513d3310f..df82b5d9ce 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/templates/offline-catalog.yml.j2 +++ b/ibm/mas_devops/roles/ibm_catalogs/templates/offline-catalog.yml.j2 @@ -9,5 +9,5 @@ spec: publisher: IBM description: Offline Catalog Source for IBM Maximo Application Suite sourceType: grpc - image: icr.io/cpopen/ibm-maximo-operator-catalog@{{ catalog_digest }} + image: icr.io/cpopen/ibm-maximo-operator-catalog@{{ mas_catalog_metadata.catalog_digest }} priority: 90 diff --git a/ibm/mas_devops/roles/install_operator/README.md b/ibm/mas_devops/roles/install_operator/README.md deleted file mode 100644 index 56f3d05acf..0000000000 --- a/ibm/mas_devops/roles/install_operator/README.md +++ /dev/null @@ -1,30 +0,0 @@ -install_operator -================ - -TODO: Summarize role - -Role Variables --------------- - -### custom_labels -List of comma separated key=value pairs for setting custom labels on instance specific resources. - -- Optional -- Environment Variable: `CUSTOM_LABELS` -- Default Value: None - - -TODO: Finish documentation - - -Example Playbook ----------------- - -```yaml -TODO: Add example -``` - -License -------- - -EPL-2.0 diff --git a/ibm/mas_devops/roles/install_operator/defaults/main.yml b/ibm/mas_devops/roles/install_operator/defaults/main.yml deleted file mode 100644 index 2573b25617..0000000000 --- a/ibm/mas_devops/roles/install_operator/defaults/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# Dev Catalog Settings -# ----------------------------------------------------------------------------- -# Only required when using the development catalog sources for pre-release installation -# These are used to set up image pull secret in the openshift-marketplace namespace -# -# They will also be added to the ibm-entitlement image pull secret, this allows released -# versions of operators to be mixed with development versions of dependencies (e.g. truststore manager) -artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" -artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" - -# Custom Labels -# ----------------------------------------------------------------------------- -custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string | ibm.mas_devops.string2dict() }}" diff --git a/ibm/mas_devops/roles/install_operator/tasks/main.yml b/ibm/mas_devops/roles/install_operator/tasks/main.yml deleted file mode 100644 index b83f2b74a8..0000000000 --- a/ibm/mas_devops/roles/install_operator/tasks/main.yml +++ /dev/null @@ -1,91 +0,0 @@ ---- -# 1. Check artifactory_username/icr_username combination -# ----------------------------------------------------------------------------- -- name: "Fail if icr_username is provided but not icr_password" - ansible.builtin.assert: - that: icr_password is defined and icr_password != "" - fail_msg: "icr_password property has not been set" - when: icr_username is defined and icr_username != "" - -- name: "Fail if artifactory_username is provided but not artifactory_token" - ansible.builtin.assert: - that: artifactory_token is defined and artifactory_token != "" - fail_msg: "artifactory_token property has not been set" - when: artifactory_username is defined and artifactory_username != "" - -# 2. Create namespace we will deploy to -# ----------------------------------------------------------------------------- -- name: "Create namespace" - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: "{{ namespace }}" - -- name: Add custom labels to namespace - when: custom_labels is defined and custom_labels | length > 0 - kubernetes.core.k8s: - state: patched - kind: Namespace - name: "{{ namespace }}" - definition: "{{ lookup('template', 'templates/custom_labels.json.j2') }}" - -# 3. Create an image pull secret for the pre-release catalog -# ----------------------------------------------------------------------------- -- name: "Debug Entitlement Secret Creation" - debug: - msg: - - "Target Namespace ....................... {{ namespace }}" - - "Artifactory Username ................... {{ artifactory_username | default('', true) }}" - - "Artifactory Password ................... {{ (artifactory_token is defined and artifactory_token != '') | ternary('************', '') }}" - - "ICR Username ........................... {{ icr_username | default('', true) }}" - - "ICR Password ........................... {{ (icr_password is defined and icr_password != '') | ternary('************', '') }}" - -- name: "Create ibm-entitlement secret" - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson - metadata: - name: ibm-entitlement - namespace: "{{ namespace }}" - data: - .dockerconfigjson: "{{ lookup('template', 'templates/ibm-entitlement-with-artifactory.json.j2') | to_nice_json | b64encode }}" - when: (icr_username is defined and icr_username != "") or (artifactory_username is defined and artifactory_username != "") - -# 4. Patch the default service account for pre-release build access -# ----------------------------------------------------------------------------- -# If we don't do this then we won't be able to pull the operator image from -# Artifactory -- name: "Patch default SA" - when: - - artifactory_username != "" - - artifactory_token != "" - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - namespace: "{{ namespace }}" - imagePullSecrets: - - name: ibm-entitlement - -# 5. Create the operator group that will scope the operator -# ----------------------------------------------------------------------------- -- name: "Create operator group" - kubernetes.core.k8s: - definition: "{{ operator_group }}" - wait: yes - wait_timeout: 60 # subsequent tasks will fail if the group isn't fully created - -# 6. Create the subscription for the operator -# ----------------------------------------------------------------------------- -- name: "Create subscription" - kubernetes.core.k8s: - definition: "{{ subscription }}" - wait: yes - wait_timeout: 300 - wait_condition: - type: "CatalogSourcesUnhealthy" - status: "False" diff --git a/ibm/mas_devops/roles/install_operator/templates/custom_labels.json.j2 b/ibm/mas_devops/roles/install_operator/templates/custom_labels.json.j2 deleted file mode 100644 index 0ed9e4f9bd..0000000000 --- a/ibm/mas_devops/roles/install_operator/templates/custom_labels.json.j2 +++ /dev/null @@ -1,7 +0,0 @@ -metadata: -{% if custom_labels is defined and custom_labels.items() %} - labels: -{% for key, value in custom_labels.items() %} - "{{ key }}": "{{ value }}" -{% endfor %} -{% endif %} \ No newline at end of file diff --git a/ibm/mas_devops/roles/install_operator/templates/ibm-entitlement-with-artifactory.json.j2 b/ibm/mas_devops/roles/install_operator/templates/ibm-entitlement-with-artifactory.json.j2 deleted file mode 100644 index 0c95133738..0000000000 --- a/ibm/mas_devops/roles/install_operator/templates/ibm-entitlement-with-artifactory.json.j2 +++ /dev/null @@ -1,32 +0,0 @@ -{ - "auths": { -{% if artifactory_username is defined and artifactory_username != "" %} - "docker-na-public.artifactory.swg-devops.com/wiotp-docker-local": { - "username": "{{ artifactory_username }}", - "password": "{{ artifactory_token }}", - "auth": "{{ (artifactory_username ~ ':' ~ artifactory_token) | b64encode }}" - }, - "docker-na-proxy-svl.artifactory.swg-devops.com/wiotp-docker-local": { - "username": "{{ artifactory_username }}", - "password": "{{ artifactory_token }}", - "auth": "{{ (artifactory_username ~ ':' ~ artifactory_token) | b64encode }}" - }, - "docker-na-proxy-rtp.artifactory.swg-devops.com/wiotp-docker-local": { - "username": "{{ artifactory_username }}", - "password": "{{ artifactory_token }}", - "auth": "{{ (artifactory_username ~ ':' ~ artifactory_token) | b64encode }}" - {% if icr_username is defined and icr_username != "" %} - }, - {% else %} - } - {% endif %} -{% endif %} -{% if icr_username is defined and icr_username != "" %} - "cp.icr.io/cp": { - "username": "{{ icr_username }}", - "password": "{{ icr_password }}", - "auth": "{{ (icr_username ~ ':' ~ icr_password) | b64encode }}" - } -{% endif %} - } -} diff --git a/ibm/mas_devops/roles/kafka/README.md b/ibm/mas_devops/roles/kafka/README.md index 4a523c051f..37b4a3ac1e 100644 --- a/ibm/mas_devops/roles/kafka/README.md +++ b/ibm/mas_devops/roles/kafka/README.md @@ -58,7 +58,7 @@ The configuration to apply, there are two configurations available: small and la - Default Value: `small` ### kafka_storage_class -The name of the storage class to configure the AMQStreams operator to use for persistent storage in the Kafka cluster. +The name of the storage class to configure the AMQStreams operator to use for persistent storage in the Kafka cluster. Storage class must support ReadWriteOnce(RWO) access mode. - Environment Variable: `KAFKA_STORAGE_CLASS` - Default Value: lookup supported storage classes in the cluster @@ -70,7 +70,7 @@ The size of the storage to configure the AMQStreams operator to use for persiste - Default Value: `100Gi` ### zookeeper_storage_class -The name of the storage class to configure the AMQStreams operator to use for persistent storage in the Zookeeper cluster. +The name of the storage class to configure the AMQStreams operator to use for persistent storage in the Zookeeper cluster. Storage class must support ReadWriteOnce(RWO) access mode. - Environment Variable: `ZOOKEEPER_STORAGE_CLASS` - Default Value: lookup supported storage classes in the cluster diff --git a/ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml index efaed67745..773a766863 100644 --- a/ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml @@ -1,21 +1,8 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Set Storage (Required) @@ -23,7 +10,7 @@ - name: Default Kafka Storage for ROKS if not set by user when: kafka_storage_class is not defined or kafka_storage_class == "" set_fact: - kafka_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + kafka_storage_class: "{{ defaultStorageClasses.rwo }}" - name: Assert that kafka storage class has been defined assert: @@ -33,17 +20,18 @@ - name: Default Zookeeper Storage for ROKS if not set by user when: zookeeper_storage_class is not defined or zookeeper_storage_class == "" set_fact: - zookeeper_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + zookeeper_storage_class: "{{ defaultStorageClasses.rwo }}" - name: Assert that zookeeper storage class has been defined assert: that: zookeeper_storage_class is defined and zookeeper_storage_class != "" fail_msg: "zookeeper_storage_class must be defined" + # 3. Debug storage class configuration # ----------------------------------------------------------------------------- - name: "Debug Kafka storage classes configuration" debug: msg: - - "Storage class (kafka) ...... {{ kafka_storage_class }}" - - "Storage class (zookeeper) .. {{ zookeeper_storage_class }}" + - "Storage class (kafka) .................. {{ kafka_storage_class }}" + - "Storage class (zookeeper) .............. {{ zookeeper_storage_class }}" diff --git a/ibm/mas_devops/roles/kafka/templates/redhat/clusters/large.yml.j2 b/ibm/mas_devops/roles/kafka/templates/redhat/clusters/large.yml.j2 index ebd1da125e..e46076e068 100644 --- a/ibm/mas_devops/roles/kafka/templates/redhat/clusters/large.yml.j2 +++ b/ibm/mas_devops/roles/kafka/templates/redhat/clusters/large.yml.j2 @@ -176,3 +176,8 @@ spec: memory: 128Mi logging: debug enableSaramaLogging: true + # ------------------------------------------------------- + clusterCa: + renewalDays: 30 + validityDays: 7200 + generateCertificateAuthority: true diff --git a/ibm/mas_devops/roles/kafka/templates/redhat/clusters/small.yml.j2 b/ibm/mas_devops/roles/kafka/templates/redhat/clusters/small.yml.j2 index d1b71d5f9a..6a51631fa4 100644 --- a/ibm/mas_devops/roles/kafka/templates/redhat/clusters/small.yml.j2 +++ b/ibm/mas_devops/roles/kafka/templates/redhat/clusters/small.yml.j2 @@ -176,3 +176,8 @@ spec: memory: 128Mi logging: debug enableSaramaLogging: true + # ------------------------------------------------------- + clusterCa: + renewalDays: 30 + validityDays: 7200 + generateCertificateAuthority: true diff --git a/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml index 5ab4d02752..1bfe8994a2 100644 --- a/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml @@ -1,24 +1,16 @@ -# Lookup storage class availabiity +--- +# 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" -- name: "determine-storage-classes : Lookup storage classes" - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" - -# Set Primary Storage (Required) +# 2. Set Primary Storage (Required) # ----------------------------------------------------------------------------- - name: "determine-storage-classes : Default Primary Storage if not set by user" when: cpd_primary_storage_class is not defined or cpd_primary_storage_class == "" set_fact: - cpd_primary_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" + cpd_primary_storage_class: "{{ defaultStorageClasses.rwx }}" - name: "determine-storage-classes : Assert that primary storage class has been defined" assert: @@ -27,4 +19,4 @@ - name: Set primary StorageClass debug: - msg: "Primary storage class ...... {{ cpd_primary_storage_class }}" + msg: "Primary storage class ................. {{ cpd_primary_storage_class }}" diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/amlen_1.1.2.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/amlen_1.1.2.yml new file mode 100644 index 0000000000..fb707b9753 --- /dev/null +++ b/ibm/mas_devops/roles/mirror_extras_prepare/vars/amlen_1.1.2.yml @@ -0,0 +1,16 @@ +--- +extra_images: + - name: amlen/operator-bundle + registry: quay.io + digest: sha256:9242f268798e522e5be5035f5f68dba1677560939d7464896fe71e8bc32be612 + tag: 1.1.2 + + - name: amlen/operator + registry: quay.io + digest: sha256:bba8f0fa2767f2aa3a308e1d045b9dcfd1a87418471de9a59140d14593428d98 + tag: 1.1.2 + + - name: kubebuilder/kube-rbac-proxy + registry: gcr.io + digest: sha256:0df4ae70e3bd0feffcec8f5cdb428f4abe666b667af991269ec5cb0bbda65869 + tag: 1.1.2 diff --git a/ibm/mas_devops/roles/mirror_ocp/templates/imagesetconfiguration.yml.j2 b/ibm/mas_devops/roles/mirror_ocp/templates/imagesetconfiguration.yml.j2 index 9fb11c4313..438e451ede 100644 --- a/ibm/mas_devops/roles/mirror_ocp/templates/imagesetconfiguration.yml.j2 +++ b/ibm/mas_devops/roles/mirror_ocp/templates/imagesetconfiguration.yml.j2 @@ -27,10 +27,10 @@ mirror: - name: gpu-operator-certified # Required by ibm.mas_devops.nvidia_gpu role channels: - name: v23.3 - # We don't use the v24.6 channel, but oc-mirror fails when the default channel is not included + # We don't use the v24.9 channel, but oc-mirror fails when the default channel is not included # - https://access.redhat.com/solutions/7013461 # - https://issues.redhat.com/browse/OCPBUGS-385 - - name: v24.6 + - name: v24.9 - name: kubeturbo-certified # Required by ibm.mas_devops.kubeturbo role channels: - name: stable diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index 9f22df63b0..7a6023c6f5 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -97,7 +97,7 @@ List of preserved settings - mongodb_replicas ### mongodb_storage_class -Required. The name of the storage class to configure the MongoDb operator to use for persistent storage in the MongoDb cluster. +**Required**: The name of the storage class to configure the MongoDb operator to use for persistent storage in the MongoDb cluster. Storage class must support ReadWriteOnce(RWO) access mode. - Environment Variable: `MONGODB_STORAGE_CLASS` - Default Value: None @@ -192,82 +192,13 @@ Set the [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - Environment Variable: `MASBR_JOB_TIMEZONE` - Default: None -### masbr_storage_type -Set `local` or `cloud` to indicate this role to save the backup files to local file system or cloud object storage. - -- **Required** -- Environment Variable: `MASBR_STORAGE_TYPE` -- Default: None - ### masbr_storage_local_folder Set local path to save the backup files. -- **Required** only when `MASBR_STORAGE_TYPE=local` +- **Required** - Environment Variable: `MASBR_STORAGE_LOCAL_FOLDER` - Default: None -### masbr_storage_cloud_rclone_file -Set the path of `rclone.conf` file. - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_RCLONE_FILE` -- Default: None - -### masbr_storage_cloud_rclone_name -Set the configuration name defined in `rclone.conf` file. - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_RCLONE_NAME` -- Default: None - -### masbr_storage_cloud_bucket -Set the object storage bucket name for saving the backup files - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_BUCKET` -- Default: None - -### masbr_slack_enabled -Set `true` or `false` to indicate whether this role will send Slack notification messages of the backup and restore progress. - -- Optional -- Environment Variable: `MASBR_SLACK_ENABLED` -- Default: `false` - -### masbr_slack_level -Set `failure`, `info` or `verbose` to indicate this role to send Slack notification messages in which backup and resore phases: - -| Slack level | Backup/Restore phases | -| ----------- | ------------------------------------------------------- | -| failure | `Failed`, `PartiallyFailed` | -| info | `Completed`, `Failed`, `PartiallyFailed` | -| verbose | `InProgress`, `Completed`, `Failed`, `PartiallyFailed` | - -- Optional -- Environment Variable: `MASBR_SLACK_LEVEL` -- Default: `info` - -### masbr_slack_token -The Slack integration token. - -- **Required** only when `MASBR_SLACK_ENABLED=true` -- Environment Variable: `MASBR_SLACK_TOKEN` -- Default: None - -### masbr_slack_channel -The Slack channel to send the notification messages to. - -- **Required** only when `MASBR_SLACK_ENABLED=true` -- Environment Variable: `MASBR_SLACK_CHANNEL` -- Default: None - -### masbr_slack_user -The sender of the Slack notification message. - -- Optional -- Environment Variable: `MASBR_SLACK_USER` -- Default: `MASBR` - ### masbr_backup_type Set `full` or `incr` to indicate the role to create a full backup or incremental backup. @@ -602,7 +533,6 @@ Example Playbooks vars: mongodb_action: backup mas_instance_id: masinst1 - masbr_storage_type: local masbr_storage_local_folder: /tmp/masbr roles: - ibm.mas_devops.mongodb @@ -616,7 +546,6 @@ Example Playbooks mongodb_action: restore mas_instance_id: masinst1 masbr_restore_from_version: 20240621021316 - masbr_storage_type: local masbr_storage_local_folder: /tmp/masbr roles: - ibm.mas_devops.mongodb @@ -745,7 +674,7 @@ If `avx` is not found in the available `flags` then either the physical processo ### LDAP Authentication -If authenticating via LDAP with PLAIN specified for `authMechanism` then `configDb` must be set to `$external` in the MongoCfg. The field `configDb` in the MongoCfg refers to the authentication database. +If authenticating via LDAP with PLAIN specified for `authMechanism` then `configDb` must be set to `$external` in the MongoCfg. The field `configDb` in the MongoCfg refers to the authentication database. ### CA Certificate Renewal diff --git a/ibm/mas_devops/roles/mongodb/tasks/determine-ibmcatalog-tag.yml b/ibm/mas_devops/roles/mongodb/tasks/determine-ibmcatalog-tag.yml index ce99d8b69a..ebcc2dd797 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/determine-ibmcatalog-tag.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/determine-ibmcatalog-tag.yml @@ -2,49 +2,38 @@ # The MongoDB version is based on the chosen ibm-operator-catalog. However the # ibm-operator-catalog does not explicitly include information about the mongo version. # -# The configuration files in common_vars/casebundles do specify the MongoDB -# version for each of the MAS published ibm-operator-catalog catalogs. -# -# To determine which common_vars/casebundles configuration file to use, this role +# To determine which catalog metadata to use, this role # extracts the catalog_tag from the ibm-operator-catalog's displayName. # # For example: # displayName: IBM Maximo Operators (v9-240625-amd64) # # catalog_tag is v9-240625-amd64 -# MongoDB version will be determined by loading common_vars/casebundles/v9-240625-amd64.yml +# MongoDB version will be determined from the catalog metadata (get_catalog_info.py) # # And FINALLY... # If the MongoDB version cannot be determined from the above logic, -# the MongoDB version will be determined by the last configuration file in -# common_vars/casebundles/ +# the MongoDB version will be determined by the most recent catalog data +# available (get_newest_catalog_tag.py) # # And if still it cannot be determined because perhaps its not specified in the # configuration file, a default value will be used. -# List the common_vars/casebundle configuration files if last_catalog_tag is not defined -- block: - - name: "List yml files in common_vars/casebundles folder" - find: - paths: "{{ role_path }}/../../common_vars/casebundles" - patterns: "*.yml" - file_type: "file" - register: find_result - - - debug: - var: find_result - - # Determine the last configuration file in the list just in case we need it later - - set_fact: - last_catalog_tag: "{{ find_result['files'] | map(attribute='path') | map('regex_replace', '^.*/(.*).yml$', '\\1') | sort |last }}" - when: find_result is defined +- name: "Lookup the latest version of the Maximo Operator Catalog" when: last_catalog_tag is not defined or last_catalog_tag == "" + block: + - name: "Get newest operator catalog tag" + ibm.mas_devops.get_newest_catalog_tag: + arch: amd64 + register: _newest_catalog + - name: "Store tag in last_catalog_tag" + set_fact: + last_catalog_tag: "{{ _newest_catalog.result }}" -# Display the Last Catalog Version - name: "Display the Last Catalog Version" debug: - msg: - - "Last Catalog Version ............................ {{ last_catalog_tag }}" + msg: "Last Catalog Version ................... {{ last_catalog_tag | default('') }}" + # 1. Get the IBM Catalog if available # ----------------------------------------------------------------------------- @@ -57,49 +46,32 @@ register: catalog_lookup - name: "Determine catalog version from catalog displayName" - block: - # extract the catalog tag from displayName - - set_fact: - catalog_tag: "{{ catalog_lookup.resources[0].spec.displayName.split('(')[1].split(')')[0].split(' ')[0] }}" - when: - - catalog_lookup is defined - - catalog_lookup.resources is defined - - catalog_lookup.resources | length > 0 - - catalog_lookup.resources[0].spec is defined - - catalog_lookup.resources[0].spec.displayName is defined - - '"(" in catalog_lookup.resources[0].spec.displayName' - - - name: Check if file exists in casebundles - stat: - path: "{{ role_path }}/../../common_vars/casebundles/{{ catalog_tag }}.yml" - register: file_exists_result - when: catalog_tag is defined and catalog_tag != "" + when: + - catalog_lookup is defined + - catalog_lookup.resources is defined + - catalog_lookup.resources | length > 0 + - catalog_lookup.resources[0].spec is defined + - catalog_lookup.resources[0].spec.displayName is defined + - '"(" in catalog_lookup.resources[0].spec.displayName' + set_fact: + catalog_tag: "{{ catalog_lookup.resources[0].spec.displayName.split('(')[1].split(')')[0].split(' ')[0] }}" - # use last_catalog_tag if the casebundles file does not exist - - set_fact: - catalog_tag: "{{ last_catalog_tag }}" - when: - - file_exists_result is defined - - file_exists_result.stat is defined - - file_exists_result.stat.exists is defined - - not file_exists_result.stat.exists - - last_catalog_tag is defined and last_catalog_tag != "" +- name: "Lookup catalog metadata" + when: catalog_tag is defined and catalog_tag != "" + ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: catalog_metadata - # use last_catalog_tag if unable to get catalog_tag from displayName - - set_fact: - catalog_tag: "{{ last_catalog_tag }}" - when: - - catalog_tag is not defined - - last_catalog_tag is defined and last_catalog_tag != "" - rescue: - # use the last_catalog_tag when the catalog_tag cannot be determined from the displayName - - set_fact: - catalog_tag: "{{ last_catalog_tag }}" - when: last_catalog_tag is defined and last_catalog_tag != "" +# 2. Fallback to the most recent catalog tag +# ----------------------------------------------------------------------------- +- name: "Default to the newest catalog metadata" + when: + - catalog_tag is not defined or catalog_tag == "" or not catalog_metadata.success + - last_catalog_tag is defined and last_catalog_tag != "" + set_fact: + catalog_tag: "{{ last_catalog_tag }}" -- name: "Catalog Version" +- name: "Catalog Version (Final)" debug: - msg: - - "Catalog Version ............................ {{ catalog_tag }}" - when: catalog_tag is defined and catalog_tag != "" + msg: "Catalog Version ........................ {{ catalog_tag | default('')}}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/after-backup-restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/after-backup-restore.yml similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/after-backup-restore.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/after-backup-restore.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml similarity index 98% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-database.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index 04a85787a7..d2504bc3d5 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -14,7 +14,7 @@ # Create mongodb role and user for backing up databases # ------------------------------------------------------------------------- - name: "Create mongodb role and user for backing up databases" - include_tasks: "tasks/providers/{{ mongodb_provider }}/create-role-user.yml" + include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/create-role-user.yml" # Prepare mongodb database backup folder @@ -96,6 +96,7 @@ }))" | tee -a {{ mongodb_backup_log }} {{ exec_in_pod_end }} register: _mongodb_db_names_output + no_log: true - name: "Set fact: database names" set_fact: @@ -132,6 +133,7 @@ {{ exec_in_pod_end }} loop: "{{ mongodb_db_names }}" register: _mongodump_output + no_log: true - name: "Debug: mongodump output" debug: @@ -191,6 +193,7 @@ |& tee -a {{ mongodb_backup_log }} {{ exec_in_pod_end }} register: _mongodump_output + no_log: true - name: "Debug: mongodump output" debug: @@ -254,7 +257,7 @@ content: "{{ mongodb_query | from_yaml | to_json }}" mode: preserve vars: - mongodb_query: "{{ lookup('template', '../../../templates/community/mongo-query.yml.j2') }}" + mongodb_query: "{{ lookup('template', 'templates/community/mongo-query.yml.j2') }}" - name: "Get query file content" changed_when: false diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/before-backup-restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml similarity index 98% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/before-backup-restore.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml index 28e69b025d..8911d64430 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/before-backup-restore.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml @@ -38,6 +38,7 @@ name: mas-mongo-ce-admin-password namespace: "{{ mongodb_namespace }}" register: _mongodb_password_output + no_log: true - name: "Set fact: mongodb admin password" set_fact: @@ -46,6 +47,7 @@ - _mongodb_password_output is defined - _mongodb_password_output.resources[0] is defined - _mongodb_password_output.resources[0].data.password is defined + no_log: true # Get mongodb ca file location @@ -72,12 +74,13 @@ changed_when: false shell: > {{ exec_in_pod_begin }} - {{ mongodb_shell }} --quiet --host={{ mongodb_pod_name }}.mas-mongo-ce-svc.mongoce.svc.cluster.local:27017 + {{ mongodb_shell }} --quiet --host={{ mongodb_pod_name }}.mas-mongo-ce-svc.{{ mongodb_namespace }}.svc.cluster.local:27017 --username=admin --password={{ mongodb_password }} --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin --eval="print(JSON.stringify(db.runCommand({hello:1})))" {{ exec_in_pod_end }} register: _mongodb_info_output + no_log: true - name: "Set fact: mongodb primary host" set_fact: diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/create-role-user.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml similarity index 95% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/create-role-user.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml index bf5044416d..c6f7379ffa 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/create-role-user.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml @@ -15,6 +15,7 @@ --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin --eval="db.getRole( \"{{ mongodb_role }}\" )"' register: _mongodb_get_role_output + no_log: true - name: "Debug: get mongodb role result" debug: @@ -31,6 +32,7 @@ --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin --eval="db.createRole({ role: \"{{ mongodb_role }}\", roles: [], privileges: [{ resource: {anyResource: true}, actions: [\"anyAction\"] }]})"' register: _mongodb_create_role_output + no_log: true - name: "Debug: create mongodb role result" debug: @@ -47,10 +49,7 @@ --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin --eval="db.getUser( \"{{ mongodb_user }}\" )"' register: _mongodb_get_user_output - -- name: "Debug: get mongodb user result" - debug: - msg: "Get mongodb user result ............... {{ _mongodb_get_user_output.stdout_lines }}" + no_log: true - name: "Create mongodb user '{{ mongodb_user }}'" when: _mongodb_get_user_output.stdout_lines[-1] == "null" @@ -63,6 +62,7 @@ --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin --eval="db.createUser({ user: \"{{ mongodb_user }}\", pwd: \"{{ mongodb_password }}\", roles: [ \"{{ mongodb_role }}\" ]})"' register: _mongodb_create_user_output + no_log: true - name: "Debug: create mongodb user result" debug: diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/get-mongo-info.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/get-mongo-info.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database-patch.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database-patch.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database-perform.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml similarity index 98% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database-perform.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml index 04724a3fa3..5651e725c2 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database-perform.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml @@ -43,7 +43,7 @@ {{ exec_in_pod_begin }} mkdir -p {{ mongodb_restore_folder }}/{{ _job_name }} && tar -xzf {{ mongodb_restore_folder }}/{{ _job_name }}.tar.gz - -C {{ mongodb_restore_folder }}/{{ _job_name }} . && + -C {{ mongodb_restore_folder }}/{{ _job_name }} && ls -lA {{ mongodb_restore_folder }}/{{ _job_name }} {{ exec_in_pod_end }} register: _extract_output @@ -100,6 +100,7 @@ |& tee -a {{ mongodb_restore_log }} {{ exec_in_pod_end }} register: _mongorestore_output + no_log: true - name: "Debug: mongorestore output" debug: diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml similarity index 95% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 52251a21c6..46941268bd 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -14,7 +14,7 @@ # Create mongodb role and user for backing up databases # ------------------------------------------------------------------------- - name: "Create mongodb role and user for backing up databases" - include_tasks: "tasks/providers/{{ mongodb_provider }}/create-role-user.yml" + include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/create-role-user.yml" # Prepare mongodb database restore folders @@ -48,7 +48,7 @@ when: masbr_restore_from_incr block: - name: "Copy based on full backup file from specified storage location to pod" - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/restore-database-perform.yml" + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database-perform.yml" vars: _job_type: "full" _job_name: "{{ masbr_restore_basedon }}" @@ -57,7 +57,7 @@ # Restore databases from the specified Full or Incremental backup # ------------------------------------------------------------------------- - name: "Restore databases from the specified {{ 'Incremental' if masbr_restore_from_incr else 'Full' }} backup" - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/restore-database-perform.yml" + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database-perform.yml" vars: _job_type: "{{ 'incr' if masbr_restore_from_incr else 'full' }}" _job_name: "{{ masbr_restore_from }}" @@ -66,7 +66,7 @@ # Do some post restoration tasks # ------------------------------------------------------------------------- - name: "Do some post restoration tasks " - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/restore-database-patch.yml" + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database-patch.yml" # Update database restore status: Completed diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index 59f137279e..fd50a9696e 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -10,7 +10,7 @@ # Get mongodb information # ------------------------------------------------------------------------- - name: "Get mongodb information" - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/get-mongo-info.yml" + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/get-mongo-info.yml" # Set common backup job variables @@ -36,31 +36,10 @@ vars: _ignore_masbr_backup_data: true _job_type: "backup" - _component_before_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/before-backup-restore.yml" + _component_before_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/before-backup-restore.yml" -# Create k8s Job to run backup tasks -# ----------------------------------------------------------------------------- -- name: "Create k8s Job to run backup tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" - vars: - _rt_role_name: "mongodb" - _rt_env: - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MONGODB_PROVIDER" - value: "{{ mongodb_provider }}" - - name: "MONGODB_ACTION" - value: "{{ mongodb_action }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_APP_ID" - value: "{{ mas_app_id }}" - - -- name: "Run backup tasks" - when: not masbr_create_task_job +- name: "Perform backup" block: # Update backup job status: New # ------------------------------------------------------------------------- @@ -71,11 +50,10 @@ - seq: "1" phase: "New" - # Run backup tasks for each data type # ------------------------------------------------------------------------- - name: "Run backup tasks for each data type" - include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-{{ job_data_item.type }}.yml" + include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/backup-{{ job_data_item.type }}.yml" vars: masbr_job_data_seq: "{{ job_data_item.seq }}" masbr_job_data_type: "{{ job_data_item.type }}" @@ -98,4 +76,4 @@ - name: "After run tasks" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/after_run_tasks.yml" vars: - _component_after_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/after-backup-restore.yml" + _component_after_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/after-backup-restore.yml" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/determine-storage-classes.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/determine-storage-classes.yml index e66d41797d..54d6c7103e 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/determine-storage-classes.yml @@ -1,21 +1,11 @@ --- - # Provide intelligent storage class selection to minimize required user knowledge + # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- - name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Set Storage (Required) @@ -23,7 +13,7 @@ - name: Default Storage if not set by user when: mongodb_storage_class is not defined or mongodb_storage_class == "" set_fact: - mongodb_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + mongodb_storage_class: "{{ defaultStorageClasses.rwo }}" - name: Assert that storage class has been defined assert: @@ -36,4 +26,4 @@ - name: "Debug MongoDb storage class configuration" debug: msg: - - "Storage class (mongodb) ...... {{ mongodb_storage_class }}" + - "Storage class (mongodb) ................ {{ mongodb_storage_class }}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml index 770e2b80e2..3a11c92d7c 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml @@ -1,53 +1,27 @@ --- -# The MongoDB version is based on the chosen ibm-operator-catalog. However the -# ibm-operator-catalog does not explicitly include information about the mongo version. -# -# The configuration files in common_vars/casebundles do specify the MongoDB -# version for each of the MAS published ibm-operator-catalog catalogs. -# -# To determine which common_vars/casebundles configuration file to use, this role -# extracts the catalog_tag from the ibm-operator-catalog's displayName. -# -# For example: -# displayName: IBM Maximo Operators (v9-240625-amd64) -# -# catalog_tag is v9-240625-amd64 -# MongoDB version will be determined by loading common_vars/casebundles/v9-240625-amd64.yml -# -# And FINALLY... -# If the MongoDB version cannot be determined from the above logic, -# the MongoDB version will be determined by the last configuration file in -# common_vars/casebundles/ -# -# And if still it cannot be determined because perhaps its not specified in the -# configuration file, a default value will be used. - -# 1. Get the IBM Catalog if available +# 1. Load default settings # ----------------------------------------------------------------------------- +- name: "Determine Version of Maximo Operator Catalog" + include_tasks: tasks/determine-ibmcatalog-tag.yml -- include_tasks: tasks/determine-ibmcatalog-tag.yml - -# 2. Load default settings -# ----------------------------------------------------------------------------- -- name: Check again if file exists in casebundles - stat: - path: "{{ role_path }}/../../common_vars/casebundles/{{ catalog_tag }}.yml" - register: stat_result - when: catalog_tag is defined and catalog_tag != "" - -- name: Load CASE bundle versions - include_vars: - file: "{{ role_path }}/../../common_vars/casebundles/{{ catalog_tag }}.yml" - when: stat_result is defined and stat_result.stat.exists - -- name: Set mongo version to default value when no catalog - set_fact: - mongo_extras_version: "{{ mongo_extras_version_default }}" - when: mongo_extras_version is not defined +- name: Load Catalog Metadata + when: + - mongo_extras_version is not defined or mongo_extras_version == "" + - catalog_tag is defined and catalog_tag != "" + block: + - ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: _mas_catalog + - debug: + var: _mas_catalog + - name: "Set mongo version to default value if not explicitly set" + set_fact: + mongo_extras_version: "{{ _mas_catalog.mongo_extras_version_default }}" - name: "Mongo Version" debug: msg: + - "Maximo Operator Catalog .................. {{ catalog_tag }}" - "Mongo Version ............................ {{ mongo_extras_version }}" - name: Load mongo defaults @@ -84,7 +58,8 @@ target_mongodb_readinessprobe_image: "{{ mongodb_readinessprobe_image_registry }}/{{ mongodb_readinessprobe_image_name }}@{{ mongodb_readinessprobe_image_digest }}" target_mongodb_image: "{{ mongodb_image_registry }}/{{ mongodb_image_name }}@{{ mongodb_image_digest }}" -# 3. Debug Properties + +# 2. Debug Properties # ----------------------------------------------------------------------------- - name: "community : install : Debug properties" debug: @@ -125,21 +100,24 @@ - existing_mongo_operator.resources[0] is defined - existing_mongo_operator.resources[0].spec.containers[0].image != target_mongodb_operator_image -# 5. Install the CRD + +# 3. Install the CRD # ----------------------------------------------------------------------------- - name: "community : install : Install MongoDBCommunity CRD" kubernetes.core.k8s: apply: yes definition: "{{ lookup('template', 'templates/community/{{ mongodb_ce_version }}/crd.yml.j2') }}" -# 6. Create namespace & install RBAC + +# 4. Create namespace & install RBAC # ----------------------------------------------------------------------------- - name: "community : install : Create namespace & install RBAC" kubernetes.core.k8s: apply: yes definition: "{{ lookup('template', 'templates/community/{{ mongodb_ce_version }}/rbac.yml.j2') }}" -# 7. Configure anyuid permissions in the MongoDb namespace + +# 5. Configure anyuid permissions in the MongoDb namespace # ----------------------------------------------------------------------------- - name: "community : install : Configure anyuid permissions in the MongoDb namespace" shell: | @@ -151,7 +129,8 @@ shell: | oc adm policy add-scc-to-user anyuid system:serviceaccount:{{ mongodb_namespace }}:mongodb-database -# 8. Install the operator + +# 6. Install the operator # ----------------------------------------------------------------------------- - name: "community : install : Install the MongoDb Operator" kubernetes.core.k8s: @@ -162,7 +141,8 @@ status: "True" type: Available -# 9. Configure TLS using cert manager + +# 7. Configure TLS using cert manager # ----------------------------------------------------------------------------- - name: "Create a issuer in '{{ mongodb_namespace }}' namespace" kubernetes.core.k8s: @@ -230,7 +210,8 @@ apply: yes definition: "{{ lookup('template', 'templates/community/tls.yml') }}" -# 10. Create MongoDb admin user credentials secret + +# 8. Create MongoDb admin user credentials secret # ----------------------------------------------------------------------------- - name: "community : install : Check for existing user password secret" kubernetes.core.k8s_info: @@ -248,7 +229,8 @@ apply: yes definition: "{{ lookup('template', 'templates/community/admin-password.yml') }}" -# 10.1 Create MongoDb metrics endpoint secret + +# 9. Create MongoDb metrics endpoint secret # ----------------------------------------------------------------------------- - name: "community : install : Check for existing metrics endpoint secret" kubernetes.core.k8s_info: @@ -269,7 +251,8 @@ apply: yes definition: "{{ lookup('template', 'templates/community/metrics-endpoint-secret.yml.j2') }}" -# 11. Deploy the cluster + +# 10. Deploy the cluster # ----------------------------------------------------------------------------- - name: "community : install : Install new MongoDB cluster CR" kubernetes.core.k8s: @@ -314,7 +297,8 @@ - not controlled_upgrade - mongodb_ce_version is version('0.7.0', '==') -# 12. Wait for the cluster to be ready + +# 11. Wait for the cluster to be ready # status.version on the mongo cr is not available with older mongo operators # ----------------------------------------------------------------------------- - name: Delete existing mongo statefulsets if running mongo upgrade @@ -404,7 +388,8 @@ - (mongodb_ce_version is version('0.7.8', '>=') and mongodb_cr.resources[0].status.version is defined) or mongodb_ce_version is version('0.7.0', '==') - (mongodb_ce_version is version('0.7.8', '>=') and mongodb_cr.resources[0].status.version == expected_mongodb_version) or mongodb_ce_version is version('0.7.0', '==') -# 12.1 Create MongoDb service monitor + +# 12. Create MongoDb service monitor # ----------------------------------------------------------------------------- - name: "community : install : Check for existing service monitor" kubernetes.core.k8s_info: @@ -425,7 +410,8 @@ apply: yes definition: "{{ lookup('template', 'templates/community/servicemonitor.yml.j2') }}" -# 12.2 Create MongoDb Grafana dashboard + +# 13. Create MongoDb Grafana dashboard # ----------------------------------------------------------------------------- - name: Get cluster info kubernetes.core.k8s_cluster_info: @@ -491,7 +477,8 @@ apply: yes definition: "{{ lookup('template', 'templates/community/dashboards/mongodb-overview-v5.yml.j2') }}" -# 13. Create MAS MongoCfg + +# 14. Create MAS MongoCfg # ----------------------------------------------------------------------------- # Lookup the mongo pods - name: Create the MAS MongoCfg only if we are not in a controlled-upgrade flow diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml index fcab35e096..38800a9eff 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml @@ -14,7 +14,7 @@ # Get mongodb information # ------------------------------------------------------------------------- - name: "Get mongodb information" - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/get-mongo-info.yml" + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/get-mongo-info.yml" # Set common restore job variables @@ -38,31 +38,10 @@ include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" vars: _job_type: "restore" - _component_before_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/before-backup-restore.yml" + _component_before_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/before-backup-restore.yml" -# Create k8s Job to run restore tasks -# ----------------------------------------------------------------------------- -- name: "Create k8s Job to run restore tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" - vars: - _rt_role_name: "mongodb" - _rt_env: - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MONGODB_PROVIDER" - value: "{{ mongodb_provider }}" - - name: "MONGODB_ACTION" - value: "{{ mongodb_action }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_APP_ID" - value: "{{ mas_app_id }}" - - -- name: "Run restore tasks" - when: not masbr_create_task_job +- name: "Perform restore" block: # Update restore job status: New # ------------------------------------------------------------------------- @@ -73,11 +52,10 @@ - seq: "1" phase: "New" - # Run restore tasks for each data type # ------------------------------------------------------------------------- - name: "Run restore tasks for each data type" - include_tasks: "tasks/providers/{{ mongodb_provider }}/restore-{{ job_data_item.type }}.yml" + include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/restore-{{ job_data_item.type }}.yml" vars: masbr_job_data_seq: "{{ job_data_item.seq }}" masbr_job_data_type: "{{ job_data_item.type }}" @@ -100,4 +78,4 @@ - name: "After run tasks" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/after_run_tasks.yml" vars: - _component_after_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/after-backup-restore.yml" + _component_after_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/after-backup-restore.yml" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/validate-upgrade.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/validate-upgrade.yml index 7f6bb7fcdb..c46b492e38 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/validate-upgrade.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/validate-upgrade.yml @@ -24,6 +24,16 @@ that: "existing_mongo_version is version_compare(target_mongodb_version, '<=')" fail_msg: "Mongo downgrade is not allowed! Current mongo version: {{ existing_mongo_version }}, target mongo version {{ target_mongodb_version }}" +# For users installed or upgraded to mongo v4.4 using old CLI didn't have featureCompatibilityVersion set correctly. +- name: Assert featureCompatibilityVersion is present and equal to existing version. + assert: + that: + - existing_mongodb.resources[0].spec.featureCompatibilityVersion is defined and existing_mongodb.resources[0].spec.featureCompatibilityVersion != "" + - existing_mongodb.resources[0].spec.featureCompatibilityVersion == existing_mongo_minor_version + fail_msg: "Mongo upgrade is not allowed due to either missing featureCompatibilityVersion field or wrong version in featureCompatibilityVersion field in MongoDBCommunity CR" + when: mongodb_v5_upgrade or mongodb_v6_upgrade or mongodb_v7_upgrade + + # Only allow Mongo upgrades to be next compatible minor version - name: "Assert Mongo upgrade compatibility: {{ target_mongodb_version }} must be in {{ mongo_compatible_target_version[existing_mongo_version] }}" when: existing_mongo_minor_version != '4.2' # will skip this assertion if mongo v4.2 as we'll assist in the auto upgrade to v4.4 and v5 in a single run diff --git a/ibm/mas_devops/roles/nvidia_gpu/README.md b/ibm/mas_devops/roles/nvidia_gpu/README.md index d8aeb42520..07e6afc8e9 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/README.md +++ b/ibm/mas_devops/roles/nvidia_gpu/README.md @@ -29,25 +29,17 @@ The namespace where the NVIDIA GPU operator will be deployed. For version 1.8.x, The channel to subscribe to for the gpu operator installation and updates. Available channels may be found in the package manifest of gpu-operator-certified operator in openshift. - Environment Variable: `GPU_CHANNEL` -- Default Value: `v23.3` +- Default Value: `v24.9` ### gpu_driver_version -The gpu driver version image that needs to be pulled from the gpu driver repository. It is recommended that the right version of GPU driver is used depending on the OS version. The default versions are shown below. See the attached links for more information and to decide which driver version to use. +By default, it will pull the latest version and the environment variable is not needed. +If a specific version is needed (due to OS version compatibilities), specify the following environment variable. - Environment Variable: `GPU_DRIVER_VERSION` -- Default Value: `470.182.03` if ocp version 4.10+ and `450.80.02` otherwise -The deciding factor for which image version to use is the RHEL or RHCOS version. -RHEL 8.6 ---> 470.182.03 -RHEL 7.9 ---> 450.80.02 -The version of RHEL that is used by OCP version is per https://cloud.ibm.com/docs/openshift?topic=openshift-rhel_migrate +- Default Value: N/A -OCP 4.10+ ---> RHEL 8 -OCP 4.9 ---> Either RHEL 8 or RHEL 7 with 7 being the default -OCP 4.6, 4.7, & 4.8 ---> RHEL 7 - -RHCOS 4.9 & 4.0 ---> 470.182.03 -RHCOS 4.4 - 4.6 ---> 450.80.02 -For other RHCOS versions find an appropriate driver version at https://catalog.ngc.nvidia.com/orgs/nvidia/containers/driver/tags +See the attached links for more information and to decide which driver version to use. +https://catalog.ngc.nvidia.com/orgs/nvidia/containers/driver/tags ### gpu_driver_repository_path The gpu driver repository. If using a different repository, you can set the value for this repo. We only support public repositories at the moment. @@ -58,7 +50,6 @@ The gpu driver repository. If using a different repository, you can set the valu For more information on the NVIDIA GPU and NFD operators, visit https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/openshift/install-gpu-ocp.html - Example Playbook ---------------- diff --git a/ibm/mas_devops/roles/nvidia_gpu/defaults/main.yml b/ibm/mas_devops/roles/nvidia_gpu/defaults/main.yml index 9b5aa30ecf..e10580030e 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/defaults/main.yml +++ b/ibm/mas_devops/roles/nvidia_gpu/defaults/main.yml @@ -3,15 +3,12 @@ gpu_namespace: "{{ lookup('env', 'GPU_NAMESPACE') | default('nvidia-gpu-operator', true) }}" # Set channel for install and updates -gpu_channel: "{{ lookup('env', 'GPU_CHANNEL') | default('v23.3', true) }}" +gpu_channel: "{{ lookup('env', 'GPU_CHANNEL') | default('v24.9', true) }}" # GPU driver settings -_gpu_driver_version: "{{ lookup('env', 'GPU_DRIVER_VERSION') }}" -rhel8_driver_version: 525.105.17 # default for Rhel 8 and corresponding rhcos -rhel7_driver_version: 450.80.02 # default for Rhel 7 and corresponding rhcos +gpu_driver_version: "{{ lookup('env', 'GPU_DRIVER_VERSION') }}" gpu_driver_repository_path: "{{ lookup('env', 'GPU_DRIVER_REPOSITORY_PATH') | default('nvcr.io/nvidia', true) }}" - # Variables for Node Feature Discovery Operator # NFD namespace nfd_namespace: "{{ lookup('env', 'NFD_NAMESPACE') | default('openshift-nfd', true) }}" diff --git a/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml b/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml index d5909feb0a..96b3f7d2a4 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml +++ b/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml @@ -18,17 +18,6 @@ set_fact: ocp_version_num: "{{ ocp_version_lookup.resources[0].status.desired.version }}" -- name: "Set gpu driver version default for ocp 4.10+" - when: ocp_version_num is version('4.10.0', '>') - set_fact: - gpu_driver_version: "{{ (_gpu_driver_version is defined and _gpu_driver_version != '') | ternary(_gpu_driver_version, rhel8_driver_version) }}" - -- name: "Set gpu driver version for other ocp versions" - when: ocp_version_num is version('4.10.0', '<') - set_fact: - gpu_driver_version: "{{ (_gpu_driver_version is defined and _gpu_driver_version != '') | ternary(_gpu_driver_version, rhel7_driver_version) }}" - - # 2. Lookup the packagemanifest for gpu-operator-certified # ----------------------------------------------------------------------------- - name: Get gpu-operator-certified package manifest @@ -62,70 +51,86 @@ - "OCP Release Version ................ {{ ocp_version_num }}" - "GPU Namespace ...................... {{ gpu_namespace }}" - "GPU Channel ...................... {{ gpu_channel }}" - - "GPU Driver Version ................. {{ gpu_driver_version }}" - - "GPU Driver Repo Path ............... {{ gpu_driver_repository_path }}" - -# 4. Create NVIDIA GPU project +# 3. Check if GPU operator is already installed # ----------------------------------------------------------------------------- -- name: "Create gpu namespace" - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: '{{ gpu_namespace }}' - - -# 5. Create NVIDIA GPU operator group and subscription -# ----------------------------------------------------------------------------- -- name: "Create gpu operator group" - kubernetes.core.k8s: - definition: "{{ lookup('template', 'templates/gpu-operatorgroup.yml.j2') }}" - wait: yes - wait_timeout: 60 #subsequent tasks will fail if the CRD isn't fully created - -- name: "Create gpu subscription" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/gpu-subscription.yml.j2') }}" - wait: yes - wait_timeout: 300 - wait_condition: - type: 'CatalogSourcesUnhealthy' - status: "False" - -- name: "Wait until ClusterPolicy CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: clusterpolicies.nvidia.com - - -# 6. Create GPU Cluster Policy -# ----------------------------------------------------------------------------- -- name: "Create Cluster Policy instance for ocp 4.10+" - when: ocp_version_num is version('4.10', '>') - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/clusterpolicy-v2.yml.j2') }}" - -- name: "Create Cluster Policy instance for other ocp versions" - when: ocp_version_num is version('4.10', '<') - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/clusterpolicy.yml.j2') }}" - - -# 7. Wait for Cluster Policy to be ready -# ----------------------------------------------------------------------------- -- name: "Wait for Cluster Policy instance to be ready (60s delay)" +- name: Check if GPU operator is already installed kubernetes.core.k8s_info: api_version: nvidia.com/v1 name: "gpu-cluster-policy" kind: ClusterPolicy - register: gpu_cr_result - until: - - gpu_cr_result.resources is defined and gpu_cr_result.resources | length == 1 - - gpu_cr_result.resources[0].status is defined - - gpu_cr_result.resources[0].status.state is defined - - gpu_cr_result.resources[0].status.state == 'ready' - retries: 30 # approx 30 minutes before we give up - delay: 60 # 1 minute + register: gpu_clusterpolicy_result + +- name: "Debug information" + debug: + msg: + - "gpu_clusterpolicy_result ...................... {{ gpu_clusterpolicy_result }}" + +# 4. Install GPU operator if not already installed +# ----------------------------------------------------------------------------- +- name: Install GPU operator if not already installed + block: + # 4.1 Create NVIDIA GPU project + # ----------------------------------------------------------------------------- + - name: "Create gpu namespace" + kubernetes.core.k8s: + api_version: v1 + kind: Namespace + name: '{{ gpu_namespace }}' + + # 4.2 Create NVIDIA GPU operator group and subscription + # ----------------------------------------------------------------------------- + - name: "Create gpu operator group" + kubernetes.core.k8s: + definition: "{{ lookup('template', 'templates/gpu-operatorgroup.yml.j2') }}" + wait: yes + wait_timeout: 60 #subsequent tasks will fail if the CRD isn't fully created + + - name: "Create gpu subscription" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/gpu-subscription.yml.j2') }}" + wait: yes + wait_timeout: 300 + wait_condition: + type: 'CatalogSourcesUnhealthy' + status: "False" + + - name: "Wait until ClusterPolicy CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: clusterpolicies.nvidia.com + + # 4.3 Create GPU Cluster Policy + # ----------------------------------------------------------------------------- + - name: "Create Cluster Policy instance using latest driver" + when: gpu_driver_version is not defined + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/clusterpolicy-v2.yml.j2') }}" + + - name: "Create Cluster Policy instance using custom driver" + when: gpu_driver_version is defined + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/clusterpolicy-customversion.yml.j2') }}" + + # 4.4. Wait for Cluster Policy to be ready + # ----------------------------------------------------------------------------- + - name: "Wait for Cluster Policy instance to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: nvidia.com/v1 + name: "gpu-cluster-policy" + kind: ClusterPolicy + register: gpu_cr_result + until: + - gpu_cr_result.resources is defined and gpu_cr_result.resources | length == 1 + - gpu_cr_result.resources[0].status is defined + - gpu_cr_result.resources[0].status.state is defined + - gpu_cr_result.resources[0].status.state == 'ready' + retries: 30 # approx 30 minutes before we give up + delay: 60 # 1 minute + when: + - gpu_clusterpolicy_result.resources | length == 0 + - gpu_clusterpolicy_result.resources[0].status is not defined + diff --git a/ibm/mas_devops/roles/nvidia_gpu/tasks/nfd_setup.yml b/ibm/mas_devops/roles/nvidia_gpu/tasks/nfd_setup.yml index f29f376fe7..9bb13a913b 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/tasks/nfd_setup.yml +++ b/ibm/mas_devops/roles/nvidia_gpu/tasks/nfd_setup.yml @@ -34,64 +34,82 @@ - "NFD Channel ...................... {{ nfd_channel }}" -# 3. Create NFD project +# 3. Check if NFD operator is already installed # ----------------------------------------------------------------------------- -- name: "Create nfd namespace" - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: '{{ nfd_namespace }}' - - -# 4. Create NFD operator group and subscription -# ----------------------------------------------------------------------------- -- name: "Create nfd operator group" - kubernetes.core.k8s: - definition: "{{ lookup('template', 'templates/nfd-operatorgroup.yml.j2') }}" - wait: yes - wait_timeout: 60 #subsequent tasks will fail if the CRD isn't fully created - -- name: "Create nfd subscription" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/nfd-subscription.yml.j2') }}" - wait: yes - wait_timeout: 300 - wait_condition: - type: 'CatalogSourcesUnhealthy' - status: "False" - -- name: "Wait until NodeFeatureDiscoveries CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: nodefeaturediscoveries.nfd.openshift.io - - -# 5. Create NFD instance -# ----------------------------------------------------------------------------- -- name: "Create NodeFeatureDiscovery instance" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/nfd-instance.yml.j2') }}" - - -# 6. Make sure NFD daemonsets have been created and all pods are ready -# ----------------------------------------------------------------------------- -# Depending on the version of NFD there may also be a nfd-master DaemonSet, but because -# newer versions use a combined worker-master model we will only wait for the nfd-worker -# DaemonSet so that this will work regardless of the version of OCP/NFD that is being used. - -- name: "Wait for 'nfd-worker' DaemonSet pods to be ready" +- name: Check if NFD operator is already installed kubernetes.core.k8s_info: api_version: apps/v1 name: nfd-worker namespace: "{{nfd_namespace}}" kind: DaemonSet - register: nfd_worker_daemonset - until: - - nfd_worker_daemonset.resources is defined - - nfd_worker_daemonset.resources | length > 0 - - nfd_worker_daemonset.resources[0].status.numberReady > 0 - - nfd_worker_daemonset.resources[0].status.numberReady == nfd_worker_daemonset.resources[0].status.desiredNumberScheduled - retries: 30 # approx 30 minutes before we give up - delay: 60 # 1 minute + register: nfd_worker_daemonset_result + +- name: "Debug information" + debug: + msg: + - "nfd_worker_daemonset_result ...................... {{ nfd_worker_daemonset_result }}" + +# 4. Install NFD operator if not already installed +# ----------------------------------------------------------------------------- +- name: Install NFD operator if not already installed + block: + # 4.1 Create NFD project + # ----------------------------------------------------------------------------- + - name: "Create nfd namespace" + kubernetes.core.k8s: + api_version: v1 + kind: Namespace + name: '{{ nfd_namespace }}' + + # 4.2 Create NFD operator group and subscription + # ----------------------------------------------------------------------------- + - name: "Create nfd operator group" + kubernetes.core.k8s: + definition: "{{ lookup('template', 'templates/nfd-operatorgroup.yml.j2') }}" + wait: yes + wait_timeout: 60 #subsequent tasks will fail if the CRD isn't fully created + + - name: "Create nfd subscription" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/nfd-subscription.yml.j2') }}" + wait: yes + wait_timeout: 300 + wait_condition: + type: 'CatalogSourcesUnhealthy' + status: "False" + + - name: "Wait until NodeFeatureDiscoveries CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: nodefeaturediscoveries.nfd.openshift.io + + # 4.3 Create NFD instance + # ----------------------------------------------------------------------------- + - name: "Create NodeFeatureDiscovery instance" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/nfd-instance.yml.j2') }}" + + # 4.4. Make sure NFD daemonsets have been created and all pods are ready + # ----------------------------------------------------------------------------- + # Depending on the version of NFD there may also be a nfd-master DaemonSet, but because + # newer versions use a combined worker-master model we will only wait for the nfd-worker + # DaemonSet so that this will work regardless of the version of OCP/NFD that is being used. + + - name: "Wait for 'nfd-worker' DaemonSet pods to be ready" + kubernetes.core.k8s_info: + api_version: apps/v1 + name: nfd-worker + namespace: "{{nfd_namespace}}" + kind: DaemonSet + register: nfd_worker_daemonset + until: + - nfd_worker_daemonset.resources is defined + - nfd_worker_daemonset.resources | length > 0 + - nfd_worker_daemonset.resources[0].status.numberReady > 0 + - nfd_worker_daemonset.resources[0].status.numberReady == nfd_worker_daemonset.resources[0].status.desiredNumberScheduled + retries: 30 # approx 30 minutes before we give up + delay: 60 # 1 minute + when: + - nfd_worker_daemonset_result.resources | length == 0 diff --git a/ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy.yml.j2 b/ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy-customversion.yml.j2 similarity index 56% rename from ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy.yml.j2 rename to ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy-customversion.yml.j2 index 6a260d6ebd..e9184292ff 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy.yml.j2 +++ b/ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy-customversion.yml.j2 @@ -5,36 +5,59 @@ metadata: name: gpu-cluster-policy spec: driver: + enabled: true repository: "{{ gpu_driver_repository_path }}" + version: "{{ gpu_driver_version }}" image: driver - imagePullSecrets: [] - licensingConfig: + upgradePolicy: + autoUpgrade: true + drain: + deleteEmptyDir: false + enable: false + force: false + timeoutSeconds: 300 + maxParallelUpgrades: 1 + maxUnavailable: 25% + podDeletion: + deleteEmptyDir: false + force: false + timeoutSeconds: 300 + waitForCompletion: + timeoutSeconds: 0 + repoConfig: configMapName: '' + certConfig: + name: '' + licensingConfig: nlsEnabled: false - version: "{{ gpu_driver_version }}" - enabled: true + configMapName: '' + virtualTopology: + config: '' + kernelModuleConfig: + name: '' vgpuDeviceManager: - config: - default: default - name: vgpu-devices-config enabled: true migManager: enabled: true operator: defaultRuntime: crio initContainer: {} - runtimeClass: nvidia use_ocp_driver_toolkit: true dcgm: enabled: true - gfd: {} + gfd: + enabled: true dcgmExporter: config: name: '' + enabled: true + serviceMonitor: + enabled: true devicePlugin: config: default: '' name: '' + enabled: true mig: strategy: single sandboxDevicePlugin: @@ -46,16 +69,18 @@ spec: value: 'true' nodeStatusExporter: enabled: true - daemonsets: {} + daemonsets: + rollingUpdate: + maxUnavailable: '1' + updateStrategy: RollingUpdate sandboxWorkloads: defaultWorkload: container enabled: false + gds: + enabled: false vgpuManager: - enabled: true + enabled: false vfioManager: enabled: true toolkit: enabled: true - image: container-toolkit - repository: nvcr.io/nvidia/k8s - version: 1.6.0-ubi8 diff --git a/ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy-v2.yml.j2 b/ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy-v2.yml.j2 index ecbdd3c508..962489baf1 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy-v2.yml.j2 +++ b/ibm/mas_devops/roles/nvidia_gpu/templates/clusterpolicy-v2.yml.j2 @@ -5,14 +5,33 @@ metadata: name: gpu-cluster-policy spec: driver: - repository: "{{ gpu_driver_repository_path }}" - image: driver - imagePullSecrets: [] - licensingConfig: + enabled: true + upgradePolicy: + autoUpgrade: true + drain: + deleteEmptyDir: false + enable: false + force: false + timeoutSeconds: 300 + maxParallelUpgrades: 1 + maxUnavailable: 25% + podDeletion: + deleteEmptyDir: false + force: false + timeoutSeconds: 300 + waitForCompletion: + timeoutSeconds: 0 + repoConfig: configMapName: '' + certConfig: + name: '' + licensingConfig: nlsEnabled: false - version: "{{ gpu_driver_version }}" - enabled: true + configMapName: '' + virtualTopology: + config: '' + kernelModuleConfig: + name: '' vgpuDeviceManager: enabled: true migManager: @@ -20,7 +39,6 @@ spec: operator: defaultRuntime: crio initContainer: {} - runtimeClass: nvidia use_ocp_driver_toolkit: true dcgm: enabled: true @@ -63,4 +81,3 @@ spec: enabled: true toolkit: enabled: true - installDir: /usr/local/nvidia diff --git a/ibm/mas_devops/roles/ocp_cluster_monitoring/README.md b/ibm/mas_devops/roles/ocp_cluster_monitoring/README.md index 07f6cb086d..113839c0eb 100644 --- a/ibm/mas_devops/roles/ocp_cluster_monitoring/README.md +++ b/ibm/mas_devops/roles/ocp_cluster_monitoring/README.md @@ -23,11 +23,11 @@ Adjust the retention period for Prometheus metrics, only used when both `prometh - Default Value: `15d` ### prometheus_storage_class -Declare the storage class for Prometheus' metrics data persistent volume. +Declare the storage class for Prometheus' metrics data persistent volume. Storage class must support ReadWriteOnce(RWO) access mode. - **Required** if one of the known supported storage classes is not installed in the cluster. - Environment Variable: `PROMETHEUS_STORAGE_CLASS` -- Default Value: `ibmc-file-gold-gid`, `ocs-storagecluster-cephfs`, `azurefiles-premium` (if available) +- Default Value: `ibmc-block-gold`, `ocs-storagecluster-ceph-rbd`, or `managed-premium` (if available) ### prometheus_storage_size Adjust the size of the volume used to store metrics, only used when both `prometheus_storage_class` and `prometheus_alertmgr_storage_class` are set. @@ -42,6 +42,7 @@ Declare the storage class for AlertManager's persistent volume. - **Required** if one of the known supported storage classes is not installed in the cluster. - Environment Variable: `PROMETHEUS_ALERTMGR_STORAGE_CLASS` - Default Value: `ibmc-file-gold-gid`, `ocs-storagecluster-cephfs`, `azurefiles-premium` (if available) +- **Note**: Storage class must support ReadWriteMany(RWX) access mode. ### prometheus_alertmgr_storage_size Adjust the size of the volume used by AlertManager, only used when both `prometheus_storage_class` and `prometheus_alertmgr_storage_class` are set. @@ -58,7 +59,7 @@ Adjust the retention period for User Workload Prometheus metrics, this parameter - Default Value: `15d` ### prometheus_userworkload_storage_class -Declare the storage class for User Workload Prometheus' metrics data persistent volume. +Declare the storage class for User Workload Prometheus' metrics data persistent volume. Storage class must support ReadWriteOnce(RWO) access mode. - Optional - Environment Variable: `PROMETHEUS_USERWORKLOAD_STORAGE_CLASS` diff --git a/ibm/mas_devops/roles/ocp_cluster_monitoring/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/ocp_cluster_monitoring/tasks/determine-storage-classes.yml index a14180d6fb..01bde77fb3 100644 --- a/ibm/mas_devops/roles/ocp_cluster_monitoring/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/ocp_cluster_monitoring/tasks/determine-storage-classes.yml @@ -1,17 +1,8 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "install : determine-storage-classes : Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses +- name: "Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Prometheus Storage (Required) @@ -19,7 +10,7 @@ - name: "install : determine-storage-classes : Default Prometheus Storage if not set by user" when: prometheus_storage_class is not defined or prometheus_storage_class == "" set_fact: - prometheus_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + prometheus_storage_class: "{{ defaultStorageClasses.rwo }}" - name: Assert that Prometheus storage class has been defined assert: @@ -32,7 +23,7 @@ - name: "install : determine-storage-classes : Default Prometheus Alert Manager Storage if not set by user" when: prometheus_alertmgr_storage_class is not defined or prometheus_alertmgr_storage_class == "" set_fact: - prometheus_alertmgr_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" + prometheus_alertmgr_storage_class: "{{ defaultStorageClasses.rwx }}" - name: "install : determine-storage-classes : Assert that Prometheus Alert Manager storage class has been defined" assert: @@ -45,7 +36,7 @@ - name: "install : determine-storage-classes : Default Prometheus User Workload Monitoring Storage if not set by user" when: prometheus_userworkload_storage_class is not defined or prometheus_userworkload_storage_class == "" set_fact: - prometheus_userworkload_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + prometheus_userworkload_storage_class: "{{ defaultStorageClasses.rwo }}" - name: "install : determine-storage-classes : Assert that Prometheus User Workload Monitoring storage class has been defined" assert: diff --git a/ibm/mas_devops/roles/ocp_simulate_disconnected_network/README.md b/ibm/mas_devops/roles/ocp_simulate_disconnected_network/README.md index 80bca5e14a..40f9a3455c 100644 --- a/ibm/mas_devops/roles/ocp_simulate_disconnected_network/README.md +++ b/ibm/mas_devops/roles/ocp_simulate_disconnected_network/README.md @@ -17,6 +17,7 @@ sh-4.4# more /host/etc/hosts The default exclusions are: + - quay.io - registry.redhat.io - registry.connect.redhat.com diff --git a/ibm/mas_devops/roles/ocp_verify/tasks/main.yml b/ibm/mas_devops/roles/ocp_verify/tasks/main.yml index 50a806e4f4..64f98f993c 100644 --- a/ibm/mas_devops/roles/ocp_verify/tasks/main.yml +++ b/ibm/mas_devops/roles/ocp_verify/tasks/main.yml @@ -14,79 +14,6 @@ - ocp_info.resources is defined - ocp_info.resources | json_query('[*].status.conditions[?type==`Available`][].status') | select ('match','True') | list | length == 1 -# 1b. Patch the ibmcloud-block-storage-driver DaemonSet image if necessary -# fix for issue https://github.ibm.com/alchemy-containers/armada-storage/issues/6503 -# v2.5.12 needs updating to v2.5.14 -# v2.4.23 needs updating to v2.4.25 -# This task can be removed once OCP provision no longer installs ibmcloud-block-storage-driver v2.5.12 or v2.4.23 -# ------------------------------------------------------------------------------------- -- name: Lookup ibmcloud-block-storage-driver image - kubernetes.core.k8s_info: - api_version: apps/v1 - kind: DaemonSet - name: ibmcloud-block-storage-driver - namespace: kube-system - register: lookup_storage_driver - -- name: "Get Driver Image" - set_fact: - driver_image: "{{ lookup_storage_driver.resources[0].spec.template.spec.containers[0].image }}" - when: - - lookup_storage_driver.resources is defined - - lookup_storage_driver.resources | length == 1 - - lookup_storage_driver.resources[0].spec.template.spec.containers[0].image is defined - -- name: "Determine New Driver Image" - set_fact: - new_driver_image: "{{ upgrade_image[driver_image] }}" - when: - - driver_image is defined - - upgrade_image[driver_image] is defined - -- debug: - msg: "Storage Driver {{ driver_image }} doesn't need to be replaced." - when: - - driver_image is defined - - new_driver_image is not defined - -- name: Update ibmcloud-block-storage-driver image - when: new_driver_image is defined - block: - - debug: - msg: "Driver {{ driver_image }} to be replaced with {{ new_driver_image }}." - - - name: Update ibmcloud-block-storage-driver image - kubernetes.core.k8s: - api_version: apps/v1 - kind: DaemonSet - name: ibmcloud-block-storage-driver - namespace: kube-system - definition: - spec: - template: - spec: - containers: - - name: ibmcloud-block-storage-driver-container - image: "{{ new_driver_image }}" - - - - name: Wait for the ibmcloud-block-storage-driver DaemonSet to update - kubernetes.core.k8s_info: - api_version: apps/v1 - kind: DaemonSet - name: ibmcloud-block-storage-driver - namespace: kube-system - register: lookup_storage_driver - until: - - lookup_storage_driver.resources is defined - - lookup_storage_driver.resources | length > 0 - - lookup_storage_driver.resources[0].status is defined - - lookup_storage_driver.resources[0].status.updatedNumberScheduled is defined - - lookup_storage_driver.resources[0].status.desiredNumberScheduled is defined - - lookup_storage_driver.resources[0].status.updatedNumberScheduled == lookup_storage_driver.resources[0].status.desiredNumberScheduled - retries: 30 # approx 10 minutes before we give up - delay: 20 # 1 minute - # 2. Wait for all catalogsources to be healthy # ----------------------------------------------------------------------------- - name: Check CatalogSource Status diff --git a/ibm/mas_devops/roles/odh/tasks/authorino-operator.yml b/ibm/mas_devops/roles/odh/tasks/authorino-operator.yml index f6bb60219e..a79ef9cbd0 100644 --- a/ibm/mas_devops/roles/odh/tasks/authorino-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/authorino-operator.yml @@ -1,18 +1,16 @@ --- -# Install Operator & create entitlement openshift-authorino +# 1. Install Operator & create entitlement openshift-authorino # ----------------------------------------------------------------------------- - name: "Install Openshift authorino Operator" - ansible.builtin.include_role: - name: ibm.mas_devops.install_operator - vars: + ibm.mas_devops.apply_subscription: namespace: "{{ openshift_namespace }}" - icr_username: "{{ mas_entitlement_username }}" - icr_password: "{{ mas_entitlement_key }}" - catalog_source: "{{ serverless_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/authorino/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/authorino/subscription.yml.j2') }}" + package_name: "authorino-operator" + package_channel: "{{ serverless_channel }}" # This looks wrong, but that's what was in the original template + catalog_source: "{{ authorino_catalog_source }}" + register: subscription + -# Wait until the Authorino CRD is available +# 2. Wait until the Authorino CRD is available # ----------------------------------------------------------------------------- - name: "Wait until the Authorino CRD is available" include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" diff --git a/ibm/mas_devops/roles/odh/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/odh/tasks/determine-storage-classes.yml index 945c3f5a21..44919a2133 100644 --- a/ibm/mas_devops/roles/odh/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/odh/tasks/determine-storage-classes.yml @@ -1,30 +1,22 @@ -# Lookup storage class availabiity +--- +# 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" -- name: "determine-storage-classes : Lookup storage classes" - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" - -# Set Primary Storage (Required) +# 2. Set Primary Storage (Required) # ----------------------------------------------------------------------------- - name: "determine-storage-classes : Default Primary Storage if not set by user" when: primary_storage_class is not defined or primary_storage_class == "" set_fact: - primary_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" + primary_storage_class: "{{ defaultStorageClasses.rwx }}" - name: "determine-storage-classes : Assert that primary storage class has been defined" assert: that: primary_storage_class is defined and primary_storage_class != "" fail_msg: "primary_storage_class must be defined" -- name: Set primary StorageClass +- name: "determine-storage-classes : Debug" debug: - msg: "Primary storage class ...... {{ primary_storage_class }}" + msg: "Primary storage class ................. {{ primary_storage_class }}" diff --git a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml index a51a2fe380..cd75ca52a8 100644 --- a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml @@ -2,15 +2,13 @@ # Install Operator & create entitlement openshift-odh # ----------------------------------------------------------------------------- - name: "Install Openshift odh Operator" - ansible.builtin.include_role: - name: ibm.mas_devops.install_operator - vars: + ibm.mas_devops.apply_subscription: namespace: "{{ openshift_namespace }}" - icr_username: "{{ mas_entitlement_username }}" - icr_password: "{{ mas_entitlement_key }}" + package_name: "opendatahub-operator" + package_channel: "{{ odh_channel }}" catalog_source: "{{ odh_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/odh/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/odh/subscription.yml.j2') }}" + register: subscription + # Wait until the Opendata Hub CRD is available # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/odh/tasks/pipeline-operator.yml b/ibm/mas_devops/roles/odh/tasks/pipeline-operator.yml index 1a80508183..7fa054424a 100644 --- a/ibm/mas_devops/roles/odh/tasks/pipeline-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/pipeline-operator.yml @@ -22,17 +22,14 @@ # Install Operator Openshift-pipelines # ----------------------------------------------------------------------------- - name: "Install Openshift pipelines" - ansible.builtin.include_role: - name: ibm.mas_devops.install_operator - vars: + when: not has_subscription + ibm.mas_devops.apply_subscription: namespace: "{{ openshift_namespace }}" - icr_username: "{{ mas_entitlement_username }}" - icr_password: "{{ mas_entitlement_key }}" + package_name: "openshift-pipelines-operator-rh" + package_channel: "{{ pipeline_channel }}" catalog_source: "{{ pipeline_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/pipelines/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/pipelines/subscription.yml.j2') }}" - when: - - not has_subscription + register: subscription + # Wait until the Openshift-pipelines CRD is available # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml b/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml index 10e2c967fe..727860b114 100644 --- a/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml @@ -1,18 +1,16 @@ --- -# Install Operator & create entitlement openshift-serverless +# 1. Install Operator & create entitlement openshift-serverless # ----------------------------------------------------------------------------- - name: "Install Openshift Serverless Operator" - ansible.builtin.include_role: - name: ibm.mas_devops.install_operator - vars: + ibm.mas_devops.apply_subscription: namespace: "{{ serverless_namespace }}" - icr_username: "{{ mas_entitlement_username }}" - icr_password: "{{ mas_entitlement_key }}" + package_name: "serverless-operator" + package_channel: "{{ serverless_channel }}" catalog_source: "{{ serverless_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/serverless/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/serverless/subscription.yml.j2') }}" + register: subscription + -# Wait until the Serverless CRD is available +# 2. Wait until the Serverless CRD is available # ----------------------------------------------------------------------------- - name: "Wait until the Serverless CRD is available" include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" diff --git a/ibm/mas_devops/roles/odh/tasks/servicemesh-operator.yml b/ibm/mas_devops/roles/odh/tasks/servicemesh-operator.yml index baca08eac4..ff24d1d241 100644 --- a/ibm/mas_devops/roles/odh/tasks/servicemesh-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/servicemesh-operator.yml @@ -1,25 +1,23 @@ --- -# Install Operator & create entitlement openshift-service-mesh +# 1. Install Operator & create entitlement openshift-service-mesh # ----------------------------------------------------------------------------- - name: "Install Openshift ServiceMesh Operator" - ansible.builtin.include_role: - name: ibm.mas_devops.install_operator - vars: + ibm.mas_devops.apply_subscription: namespace: "{{ service_mesh_namespace }}" - icr_username: "{{ mas_entitlement_username }}" - icr_password: "{{ mas_entitlement_key }}" + package_name: "servicemeshoperator" + package_channel: "{{ service_mesh_channel }}" catalog_source: "{{ service_mesh_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/servicemesh/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/servicemesh/subscription.yml.j2') }}" + register: subscription + -# Wait until the ServiceMesh CRD is available +# 2. Wait until the ServiceMesh CRD is available # ----------------------------------------------------------------------------- - name: "Wait until the ServiceMesh CRD is available" include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" vars: crd_name: servicemeshcontrolplanes.maistra.io -# Create Service account +# 3. Create Service account # ----------------------------------------------------------------------------- - name: "Create Service account" kubernetes.core.k8s: diff --git a/ibm/mas_devops/roles/odh/templates/authorino/operator-group.yml.j2 b/ibm/mas_devops/roles/odh/templates/authorino/operator-group.yml.j2 deleted file mode 100644 index 81bd256597..0000000000 --- a/ibm/mas_devops/roles/odh/templates/authorino/operator-group.yml.j2 +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: global-operators - namespace: "{{ openshift_namespace }}" -spec: {} diff --git a/ibm/mas_devops/roles/odh/templates/authorino/subscription.yml.j2 b/ibm/mas_devops/roles/odh/templates/authorino/subscription.yml.j2 deleted file mode 100644 index c3ed424ca8..0000000000 --- a/ibm/mas_devops/roles/odh/templates/authorino/subscription.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: authorino-operator - namespace: "{{ openshift_namespace }}" -spec: - channel: "{{ serverless_channel }}" - installPlanApproval: Automatic - name: authorino-operator - source: "{{ authorino_catalog_source }}" - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/odh/templates/odh/operator-group.yml.j2 b/ibm/mas_devops/roles/odh/templates/odh/operator-group.yml.j2 deleted file mode 100644 index 81bd256597..0000000000 --- a/ibm/mas_devops/roles/odh/templates/odh/operator-group.yml.j2 +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: global-operators - namespace: "{{ openshift_namespace }}" -spec: {} diff --git a/ibm/mas_devops/roles/odh/templates/odh/subscription.yml.j2 b/ibm/mas_devops/roles/odh/templates/odh/subscription.yml.j2 deleted file mode 100644 index 0576745b47..0000000000 --- a/ibm/mas_devops/roles/odh/templates/odh/subscription.yml.j2 +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: opendatahub-operator - namespace: "{{ openshift_namespace }}" -spec: - channel: "{{ odh_channel }}" - installPlanApproval: Manual - name: opendatahub-operator - source: "{{ odh_catalog_source }}" - sourceNamespace: openshift-marketplace - startingCSV: "{{ odh_operator_version }}" diff --git a/ibm/mas_devops/roles/odh/templates/pipelines/operator-group.yml.j2 b/ibm/mas_devops/roles/odh/templates/pipelines/operator-group.yml.j2 deleted file mode 100644 index 81bd256597..0000000000 --- a/ibm/mas_devops/roles/odh/templates/pipelines/operator-group.yml.j2 +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: global-operators - namespace: "{{ openshift_namespace }}" -spec: {} diff --git a/ibm/mas_devops/roles/odh/templates/pipelines/subscription.yml.j2 b/ibm/mas_devops/roles/odh/templates/pipelines/subscription.yml.j2 deleted file mode 100644 index 9fb7b51cfb..0000000000 --- a/ibm/mas_devops/roles/odh/templates/pipelines/subscription.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: openshift-pipelines-operator - namespace: {{ openshift_namespace }} -spec: - channel: "{{ pipeline_channel }}" - installPlanApproval: Automatic - name: openshift-pipelines-operator-rh - source: "{{ pipeline_catalog_source }}" - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/odh/templates/serverless/operator-group.yml.j2 b/ibm/mas_devops/roles/odh/templates/serverless/operator-group.yml.j2 deleted file mode 100644 index 2bbbd0ca3e..0000000000 --- a/ibm/mas_devops/roles/odh/templates/serverless/operator-group.yml.j2 +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: serverless-operators - namespace: "{{ serverless_namespace }}" -spec: {} diff --git a/ibm/mas_devops/roles/odh/templates/serverless/subscription.yml.j2 b/ibm/mas_devops/roles/odh/templates/serverless/subscription.yml.j2 deleted file mode 100644 index c6f79bd99e..0000000000 --- a/ibm/mas_devops/roles/odh/templates/serverless/subscription.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: serverless-operator - namespace: "{{ serverless_namespace }}" -spec: - channel: "{{ serverless_channel }}" - installPlanApproval: Automatic - name: serverless-operator - source: "{{ serverless_catalog_source }}" - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/odh/templates/servicemesh/operator-group.yml.j2 b/ibm/mas_devops/roles/odh/templates/servicemesh/operator-group.yml.j2 deleted file mode 100644 index 81bd256597..0000000000 --- a/ibm/mas_devops/roles/odh/templates/servicemesh/operator-group.yml.j2 +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: global-operators - namespace: "{{ openshift_namespace }}" -spec: {} diff --git a/ibm/mas_devops/roles/odh/templates/servicemesh/subscription.yml.j2 b/ibm/mas_devops/roles/odh/templates/servicemesh/subscription.yml.j2 deleted file mode 100644 index 0b5334d138..0000000000 --- a/ibm/mas_devops/roles/odh/templates/servicemesh/subscription.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: servicemeshoperator - namespace: "{{ service_mesh_namespace }}" -spec: - channel: "{{ service_mesh_channel }}" - installPlanApproval: Automatic - name: servicemeshoperator - source: "{{ service_mesh_catalog_source }}" - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/opentelemetry/tasks/install.yml b/ibm/mas_devops/roles/opentelemetry/tasks/install.yml index 4ab1822d47..c96fba8dea 100644 --- a/ibm/mas_devops/roles/opentelemetry/tasks/install.yml +++ b/ibm/mas_devops/roles/opentelemetry/tasks/install.yml @@ -3,11 +3,11 @@ # 1. Create OpenTelemetry Subscription # ----------------------------------------------------------------------------- - name: "install : opentelemetry : Create OpenTelemetry Subscription" - include_tasks: "{{ role_path }}/../../common_tasks/create_subscription.yml" - vars: + ibm.mas_devops.apply_subscription: + namespace: openshift-operators package_name: opentelemetry-operator - channel_name: alpha - subscription_namespace: openshift-operators + package_channel: alpha + register: subscription # 2. Wait until the OpenTelemetryCollector CRD is available diff --git a/ibm/mas_devops/roles/registry/README.md b/ibm/mas_devops/roles/registry/README.md index 424dc47e15..a9ef1f68c6 100644 --- a/ibm/mas_devops/roles/registry/README.md +++ b/ibm/mas_devops/roles/registry/README.md @@ -49,7 +49,7 @@ Usage for tear-down action -------------------------- This role can also be used to permanently delete a mirror registry from a given cluster by setting the `registry_action` to `tear-down` and specifying the corresponding `registry_namespace`, if not using the default value. -Note that the tear-down action deletes the registry completely including the PVC storage and the registry namespace. To start up the registry again, the role needs to be run again with the registry_action on default or `setup`. Images previously stored in the registry before the tear-down will no longer be available and will need to be mirrored again once the registry setup has completed. Take precaution when using this function and expect that images can no longer be accessed from the registry that has been torn down. +Note that the tear-down action deletes the registry completely including the PVC storage and the registry namespace. To start up the registry again, the role needs to be run again with the registry_action on default or `setup`. Images previously stored in the registry before the tear-down will no longer be available and will need to be mirrored again once the registry setup has completed. Take precaution when using this function and expect that images can no longer be accessed from the registry that has been torn down. **Note:** Recreating the registry will also create a new ca cert for the new registry. @@ -74,7 +74,7 @@ The namespace where the registry to run - Default Value: `airgap-registry` ### registry_storage_class -Required. The name of the storage class to configure the MongoDb operator to use for persistent storage in the MongoDb cluster. +**Required**: The name of the storage class to configure the MongoDb operator to use for persistent storage in the MongoDb cluster. Storage class must support ReadWriteOnce(RWO) access mode. - **Required**, unless running in IBM Cloud ROKS, where the storage class will default to `ibmc-block-gold`. - Environment Variable: `REGISTRY_STORAGE_CLASS` diff --git a/ibm/mas_devops/roles/registry/tasks/setup/determine-storage-classes.yml b/ibm/mas_devops/roles/registry/tasks/setup/determine-storage-classes.yml index 19020b23fd..389cd0ed8a 100644 --- a/ibm/mas_devops/roles/registry/tasks/setup/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/registry/tasks/setup/determine-storage-classes.yml @@ -1,31 +1,18 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Set Storage (Required) # ----------------------------------------------------------------------------- -- name: Default Storage if not set by user +- name: "determine-storage-classes : Default Storage if not set by user" when: registry_storage_class is not defined or registry_storage_class == "" - vars: - # ROKS - supported_storage_classes: [ibmc-block-gold] set_fact: - registry_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(supported_storage_classes) }}" + registry_storage_class: "{{ defaultStorageClasses.rwo }}" -- name: Assert that storage class has been defined +- name: "determine-storage-classes : Assert that storage class has been defined" assert: that: registry_storage_class is defined and registry_storage_class != "" fail_msg: "registry_storage_class must be defined" @@ -33,7 +20,7 @@ # 3. Debug storage class configuration # ----------------------------------------------------------------------------- -- name: "Debug registry storage class configuration" +- name: "determine-storage-classes : Debug registry storage class configuration" debug: msg: - - "Storage class (registry) ...... {{ registry_storage_class }}" + - "Storage class (registry) ............... {{ registry_storage_class }}" diff --git a/ibm/mas_devops/roles/sls/defaults/main.yml b/ibm/mas_devops/roles/sls/defaults/main.yml index 5907d608f1..6ffdc3e97c 100644 --- a/ibm/mas_devops/roles/sls/defaults/main.yml +++ b/ibm/mas_devops/roles/sls/defaults/main.yml @@ -17,6 +17,10 @@ sls_entitlement_username: "{{ lookup('env', 'SLS_ENTITLEMENT_USERNAME') | defaul ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" sls_entitlement_key: "{{ lookup('env', 'SLS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" +# Development Registry Entitlement +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + db_provider: "{{ lookup('env', 'DB_PROVIDER') | default('community', true) }}" # SLS settings diff --git a/ibm/mas_devops/roles/sls/tasks/install/main.yml b/ibm/mas_devops/roles/sls/tasks/install/main.yml index fdeb7ec10f..6939489444 100644 --- a/ibm/mas_devops/roles/sls/tasks/install/main.yml +++ b/ibm/mas_devops/roles/sls/tasks/install/main.yml @@ -183,16 +183,22 @@ # 7. Install Operator & create entitlement # ----------------------------------------------------------------------------- -- name: "Install SLS Operator" - ansible.builtin.include_role: - name: ibm.mas_devops.install_operator - vars: +- name: "Create IBM Entitlement Key" + when: before_sls_380 or (artifactory_username is defined and artifactory_username != "") + ibm.mas_devops.update_ibm_entitlement: + namespace: "{{ sls_namespace }}" + icr_username: "{{ sls_entitlement_username }}" + icr_password: "{{ sls_entitlement_key }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + +- name: "Create ibm-mas Subscription" + ibm.mas_devops.apply_subscription: namespace: "{{ sls_namespace }}" - icr_username: "{{ before_sls_380 | ternary(sls_entitlement_username, '') }}" - icr_password: "{{ before_sls_380 | ternary(sls_entitlement_key, '') }}" + package_name: "ibm-sls" + package_channel: "{{ sls_channel }}" catalog_source: "{{ sls_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/subscription.yml.j2') }}" + register: subscription # 8. Wait until the LicenseService CRD is available diff --git a/ibm/mas_devops/roles/sls/templates/operator-group.yml.j2 b/ibm/mas_devops/roles/sls/templates/operator-group.yml.j2 deleted file mode 100644 index 8c400e9332..0000000000 --- a/ibm/mas_devops/roles/sls/templates/operator-group.yml.j2 +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: ibm-sls-operator-group - namespace: "{{ sls_namespace }}" -spec: - targetNamespaces: - - "{{ sls_namespace }}" diff --git a/ibm/mas_devops/roles/sls/templates/subscription.yml.j2 b/ibm/mas_devops/roles/sls/templates/subscription.yml.j2 deleted file mode 100644 index 25185851fb..0000000000 --- a/ibm/mas_devops/roles/sls/templates/subscription.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: ibm-sls - namespace: "{{ sls_namespace }}" -spec: - channel: "{{ sls_channel }}" - installPlanApproval: Automatic - name: ibm-sls - source: "{{sls_catalog_source}}" - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/README.md b/ibm/mas_devops/roles/suite_app_backup_restore/README.md index 6807471c0b..6b4826ed19 100644 --- a/ibm/mas_devops/roles/suite_app_backup_restore/README.md +++ b/ibm/mas_devops/roles/suite_app_backup_restore/README.md @@ -70,82 +70,13 @@ Set the [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - Environment Variable: `MASBR_JOB_TIMEZONE` - Default: None -### masbr_storage_type -Set `local` or `cloud` to indicate this role to save the backup files to local file system or cloud object storage. - -- **Required** -- Environment Variable: `MASBR_STORAGE_TYPE` -- Default: None - ### masbr_storage_local_folder Set local path to save the backup files. -- **Required** only when `MASBR_STORAGE_TYPE=local` +- **Required** - Environment Variable: `MASBR_STORAGE_LOCAL_FOLDER` - Default: None -### masbr_storage_cloud_rclone_file -Set the path of `rclone.conf` file. - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_RCLONE_FILE` -- Default: None - -### masbr_storage_cloud_rclone_name -Set the configuration name defined in `rclone.conf` file. - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_RCLONE_NAME` -- Default: None - -### masbr_storage_cloud_bucket -Set the object storage bucket name for saving the backup files - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_BUCKET` -- Default: None - -### masbr_slack_enabled -Set `true` or `false` to indicate whether this role will send Slack notification messages of the backup and restore progress. - -- Optional -- Environment Variable: `MASBR_SLACK_ENABLED` -- Default: `false` - -### masbr_slack_level -Set `failure`, `info` or `verbose` to indicate this role to send Slack notification messages in which backup and resore phases: - -| Slack level | Backup/Restore phases | -| ----------- | ------------------------------------------------------- | -| failure | `Failed`, `PartiallyFailed` | -| info | `Completed`, `Failed`, `PartiallyFailed` | -| verbose | `InProgress`, `Completed`, `Failed`, `PartiallyFailed` | - -- Optional -- Environment Variable: `MASBR_SLACK_LEVEL` -- Default: `info` - -### masbr_slack_token -The Slack integration token. - -- **Required** only when `MASBR_SLACK_ENABLED=true` -- Environment Variable: `MASBR_SLACK_TOKEN` -- Default: None - -### masbr_slack_channel -The Slack channel to send the notification messages to. - -- **Required** only when `MASBR_SLACK_ENABLED=true` -- Environment Variable: `MASBR_SLACK_CHANNEL` -- Default: None - -### masbr_slack_user -The sender of the Slack notification message. - -- Optional -- Environment Variable: `MASBR_SLACK_USER` -- Default: `MASBR` - Role Variables - Backup ------------------------------------------------------------------------------- @@ -216,7 +147,7 @@ Set the types of data to be restored, multiple data types are separated by comma Role Variables - Manage ------------------------------------------------------------------------------- ### masbr_manage_pvc_paths -Set the Manage PVC paths to use in backup and restore. The PVC path is in the format of `:/`. Multiple PVC paths are separated by commas (e.g. `manage-doclinks1-pvc:/mnt/doclinks1/attachments,manage-doclinks2-pvc:/mnt/doclinks2`). +Set the Manage PVC paths to use in backup and restore. The PVC path is in the format of `:/`. Multiple PVC paths are separated by commas (e.g. `manage-doclinks1-pvc:/mnt/doclinks1/attachments,manage-doclinks2-pvc:/mnt/doclinks2`). The `` and `` are defined in the `ManageWorkspace` CRD instance `spec.settings.deployment.persistentVolumes`: ``` @@ -261,7 +192,6 @@ Backup Manage attachments, note that this does not include backup of any data in mas_app_id: manage masbr_backup_data: pv masbr_manage_pvc_paths: "manage-doclinks1-pvc:/mnt/doclinks1" - masbr_storage_type: local masbr_storage_local_folder: /tmp/masbr roles: - ibm.mas_devops.suite_app_backup_restore @@ -281,7 +211,6 @@ Restore Manage attachments, note that this does not include restore of any data mas_app_id: manage masbr_backup_data: pv masbr_manage_pvc_paths: "manage-doclinks1-pvc:/mnt/doclinks1" - masbr_storage_type: local masbr_storage_local_folder: /tmp/masbr roles: - ibm.mas_devops.suite_app_backup_restore diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/get-app-info.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/get-app-info.yml index cf73e2e532..ae61fac2af 100644 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/get-app-info.yml +++ b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/get-app-info.yml @@ -40,6 +40,7 @@ assert: that: mas_app_version is defined fail_msg: "{{ mas_app_kind }}/{{ mas_app_cr_name }} does not exists!" + when: masbr_action is defined and masbr_action == "backup" - name: "Set fact: {{ mas_app_kind }}/{{ mas_app_cr_name }} status" set_fact: @@ -51,7 +52,7 @@ # When performing restore, we shouldn't care about the status of app. - name: "Fail if {{ mas_app_kind }}/{{ mas_app_cr_name }} is not ready" - when: false + when: masbr_action is defined and masbr_action == "backup" assert: that: mas_app_ready is defined and mas_app_ready fail_msg: "{{ mas_app_kind }}/{{ mas_app_cr_name }} is not ready!" @@ -83,6 +84,7 @@ assert: that: mas_ws_version is defined fail_msg: "{{ mas_ws_kind }}/{{ mas_ws_cr_name }} does not exists!" + when: masbr_action is defined and masbr_action == "backup" - name: "Set fact: {{ mas_ws_kind }}/{{ mas_ws_cr_name }} status" set_fact: @@ -94,7 +96,7 @@ # When performing restore, we shouldn't care about the status of app. - name: "Fail if {{ mas_ws_kind }}/{{ mas_ws_cr_name }} is not ready" - when: false + when: masbr_action is defined and masbr_action == "backup" assert: that: mas_ws_ready is defined and mas_ws_ready fail_msg: "{{ mas_ws_kind }}/{{ mas_ws_cr_name }} is not ready!" @@ -103,6 +105,7 @@ # Output app information # ----------------------------------------------------------------------------- - name: "Debug: {{ mas_app_id | capitalize }} information" + when: masbr_action is defined and masbr_action == "backup" debug: msg: - "{{ mas_app_kind }}/{{ mas_app_cr_name }} version ............ {{ mas_app_version }}" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/main.yml index 72a88f31cc..6722ca867a 100644 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/main.yml @@ -54,26 +54,7 @@ _job_type: "{{ masbr_action }}" -# Create k8s Job to run backup/restore tasks -# ----------------------------------------------------------------------------- -- name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" - vars: - _rt_role_name: "suite_app_backup_restore" - _rt_env: - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - name: "MAS_WORKSPACE_ID" - value: "{{ mas_workspace_id }}" - - name: "MAS_APP_ID" - value: "{{ mas_app_id }}" - - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job block: # Update job status: New # ------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/pv-info.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/pv-info.yml index adfd96484f..cb5971a048 100644 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/pv-info.yml +++ b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/pv-info.yml @@ -22,16 +22,18 @@ - name: "Set fact: workspace pv spec" set_fact: manage_ws_pvs: "{{ _ws_output.resources[0].spec.settings.deployment.persistentVolumes }}" + when: masbr_manage_pvc_paths is defined and masbr_manage_pvc_paths | length > 0 - name: "Debug: workspace pv spec" debug: msg: "{{ manage_ws_pvs }}" + when: manage_ws_pvs is defined # Only go on processing when manage has pv defined # ----------------------------------------------------------------------------- - name: "Only go on processing when manage has pv defined" - when: manage_ws_pvs | length > 0 + when: manage_ws_pvs is defined and manage_ws_pvs | length > 0 block: # Get maxinst pod information - name: "Get maxinst pod information" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-namespace.yml index 72c342cff1..c39f755c5f 100644 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-namespace.yml +++ b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-namespace.yml @@ -17,12 +17,29 @@ .name="{{ mas_workspace_id }}-manage-encryptionsecret-operator" )' {{ masbr_ns_restore_folder }}/Secret-{{ masbr_restore_from_workspace }}-manage-encryptionsecret-operator.yaml; -- name: "Apply encryption secret yaml files" +# Restore namespace resoruces +# ------------------------------------------------------------------------- +# Loop through the folder +- name: "Get the list of files from restore directory" + find: + paths: "{{ masbr_ns_restore_folder }}" + patterns: '*.yml,*.yaml' + recurse: no + register: find_result + +- name: "Apply configs" kubernetes.core.k8s: - apply: true - src: "{{ masbr_ns_restore_folder }}/{{ _ns_resource_file_name }}" - loop: - - "Secret-{{ masbr_restore_from_workspace }}-manage-encryptionsecret.yaml" - - "Secret-{{ masbr_restore_from_workspace }}-manage-encryptionsecret-operator.yaml" - loop_control: - loop_var: _ns_resource_file_name + state: present + definition: "{{ lookup('template', item.path) }}" + with_items: "{{ find_result.files }}" + when: find_result is defined + +# - name: "Apply encryption secret yaml files" +# kubernetes.core.k8s: +# apply: true +# src: "{{ masbr_ns_restore_folder }}/{{ _ns_resource_file_name }}" +# loop: +# - "Secret-{{ masbr_restore_from_workspace }}-manage-encryptionsecret.yaml" +# - "Secret-{{ masbr_restore_from_workspace }}-manage-encryptionsecret-operator.yaml" +# loop_control: +# loop_var: _ns_resource_file_name diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-namespace.yml index 92f82d949f..92931ceddf 100644 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-namespace.yml +++ b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-namespace.yml @@ -55,7 +55,7 @@ changed_when: true shell: > tar -xzf {{ masbr_local_job_folder }}/{{ masbr_job_data_type }}/{{ masbr_ns_restore_from_name }}.tar.gz - -C {{ masbr_ns_restore_folder }} . && + -C {{ masbr_ns_restore_folder }} && ls -lA {{ masbr_ns_restore_folder }} register: _extract_output diff --git a/ibm/mas_devops/roles/suite_app_config/README.md b/ibm/mas_devops/roles/suite_app_config/README.md index 2e12c2369a..569f6edd71 100644 --- a/ibm/mas_devops/roles/suite_app_config/README.md +++ b/ibm/mas_devops/roles/suite_app_config/README.md @@ -223,7 +223,7 @@ There are two defaulted File Storage Persistent Volumes Claim resources that wil The following properties can be defined to customize the persistent volumes for the JMS queues setup for Manage. ### mas_app_settings_jms_queue_pvc_storage_class -Provide the persistent volume storage class to be used for JMS queue configuration. +Provide the persistent volume storage class to be used for JMS queue configuration. Both `ReadWriteOnce` (if using a block storage class) or `ReadWriteMany` (if using file storage class) access modes are supported. **Note:** JMS configuration will only be done if `mas_app_settings_server_bundles_size` property is set to `jms`. - Optional @@ -276,6 +276,7 @@ The following properties can be defined to customize the persistent volumes for ### mas_app_settings_doclinks_pvc_storage_class Provide the persistent volume storage class to be used for doclinks/attachments configuration. +Both `ReadWriteOnce` (if using a block storage class) or `ReadWriteMany` (if using file storage class) are supported. - Optional - Environment Variable: `MAS_APP_SETTINGS_DOCLINKS_PVC_STORAGE_CLASS` @@ -316,7 +317,7 @@ The following properties can be defined to customize the persistent volumes for ### mas_app_settings_bim_pvc_storage_class Provide the persistent volume storage class to be used for Building Information Models configuration. - +Both `ReadWriteOnce` (if using a block storage class) or `ReadWriteMany` (if using file storage class) are supported. - Optional - Environment Variable: `MAS_APP_SETTINGS_BIM_PVC_STORAGE_CLASS` - Default: None - If not set, a default storage class will be auto defined accordingly to your cluster's available storage classes. diff --git a/ibm/mas_devops/roles/suite_app_config/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/suite_app_config/tasks/determine-storage-classes.yml index de49def622..d670c3e937 100644 --- a/ibm/mas_devops/roles/suite_app_config/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/suite_app_config/tasks/determine-storage-classes.yml @@ -1,54 +1,43 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "determine-storage-classes : Lookup storage classes" - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" - -# 2. Set Primary Storage (Required) +# 2. Set Default Manage Storage Class # ----------------------------------------------------------------------------- -- name: "determine-storage-classes : Default Manage PVC Storage if not set by user" +- name: "determine-storage-classes : Set Default Manage PVC StorageClass (RWX)" set_fact: - mas_app_settings_default_pvc_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(mas_app_settings_default_manage_supported_storage_classes) }}" + mas_app_settings_default_pvc_storage_class: "{{ defaultStorageClasses.rwx }}" -# 3. Debug storage class configuration -# ----------------------------------------------------------------------------- -- name: "determine-storage-classes : Debug Manage PVC storage class configuration" +- name: "determine-storage-classes : Debug Manage PVC storage class" debug: msg: - - "Manage Default PVC Storage class .......... {{ mas_app_settings_default_pvc_storage_class }}" + - "Manage Default PVC Storage class ....... {{ mas_app_settings_default_pvc_storage_class }}" -# 4. Auto define storage classes when required + +# 3. Auto define storage classes when required # ----------------------------------------------------------------------------- -- name: Set doclinks default pvc storage class +- name: "determine-storage-classes : Set doclinks default pvc storage class" + when: mas_app_settings_doclinks_pvc_storage_class is not defined or mas_app_settings_doclinks_pvc_storage_class == "" set_fact: mas_app_settings_doclinks_pvc_storage_class: "{{ mas_app_settings_default_pvc_storage_class }}" - when: - - mas_app_settings_doclinks_pvc_storage_class is not defined or mas_app_settings_doclinks_pvc_storage_class == "" -- name: Set bim default pvc storage class +- name: "determine-storage-classes : Set bim default pvc storage class" + when: mas_app_settings_bim_pvc_storage_class is not defined or mas_app_settings_bim_pvc_storage_class == "" set_fact: mas_app_settings_bim_pvc_storage_class: "{{ mas_app_settings_default_pvc_storage_class }}" - when: - - mas_app_settings_bim_pvc_storage_class is not defined or mas_app_settings_bim_pvc_storage_class == "" -- name: Set jms queue default pvc storage class +- name: "determine-storage-classes : Set jms queue default pvc storage class" + when: mas_app_settings_jms_queue_pvc_storage_class is not defined or mas_app_settings_jms_queue_pvc_storage_class == "" set_fact: mas_app_settings_jms_queue_pvc_storage_class: "{{ mas_app_settings_default_pvc_storage_class }}" - when: - - mas_app_settings_jms_queue_pvc_storage_class is not defined or mas_app_settings_jms_queue_pvc_storage_class == "" -- name: Assert Manage related storage classes are defined + +# 4. Check that all storage classes are defined +# ----------------------------------------------------------------------------- +- name: "determine-storage-classes : Assert Manage related storage classes are defined" assert: that: - mas_app_settings_doclinks_pvc_storage_class is defined @@ -58,12 +47,8 @@ - mas_app_settings_jms_queue_pvc_storage_class is defined - mas_app_settings_jms_queue_pvc_storage_class != '' fail_msg: - - "Failed! One of more storage classes are not defined for Manage!" - - "It was not possible to auto detect a valid storage class as none of the defaulted options were found ....................... {{ mas_app_settings_default_manage_supported_storage_classes }}" - - "Storage Classes available in your cluster .................................................................................. {{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" - - "" - - "Please, ensure you define a valid storage class for the following:" + - "Failed: One of more storage classes are not defined for Manage" - "" - - "mas_app_settings_doclinks_pvc_storage_class' property ...................................................................... {{ mas_app_settings_doclinks_pvc_storage_class | default('Undefined!', true )}}" - - "mas_app_settings_bim_pvc_storage_class' property ........................................................................... {{ mas_app_settings_bim_pvc_storage_class | default('Undefined!', true )}}" - - "mas_app_settings_jms_queue_pvc_storage_class' property ..................................................................... {{ mas_app_settings_jms_queue_pvc_storage_class | default('Undefined!', true )}}" + - Doclinks PVC Storage Class .............. {{ mas_app_settings_doclinks_pvc_storage_class | default('', true) }}" + - BIM PVC Storage Class .................. {{ mas_app_settings_bim_pvc_storage_class | default('', true) }}" + - JMS Queue Storage Class ................. {{ mas_app_settings_jms_queue_pvc_storage_class | default('', true) }}" diff --git a/ibm/mas_devops/roles/suite_app_config/tasks/manage/post-config/main.yml b/ibm/mas_devops/roles/suite_app_config/tasks/manage/post-config/main.yml index 6e5aacc530..dc26e069db 100644 --- a/ibm/mas_devops/roles/suite_app_config/tasks/manage/post-config/main.yml +++ b/ibm/mas_devops/roles/suite_app_config/tasks/manage/post-config/main.yml @@ -7,6 +7,7 @@ - "Attachment Provider .................... {{ mas_manage_attachments_provider }}" - "Instance ID ............................ {{ mas_instance_id }}" - "Workspace ID ........................... {{ mas_workspace_id }}" + - "Application ID ......................... {{ mas_app_id }}" - "Manage DB2 Scope ....................... {{ mas_appws_bindings_jdbc }}" - "DB2 Namespace .......................... {{ db2_namespace }}" - "DB2 Instance Name ...................... {{ db2_instance_name }}" @@ -14,6 +15,12 @@ - "COS Bucket Name ........................ {{ cos_bucket_name }}" - "IBM Cloud Resource Group ............... {{ ibmcloud_resourcegroup }}" +# Manage post-configuration: Image stitching +- name: "Run Manage post-configuration: Set up image stitching" + include_role: + name: ibm.mas_devops.suite_manage_imagestitching_config + when: mas_app_id is in ['manage'] + - name: "Run Manage post-configuration: Set up attachments - Get DB2 Instancd Name" set_fact: db2_instance_name: "{{ mas_appws_bindings_jdbc | ibm.mas_devops.get_db2_instance_name(mas_instance_id, mas_workspace_id, 'manage') }}" diff --git a/ibm/mas_devops/roles/suite_app_config/vars/health.yml b/ibm/mas_devops/roles/suite_app_config/vars/health.yml index 12a6c9176d..917a9cb971 100644 --- a/ibm/mas_devops/roles/suite_app_config/vars/health.yml +++ b/ibm/mas_devops/roles/suite_app_config/vars/health.yml @@ -18,12 +18,6 @@ mas_app_settings_indexspace: "{{ lookup('env', 'MAS_APP_SETTINGS_INDEXSPACE') | # otherwise no persistent storage will be configured at all mas_app_settings_persistent_volumes_flag: "{{ lookup('env', 'MAS_APP_SETTINGS_PERSISTENT_VOLUMES_FLAG') | default(False, true) | bool }}" mas_app_settings_persistent_volumes_default_accessmode: "ReadWriteMany" -mas_app_settings_default_manage_supported_storage_classes: - - ibmc-file-gold-gid - - ocs-storagecluster-cephfs - - azurefiles-premium - - efs - - nfs-client # properties to configure persistent volumes for doclinks i.e attachments mas_app_settings_doclinks_pvc_storage_class: "{{ lookup('env', 'MAS_APP_SETTINGS_DOCLINKS_PVC_STORAGE_CLASS') }}" # if not defined by user, it will be automatically defined while setting persistent storage diff --git a/ibm/mas_devops/roles/suite_app_config/vars/manage.yml b/ibm/mas_devops/roles/suite_app_config/vars/manage.yml index b78c953edd..ca0be0b50f 100644 --- a/ibm/mas_devops/roles/suite_app_config/vars/manage.yml +++ b/ibm/mas_devops/roles/suite_app_config/vars/manage.yml @@ -19,12 +19,6 @@ mas_app_settings_indexspace: "{{ lookup('env', 'MAS_APP_SETTINGS_INDEXSPACE') | # otherwise no persistent storage will be configured at all mas_app_settings_persistent_volumes_flag: "{{ lookup('env', 'MAS_APP_SETTINGS_PERSISTENT_VOLUMES_FLAG') | default(False, true) | bool }}" mas_app_settings_persistent_volumes_default_accessmode: "ReadWriteMany" -mas_app_settings_default_manage_supported_storage_classes: - - ibmc-file-gold-gid - - ocs-storagecluster-cephfs - - azurefiles-premium - - efs - - nfs-client # properties to configure persistent volumes for doclinks i.e attachments mas_app_settings_doclinks_pvc_storage_class: "{{ lookup('env', 'MAS_APP_SETTINGS_DOCLINKS_PVC_STORAGE_CLASS') }}" # if not defined by user, it will be automatically defined while setting persistent storage diff --git a/ibm/mas_devops/roles/suite_app_install/README.md b/ibm/mas_devops/roles/suite_app_install/README.md index 315216847a..15ec9f7712 100644 --- a/ibm/mas_devops/roles/suite_app_install/README.md +++ b/ibm/mas_devops/roles/suite_app_install/README.md @@ -89,7 +89,7 @@ Set the binding scope for the application's JDBC binding (`system` or `applicati - Default: `system` ### mas_app_plan -Optional. Defines what plan will be used in application install. +**Optional**: Defines what plan will be used in application install. - Environment Variable: `MAS_APP_PLAN` - Default: Application-specific, see details below. @@ -107,13 +107,13 @@ Provide the directory where supported pod templates configuration files are defi Role Variables - Visual Inspection Configuration ------------------------------------------------------------------------------- ### mas_app_settings_visualinspection_storage_class -Optional - Storage class used for user data. This must support ReadWriteMany +**Optional**: Storage class used for user data. This must support ReadWriteMany(RWX). - Environment Variable: `MAS_APP_SETTINGS_VISUALINSPECTION_STORAGE_CLASS` - Default: Auto-selected from storage classes installed in the cluster. ### mas_app_settings_visualinspection_storage_size -Optional. Size of data persistent volume. +**Optional**: Size of data persistent volume. - Environment Variable: `MAS_APP_SETTINGS_VISUALINSPECTION_STORAGE_SIZE` - Default: `100Gi` @@ -122,7 +122,7 @@ Optional. Size of data persistent volume. Role Variables - IoT Configuration ------------------------------------------------------------------------------- ### mas_app_settings_iot_deployment_size -Optional, The IoT deployment size, one of `dev`, `small` or `large`. +**Optional**: The IoT deployment size, one of `dev`, `small` or `large`. - Environment Variable: `MAS_APP_SETTINGS_IOT_DEPLOYMENT_SIZE` - Default: `small` @@ -130,15 +130,15 @@ Optional, The IoT deployment size, one of `dev`, `small` or `large`. - IoT 8.6+ ### mas_app_settings_iot_fpl_pvc_storage_class -Optional. The persistent volume storage class used by the iot fpl component for transient state storage - +**Optional**: The persistent volume storage class used by the iot fpl component for transient state storage. +The storage class can be used to dynamically provision a persistent volume with access mode RWO (ReadWriteOnce). - Environment Variable: `MAS_APP_SETTINGS_IOT_FPL_PVC_STORAGE_CLASS` - Default: Auto-selected from storage classes installed in the cluster. - Application Support: - IoT 8.6+ ### mas_app_settings_iot_fpl_router_pvc_size -Optional. The persistent volume size used by the iot fpl pipeline router for transient state storage +**Optional**: The persistent volume size used by the iot fpl pipeline router for transient state storage - Environment Variable: `MAS_APP_SETTINGS_IOT_FPL_ROUTER_PVC_SIZE` - Default: 100Gi. @@ -146,7 +146,7 @@ Optional. The persistent volume size used by the iot fpl pipeline router for tra - IoT 8.6+ ### mas_app_settings_iot_fpl_executor_pvc_size -Optional. The persistent volume size used by the iot fpl pipeline router for transient state storage +**Optional**: The persistent volume size used by the iot fpl pipeline router for transient state storage - Environment Variable: `MAS_APP_SETTINGS_IOT_FPL_EXECUTOR_PVC_SIZE` - Default: 100Gi. @@ -154,15 +154,15 @@ Optional. The persistent volume size used by the iot fpl pipeline router for tra - IoT 8.6+ ### mas_app_settings_iot_mqttbroker_pvc_storage_class -Optional. The persistent volume storage class used by the iot mqtt broker (messagesight) - +**Optional**: The persistent volume storage class used by the iot mqtt broker (messagesight) +The storage class can be used to dynamically provision a persistent volume with access mode RWO (ReadWriteOnce). - Environment Variable: `MAS_APP_SETTINGS_IOT_MQTTBROKER_PVC_STORAGE_CLASS` - Default: Auto-selected from storage classes installed in the cluster, if a default compatible one is found. - Application Support: - IoT 8.3+ ### mas_app_settings_iot_mqttbroker_pvc_size -Optional. The persistent volume size used by the iot mqtt broker (messagesight) +**Optional**: The persistent volume size used by the iot mqtt broker (messagesight) - Environment Variable: `MAS_APP_SETTINGS_IOT_MQTTBROKER_PVC_SIZE` - Default: 100Gi. diff --git a/ibm/mas_devops/roles/suite_app_install/defaults/main.yml b/ibm/mas_devops/roles/suite_app_install/defaults/main.yml index 8154a71de7..a74ba103c5 100644 --- a/ibm/mas_devops/roles/suite_app_install/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_app_install/defaults/main.yml @@ -14,6 +14,10 @@ mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | defaul ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" +# Development Registry Entitlement +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + # MAS application configuration mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" mas_app_plan: "{{ lookup('env', 'MAS_APP_PLAN') }}" diff --git a/ibm/mas_devops/roles/suite_app_install/tasks/iot.yml b/ibm/mas_devops/roles/suite_app_install/tasks/iot.yml index ea499b8c13..881d44925d 100644 --- a/ibm/mas_devops/roles/suite_app_install/tasks/iot.yml +++ b/ibm/mas_devops/roles/suite_app_install/tasks/iot.yml @@ -1,18 +1,8 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "determine-storage-classes : Lookup storage classes" - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Set FPL Storage (Required) @@ -21,11 +11,8 @@ when: - mas_app_settings_iot_fpl_pvc_storage_class is not defined or mas_app_settings_iot_fpl_pvc_storage_class == "" - mas_app_settings_iot_deployment_size != "dev" - vars: - # ROKS, OCS, AWS, Azure - supported_storage_classes: [ibmc-block-gold, ocs-storagecluster-ceph-rbd, gp3-csi, managed-premium] set_fact: - mas_app_settings_iot_fpl_pvc_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(supported_storage_classes) }}" + mas_app_settings_iot_fpl_pvc_storage_class: "{{ defaultStorageClasses.rwo }}" # 3. Set MQTT Broker Storage (Required) @@ -34,11 +21,8 @@ when: - mas_app_settings_iot_mqttbroker_pvc_storage_class is not defined or mas_app_settings_iot_mqttbroker_pvc_storage_class == "" - mas_app_settings_iot_deployment_size != "dev" - vars: - # ROKS, OCS, AWS, Azure - supported_storage_classes: [ibmc-block-gold, ocs-storagecluster-ceph-rbd, gp3-csi, managed-premium] set_fact: - mas_app_settings_iot_mqttbroker_pvc_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(supported_storage_classes) }}" + mas_app_settings_iot_mqttbroker_pvc_storage_class: "{{ defaultStorageClasses.rwo }}" - name: Assert IoT FPL PVC Storage is defined when: mas_app_settings_iot_deployment_size != "dev" @@ -57,20 +41,20 @@ fail_msg: "IoT MQTT Broker Storage for MessageSight (mas_app_settings_iot_mqttbroker_pvc_storage_class property) must be defined when IoT deployment size = '{{ mas_app_settings_iot_deployment_size }}'." -# 3. Debug storage class configuration +# 4. Debug storage class configuration # ----------------------------------------------------------------------------- - name: "determine-storage-classes : Debug IoT PVC storage class configuration" debug: msg: - - "IoT Deployment Size ............... {{ mas_app_settings_iot_deployment_size }}" - - "IoT FPL PVC Storage Class ......... {{ mas_app_settings_iot_fpl_pvc_storage_class }}" - - "IoT FPL PVC Router Size ........... {{ mas_app_settings_iot_fpl_router_pvc_size }}" - - "IoT FPL PVC Executor Size ......... {{ mas_app_settings_iot_fpl_executor_pvc_size }}" - - "IoT MQTT Broker PVC Storage Class . {{ mas_app_settings_iot_mqttbroker_pvc_storage_class }}" - - "IoT MQTT Broker PVC Size .......... {{ mas_app_settings_iot_mqttbroker_pvc_size }}" + - "IoT Deployment Size .................... {{ mas_app_settings_iot_deployment_size }}" + - "IoT FPL PVC Storage Class .............. {{ mas_app_settings_iot_fpl_pvc_storage_class }}" + - "IoT FPL PVC Router Size ................ {{ mas_app_settings_iot_fpl_router_pvc_size }}" + - "IoT FPL PVC Executor Size .............. {{ mas_app_settings_iot_fpl_executor_pvc_size }}" + - "IoT MQTT Broker PVC Storage Class ...... {{ mas_app_settings_iot_mqttbroker_pvc_storage_class }}" + - "IoT MQTT Broker PVC Size ............... {{ mas_app_settings_iot_mqttbroker_pvc_size }}" -# 4. Load PodTemplates configuration +# 5. Load PodTemplates configuration # ----------------------------------------------------------------------------- - name: "Load podTemplates configuration" include_tasks: "{{ role_path }}/../../common_tasks/pod_templates/main.yml" diff --git a/ibm/mas_devops/roles/suite_app_install/tasks/main.yml b/ibm/mas_devops/roles/suite_app_install/tasks/main.yml index cf5bfe6bcc..c0b665abe0 100644 --- a/ibm/mas_devops/roles/suite_app_install/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_app_install/tasks/main.yml @@ -57,18 +57,23 @@ oc adm policy add-scc-to-user anyuid system:serviceaccount:{{ mas_app_namespace }}:ibm-mas-visualinspection-operator -# 5. Install the operator +# 5. Create entitlement secret and install the operator # ----------------------------------------------------------------------------- -- name: "Install Operator" - include_role: - name: ibm.mas_devops.install_operator - vars: +- name: "Create IBM Entitlement Key" + ibm.mas_devops.update_ibm_entitlement: namespace: "{{ mas_app_namespace }}" icr_username: "{{ mas_entitlement_username }}" icr_password: "{{ mas_entitlement_key }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + +- name: "Create ibm-mas Subscription" + ibm.mas_devops.apply_subscription: + namespace: "{{ mas_app_namespace }}" + package_name: "ibm-mas-{{ 'manage' if (mas_app_id == 'health') else mas_app_id }}" + package_channel: "{{ mas_app_channel }}" catalog_source: "{{ mas_app_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/subscription.yml.j2') }}" + register: subscription # 6. Wait until the Application's CRD is available diff --git a/ibm/mas_devops/roles/suite_app_install/tasks/visualinspection.yml b/ibm/mas_devops/roles/suite_app_install/tasks/visualinspection.yml index 9ba601d50f..7a065d9f5e 100644 --- a/ibm/mas_devops/roles/suite_app_install/tasks/visualinspection.yml +++ b/ibm/mas_devops/roles/suite_app_install/tasks/visualinspection.yml @@ -1,20 +1,8 @@ --- -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" +- name: "visualinspection : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Set Storage (Required) @@ -22,7 +10,7 @@ - name: Default Storage if not set by user when: mas_app_settings_visualinspection_storage_class is not defined or mas_app_settings_visualinspection_storage_class == "" set_fact: - mas_app_settings_visualinspection_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwx) }}" + mas_app_settings_visualinspection_storage_class: "{{ defaultStorageClasses.rwx }}" - name: Assert that storage class has been defined assert: @@ -42,4 +30,4 @@ - name: "Debug visualinspection storage class configuration" debug: msg: - - "Storage class (visualinspection) ...... {{ mas_app_settings_visualinspection_storage_class }}" + - "Storage class (visualinspection) ....... {{ mas_app_settings_visualinspection_storage_class }}" diff --git a/ibm/mas_devops/roles/suite_app_install/templates/operator-group.yml.j2 b/ibm/mas_devops/roles/suite_app_install/templates/operator-group.yml.j2 deleted file mode 100644 index 6e27d27f47..0000000000 --- a/ibm/mas_devops/roles/suite_app_install/templates/operator-group.yml.j2 +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: {{ mas_app_namespace }}-operator-group - namespace: {{ mas_app_namespace }} -{% if custom_labels is defined and custom_labels.items() %} - labels: -{% for key, value in custom_labels.items() %} - "{{ key }}": "{{ value }}" -{% endfor %} -{% endif %} -spec: - targetNamespaces: - - {{ mas_app_namespace }} diff --git a/ibm/mas_devops/roles/suite_app_install/templates/subscription.yml.j2 b/ibm/mas_devops/roles/suite_app_install/templates/subscription.yml.j2 deleted file mode 100644 index 55c92d3e4a..0000000000 --- a/ibm/mas_devops/roles/suite_app_install/templates/subscription.yml.j2 +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: ibm-mas-{{ mas_app_id }} - namespace: {{ mas_app_namespace }} -{% if custom_labels is defined and custom_labels.items() %} - labels: -{% for key, value in custom_labels.items() %} - "{{ key }}": "{{ value }}" -{% endfor %} -{% endif %} -spec: - channel: {{ mas_app_channel }} - installPlanApproval: Automatic - name: ibm-mas-{{ 'manage' if (mas_app_id == 'health') else mas_app_id }} - source: {{ mas_app_catalog_source }} - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/suite_backup_restore/README.md b/ibm/mas_devops/roles/suite_backup_restore/README.md index eeb928012d..35792f40b3 100644 --- a/ibm/mas_devops/roles/suite_backup_restore/README.md +++ b/ibm/mas_devops/roles/suite_backup_restore/README.md @@ -46,82 +46,13 @@ Set the [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - Environment Variable: `MASBR_JOB_TIMEZONE` - Default: None -### masbr_storage_type -Set `local` or `cloud` to indicate this role to save the backup files to local file system or cloud object storage. - -- **Required** -- Environment Variable: `MASBR_STORAGE_TYPE` -- Default: None - ### masbr_storage_local_folder Set local path to save the backup files. -- **Required** only when `MASBR_STORAGE_TYPE=local` +- **Required** - Environment Variable: `MASBR_STORAGE_LOCAL_FOLDER` - Default: None -### masbr_storage_cloud_rclone_file -Set the path of `rclone.conf` file. - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_RCLONE_FILE` -- Default: None - -### masbr_storage_cloud_rclone_name -Set the configuration name defined in `rclone.conf` file. - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_RCLONE_NAME` -- Default: None - -### masbr_storage_cloud_bucket -Set the object storage bucket name for saving the backup files - -- **Required** only when `MASBR_STORAGE_TYPE=cloud` -- Environment Variable: `MASBR_STORAGE_CLOUD_BUCKET` -- Default: None - -### masbr_slack_enabled -Set `true` or `false` to indicate whether this role will send Slack notification messages of the backup and restore progress. - -- Optional -- Environment Variable: `MASBR_SLACK_ENABLED` -- Default: `false` - -### masbr_slack_level -Set `failure`, `info` or `verbose` to indicate this role to send Slack notification messages in which backup and resore phases: - -| Slack level | Backup/Restore phases | -| ----------- | ------------------------------------------------------- | -| failure | `Failed`, `PartiallyFailed` | -| info | `Completed`, `Failed`, `PartiallyFailed` | -| verbose | `InProgress`, `Completed`, `Failed`, `PartiallyFailed` | - -- Optional -- Environment Variable: `MASBR_SLACK_LEVEL` -- Default: `info` - -### masbr_slack_token -The Slack integration token. - -- **Required** only when `MASBR_SLACK_ENABLED=true` -- Environment Variable: `MASBR_SLACK_TOKEN` -- Default: None - -### masbr_slack_channel -The Slack channel to send the notification messages to. - -- **Required** only when `MASBR_SLACK_ENABLED=true` -- Environment Variable: `MASBR_SLACK_CHANNEL` -- Default: None - -### masbr_slack_user -The sender of the Slack notification message. - -- Optional -- Environment Variable: `MASBR_SLACK_USER` -- Default: `MASBR` - Role Variables - Backup ------------------------------------------------------------------------------- @@ -155,7 +86,6 @@ Backup MAS Core namespace resources, note that this does not include backup of a vars: masbr_action: backup mas_instance_id: main - masbr_storage_type: local masbr_storage_local_folder: /tmp/masbr roles: - ibm.mas_devops.suite_backup_restore @@ -171,7 +101,6 @@ Restore MAS Core namespace resources, note that this does not include backup of masbr_action: restore masbr_restore_from_version: 20240621021316 mas_instance_id: main - masbr_storage_type: local masbr_storage_local_folder: /tmp/masbr roles: - ibm.mas_devops.suite_backup_restore diff --git a/ibm/mas_devops/roles/suite_backup_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_backup_restore/tasks/main.yml index abcc0a21ab..d22d41f048 100644 --- a/ibm/mas_devops/roles/suite_backup_restore/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_backup_restore/tasks/main.yml @@ -38,22 +38,7 @@ _component_before_task_path: "{{ role_path }}/tasks/get-suite-info.yml" -# Create k8s Job to run backup/restore tasks -# ----------------------------------------------------------------------------- -- name: "Create k8s Job to run {{ masbr_action }} tasks" - when: masbr_create_task_job - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_run_tasks_job.yml" - vars: - _rt_role_name: "suite_backup_restore" - _rt_env: - - name: "MASBR_JOB_VERSION" - value: "{{ masbr_job_version }}" - - name: "MAS_INSTANCE_ID" - value: "{{ mas_instance_id }}" - - - name: "Run {{ masbr_action }} tasks" - when: not masbr_create_task_job block: # Update job status: New # ------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-namespace.yml b/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-namespace.yml index 4c96bb0d8f..771a12681b 100644 --- a/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-namespace.yml +++ b/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-namespace.yml @@ -53,7 +53,7 @@ shell: > mkdir -p {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }} && tar -xzf {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }}.tar.gz - -C {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }} . && + -C {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }} && ls -lA {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }} register: _extract_output @@ -66,7 +66,20 @@ # Restore namespace resoruces # ------------------------------------------------------------------------- - # (TODO) + # Loop through the folder + - name: "Get the list of files from restore directory" + find: + paths: "{{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }}" + patterns: '*.yml,*.yaml' + recurse: no + register: find_result + + - name: "Apply configs" + kubernetes.core.k8s: + state: present + definition: "{{ lookup('template', item.path) }}" + with_items: "{{ find_result.files }}" + when: find_result is defined # Update database restore status: Completed diff --git a/ibm/mas_devops/roles/suite_db2_setup_for_manage/tasks/apply-db2-config-version.yml b/ibm/mas_devops/roles/suite_db2_setup_for_manage/tasks/apply-db2-config-version.yml index 601ef80358..23bd024d01 100644 --- a/ibm/mas_devops/roles/suite_db2_setup_for_manage/tasks/apply-db2-config-version.yml +++ b/ibm/mas_devops/roles/suite_db2_setup_for_manage/tasks/apply-db2-config-version.yml @@ -23,15 +23,6 @@ msg: - "Pod name ............................... {{ db2_pod_name }}" -- name: "Creating MIRRORLOGPATH folder in {{ db2_pod_name }}" - shell: | - oc exec -it -n {{ db2_namespace }} {{ db2_pod_name }} -- su -lc "mkdir -p /mnt/backup/MIRRORLOGPATH" db2inst1 - register: creating_mirrorlog_folder_output - -- fail: msg="Failed to create path for MIRRORLOGPATH" - when: - - creating_mirrorlog_folder_output.rc != 0 - - name: "Creating LOGARCHMETH1 folder in {{ db2_pod_name }}" shell: | oc exec -it -n {{ db2_namespace }} {{ db2_pod_name }} -- su -lc "mkdir -p /mnt/bludata0/db2/archive_log/" db2inst1 diff --git a/ibm/mas_devops/roles/suite_db2_setup_for_manage/vars/main.yml b/ibm/mas_devops/roles/suite_db2_setup_for_manage/vars/main.yml index 74364cb72c..22659bd190 100644 --- a/ibm/mas_devops/roles/suite_db2_setup_for_manage/vars/main.yml +++ b/ibm/mas_devops/roles/suite_db2_setup_for_manage/vars/main.yml @@ -27,7 +27,7 @@ db2_configs: LOGSECOND: '156' LOGFILSIZ: '32768' LOGARCHMETH1: 'DISK:/mnt/bludata0/db2/archive_log/' - MIRRORLOGPATH: '/mnt/backup/MIRRORLOGPATH' + MIRRORLOGPATH: '/mnt/backup' STMT_CONC: 'LITERALS' DDL_CONSTRAINT_DEF: 'YES' TRACKMOD: 'YES' @@ -41,8 +41,6 @@ db2_configs: AUTO_REORG: 'OFF' AUTO_DB_BACKUP: 'OFF' WLM_ADMISSION_CTRL: 'NO' - SHEAPTHRES_SHR: 'automatic' - SORTHEAP: 'automatic' AUTHN_CACHE_USERS: '100' AUTHN_CACHE_DURATION: '10' instance: diff --git a/ibm/mas_devops/roles/suite_install/defaults/main.yml b/ibm/mas_devops/roles/suite_install/defaults/main.yml index 8a7408cceb..7d227722a0 100644 --- a/ibm/mas_devops/roles/suite_install/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_install/defaults/main.yml @@ -54,6 +54,10 @@ mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | defaul ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" +# Development Registry Entitlement +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + # Data Dictionary # ----------------------------------------------------------------------------- mas_add_catalog: "{{ lookup('env', 'MAS_ADD_CATALOG') | default('ibm-operator-catalog', true) }}" diff --git a/ibm/mas_devops/roles/suite_install/tasks/main.yml b/ibm/mas_devops/roles/suite_install/tasks/main.yml index c699062a09..3d36c16f60 100644 --- a/ibm/mas_devops/roles/suite_install/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_install/tasks/main.yml @@ -176,18 +176,23 @@ - "IPv6 Enabled .................. {{ enable_ipv6 }}" -# 11. Install the operator & create entitlement secret +# 11. Create entitlement secret and install the operator # ----------------------------------------------------------------------------- -- name: "Install MAS Operator" - include_role: - name: ibm.mas_devops.install_operator - vars: +- name: "Create IBM Entitlement Key" + ibm.mas_devops.update_ibm_entitlement: namespace: "{{ mas_namespace }}" icr_username: "{{ mas_entitlement_username }}" icr_password: "{{ mas_entitlement_key }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + +- name: "Create ibm-mas Subscription" + ibm.mas_devops.apply_subscription: + namespace: "{{ mas_namespace }}" + package_name: ibm-mas + package_channel: "{{ mas_channel }}" catalog_source: "{{ mas_catalog_source }}" - operator_group: "{{ lookup('template', 'templates/operator-group.yml.j2') }}" - subscription: "{{ lookup('template', 'templates/subscription.yml.j2') }}" + register: subscription # 12. Wait until the Suite CRD is available diff --git a/ibm/mas_devops/roles/suite_install/templates/operator-group.yml.j2 b/ibm/mas_devops/roles/suite_install/templates/operator-group.yml.j2 deleted file mode 100644 index e462e6ed17..0000000000 --- a/ibm/mas_devops/roles/suite_install/templates/operator-group.yml.j2 +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: ibm-mas-operator-group - namespace: "{{ mas_namespace }}" -{% if custom_labels is defined and custom_labels.items() %} - labels: -{% for key, value in custom_labels.items() %} - "{{ key }}": "{{ value }}" -{% endfor %} -{% endif %} -spec: - targetNamespaces: - - "{{ mas_namespace }}" diff --git a/ibm/mas_devops/roles/suite_install/templates/subscription.yml.j2 b/ibm/mas_devops/roles/suite_install/templates/subscription.yml.j2 deleted file mode 100644 index d6274123fd..0000000000 --- a/ibm/mas_devops/roles/suite_install/templates/subscription.yml.j2 +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: ibm-mas-operator - namespace: "{{ mas_namespace }}" -{% if custom_labels is defined and custom_labels.items() %} - labels: -{% for key, value in custom_labels.items() %} - "{{ key }}": "{{ value }}" -{% endfor %} -{% endif %} -spec: - channel: "{{ mas_channel }}" - installPlanApproval: Automatic - name: ibm-mas - source: "{{ mas_catalog_source }}" - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/suite_manage_attachments_config/README.md b/ibm/mas_devops/roles/suite_manage_attachments_config/README.md index 5fa22d03e3..fc824447e5 100644 --- a/ibm/mas_devops/roles/suite_manage_attachments_config/README.md +++ b/ibm/mas_devops/roles/suite_manage_attachments_config/README.md @@ -18,7 +18,7 @@ Available options are: - `ibm`: Configures IBM Cloud Object Storage as storage system for Manage attachments. - `aws`: Configures Amazon S3 buckets as storage system for Manage attachments. - **Note:** If using `ibm` or `aws` as attachments provider, the [`cos_bucket`](../roles/cos_bucket.md) role will be executed to setup a new or existing targeted COS bucket to be used to store Manage attachments, therefore make sure you set the expected variables to customize your COS bucket for Manage attachments, i.e. `IBMCLOUD_APIKEY` and `COS_INSTANCE_NAME`. + **Note:** If using `ibm` or `aws` as attachments provider, the [`cos_bucket`](../roles/cos_bucket.md) role will be executed to setup a new or existing targeted COS bucket to be used to store Manage attachments, therefore make sure you set the expected variables to customize your COS bucket for Manage attachments, i.e. `COS_APIKEY` and `COS_INSTANCE_NAME`. **Note about S3:** To run this role successfully for AWS s3 buckets, you must have already installed the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).Also, you need to have AWS user credentials configured via `aws configure` command or simply export `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables with your corresponding AWS username credentials prior running this role. diff --git a/ibm/mas_devops/roles/suite_manage_imagestitching_config/README.md b/ibm/mas_devops/roles/suite_manage_imagestitching_config/README.md new file mode 100644 index 0000000000..7ff4cdedb4 --- /dev/null +++ b/ibm/mas_devops/roles/suite_manage_imagestitching_config/README.md @@ -0,0 +1,82 @@ +suite_manage_imagestitching_config +=== + +This role configures the manage workspace to autodeploy the image stitching application. The role will only make changes if the Civil component has been installed. It will configure a PVC, patch the ManageWorkspace CR with the name of the PVC and also patch PV specifications based on the PVC. It also sets two key system properties required by the image stitching application: 'mci.imagestitching.apiurl' and 'imagestitching.dataInputPath'. + +**Note:** This role should be executed **after** Manage application is deployed and activated as it needs Manage up and running in order to set system properties. + +Role Variables +-------------- + +### stitching_pvcname +Required. The postfix for the PVC created for image stitching. The ManageWorkspace CR will prepend 'mas_instance_id-mas_workspace_id-' to generate the actual name of the PVC + +- Environment Variable: `IMAGESTITCHING_PVCNAME` +- Default Value: manage-imagestitching + +### stitching_storage_class +**Optional**: The storage class used for the PVC. If not specified the default value will be found by discovery. The storage class must support ReadWriteMany(RWX) access mode. + +- Environment Variable: `IMAGESTITCHING_STORAGE_CLASS` +- Default Value: None + +### stitching_storage_size +**Required**: The size of the persistent volume claim. + +- Environment Variable: `IMAGESTITCHING_STORAGE_SIZE` +- Default Value: 20 Gi + +### stitching_storage_mode +**Required**: The access mode for the PVC. + +- Environment Variable: `IMAGESTITCHING_STORAGE_MODE` +- Default Value: ReadWriteMany + +### stitching_storage_mountpath +**Required**: The mount path of the Persistent Volume. + +- Environment Variable: `IMAGESTITCHING_STORAGE_MOUNTPATH` +- Default Value: imagestitching + +### mas_instance_id +**Required**:. The instance ID of Maximo Application Suite. This will be used to lookup for Manage application resources. + +- Environment Variable: `MAS_INSTANCE_ID` +- Default Value: None + +### mas_workspace_id +**Required**:. The workspace ID of Maximo Application Suite. This will be used to lookup for Manage application resources. + +- Environment Variable: `MAS_WORKSPACE_ID` +- Default Value: None + +### mas_domain +**Required**: The domain name for the Manage cluster + +- Environment Variable: `MAS_DOMAIN` +- Default Value: Discovered from Suite CR + +Example Playbook +---------------- +The following sample will configure image stitching for an existing Manage application instance via ManageWorkspace CR update: + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: civil + mas_workspace_id: masdev + mas_domain: civil.ibmmasdev.com + stitching_pvcname: manage-imagestitching + stitching_storage_class: nfs-client + stitching_storage_size: 20Gi + stitching_storage_mode: ReadWriteMany + stitching_storage_mountpath: imagestitching + roles: + - ibm.mas_devops.suite_manage_imagestitching_config +``` + +License +------- + +EPL-2.0 diff --git a/ibm/mas_devops/roles/suite_manage_imagestitching_config/defaults/main.yml b/ibm/mas_devops/roles/suite_manage_imagestitching_config/defaults/main.yml new file mode 100644 index 0000000000..aa1bc62eb6 --- /dev/null +++ b/ibm/mas_devops/roles/suite_manage_imagestitching_config/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# MAS Domain +# ----------------------------------------------------------------------------- +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" +mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" +manage_namespace: "mas-{{ mas_instance_id }}-manage" +mas_domain: "{{ lookup('env', 'MAS_DOMAIN') }}" +mas_ws_cr_name: "{{ mas_instance_id }}-{{ mas_workspace_id }}" + +# Image stitching properties +# ----------------------------------------------------------------------------- +stitching_pvcname: "{{ lookup('env', 'IMAGESTITCHING_PVCNAME') | default('manage-imagestitching', true) }}" +stitching_storage_class: "{{ lookup('env', 'IMAGESTITCHING_STORAGE_CLASS') }}" +stitching_storage_size: "{{ lookup('env', 'IMAGESTITCHING_STORAGE_SIZE') | default('20Gi', true) }}" +stitching_storage_mode: "{{ lookup('env', 'IMAGESTITCHING_STORAGE_MODE') | default('ReadWriteMany', true) }}" +stitching_storage_mountpath: "{{ lookup('env', 'IMAGESTITCHING_STORAGE_MOUNTPATH') | default('imagestitching', true) }}" diff --git a/ibm/mas_devops/roles/install_operator/meta/main.yml b/ibm/mas_devops/roles/suite_manage_imagestitching_config/meta/main.yml similarity index 58% rename from ibm/mas_devops/roles/install_operator/meta/main.yml rename to ibm/mas_devops/roles/suite_manage_imagestitching_config/meta/main.yml index 75c062f8f1..f729a07d3d 100644 --- a/ibm/mas_devops/roles/install_operator/meta/main.yml +++ b/ibm/mas_devops/roles/suite_manage_imagestitching_config/meta/main.yml @@ -1,6 +1,6 @@ galaxy_info: - author: David Parker (@durera) - description: Install an operator, supports both pre-release and released operator installation (access to pre-release operator builds is limited) + author: Brendan More O'Ferrall (@mooreof) + description: Configure ibm cloud object storage as persistent storage for Manage application imagestitching files. company: IBM license: EPL-2.0 @@ -16,6 +16,7 @@ galaxy_info: - ibm - mas - devops + - rhocp dependencies: - role: ibm.mas_devops.ansible_version_check diff --git a/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/configure-stitching.yml b/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/configure-stitching.yml new file mode 100644 index 0000000000..5063bce5f6 --- /dev/null +++ b/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/configure-stitching.yml @@ -0,0 +1,195 @@ +--- +# 1. Lookup storage class availabiity +# ----------------------------------------------------------------------------- +- name: "configure-stitching : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" + + +# 2. Set Defaults +# ----------------------------------------------------------------------------- +- name: Storage class if not set by user (ReadWriteMany) + when: stitching_storage_class is not defined or stitching_storage_class == "" + set_fact: + stitching_storage_class: "{{ defaultStorageClasses.rwx }}" + +- name: Assert that a storage class has been defined + assert: + that: stitching_storage_class is defined and stitching_storage_class != "" + fail_msg: "stitching_storage_class must be defined" + +- name: "Fail if stitching_pvcname is not provided" + assert: + that: stitching_pvcname is defined and stitching_pvcname != "" + fail_msg: "stitching_pvcname property is required" + +- name: "Fail if stitching_storage_mountpath is not provided" + assert: + that: stitching_storage_mountpath is defined and stitching_storage_mountpath != "" + fail_msg: "stitching_storage_mountpath property is required" + +- name: "Fail if stitching_storage_size is not provided" + assert: + that: stitching_storage_size is defined and stitching_storage_size != "" + fail_msg: "stitching_storage_size property is required" + +- name: "Fail if stitching_storage_mode is not provided" + assert: + that: stitching_storage_mode is defined and stitching_storage_mode != "" + fail_msg: "stitching_storage_mode property is required" + +- name: "Get MAS instance domain" + kubernetes.core.k8s_info: + api_version: "core.mas.ibm.com/v1" + kind: "Suite" + name: "{{ mas_instance_id }}" + namespace: "mas-{{mas_instance_id}}-core" + register: suite_cr_result + retries: 3 + delay: 0 + until: + - suite_cr_result.resources[0].spec.domain is defined + - suite_cr_result.resources[0].spec.domain | length > 0 + +- name: Set MAS domain + set_fact: + mas_domain: "{{ suite_cr_result.resources[0].spec.domain }}" + when: mas_domain is not defined or mas_domain == "" + +- name: "Run Manage image stitching post-configuration: Debug" + debug: + msg: + - "Stitching PVC name ..................... {{ stitching_pvcname }}" + - "Stitching storage class ................ {{ stitching_storage_class }}" + - "Stitching storage size ................. {{ stitching_storage_size }}" + - "Stitching storage mode ................. {{ stitching_storage_mode }}" + - "Stitching storage mountpath ............ {{ stitching_storage_mountpath }}" + - "Instance ID ............................ {{ mas_instance_id }}" + - "Workspace ID ........................... {{ mas_workspace_id }}" + - "Manage namespace ....................... {{ manage_namespace }}" + - "MAS domain ............................. {{ mas_domain }}" + - "MAS workspace CR name .................. {{ mas_ws_cr_name }}" + +- name: Retrieve Manage workspace CR + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_ws_cr_name }}" + namespace: "{{ manage_namespace }}" + kind: ManageWorkspace + register: manageWorkspace + +- name: Retrieve Manage workspace CR + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_ws_cr_name }}" + namespace: "{{ manage_namespace }}" + kind: ManageWorkspace + register: workspaceCR + +- name: Retrieve Image Stitching CR if available + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + kind: Imagestitching + register: stitchingCR + +- name: Retrieve current stitching status + set_fact: + deployimagestitching: "{{ manageWorkspace.resources[0].status.components.civil is defined and manageWorkspace.resources[0].status.components.civil.deployimagestitching }}" + stitchingcrReady: "{{ stitchingCR is defined and stitchingCR.resources | length > 0 }}" + +- name: Debug + debug: + msg: "Stitching CR ready: {{ stitchingcrReady }}, deployimagestitching: {{ deployimagestitching }}" + +- name: Check if stitching PVC has already been created + kubernetes.core.k8s_info: + api_version: v1 + kind: PersistentVolumeClaim + name: "{{ mas_instance_id }}-{{ mas_workspace_id }}-{{ stitching_pvcname }}" + namespace: "{{ manage_namespace }}" + register: stitchingPvc + +- name: Create stitching PVC + when: + - stitchingPvc.resources | length == 0 + kubernetes.core.k8s: + state: present + template: templates/imagestitching-pvc.yml.j2 + +- name: Check if PV has been defined in workspace + set_fact: + stitchingpvAdded: "{{ manageWorkspace.resources | length > 0 and manageWorkspace.resources[0].spec.settings.deployment.persistentVolumes is defined and manageWorkspace.resources[0].spec.settings.deployment.persistentVolumes|selectattr('mountPath', 'contains', stitching_storage_mountpath)|length > 0 }}" + +- name: Get current PV count + set_fact: + pvcount: "{{ manageWorkspace.resources[0].spec.settings.deployment.persistentVolumes|length }}" + +- name: Patch Manage Workspace CR with persistent volume definition + when: not stitchingpvAdded + kubernetes.core.k8s_json_patch: + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_ws_cr_name }}" + namespace: "{{ manage_namespace }}" + kind: ManageWorkspace + patch: + - op: add + path: "/spec/settings/deployment/persistentVolumes/{{ pvcount }}" + value: + accessModes: + - "{{ stitching_storage_mode }}" + mountPath: "/{{ stitching_storage_mountpath }}" + pvcName: "{{ stitching_pvcname }}" + size: "{{ stitching_storage_size }}" + storageClassName: "{{ stitching_storage_class }}" + register: patchWorkspace + retries: 10 + delay: 10 + until: + - patchWorkspace.result is defined + - patchWorkspace.result | length > 0 + +- name: Patch Manage Workspace CR with stitching pvc name + when: + - not deployimagestitching + - not stitchingcrReady + kubernetes.core.k8s: + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_ws_cr_name }}" + namespace: "{{ manage_namespace }}" + kind: ManageWorkspace + definition: + spec: + components: + civil: + imagestitchingpvcname: "{{ stitching_pvcname }}" + register: manageWorkspaceUpdate + retries: 10 + delay: 10 + until: + - manageWorkspaceUpdate.result is defined + - manageWorkspaceUpdate.result | length > 0 + +- name: Set system properties for stitching + include_tasks: tasks/set-imagestitching-properties.yml + +- name: Wait 90 mins for image stitching deployment to complete + ignore_errors: true + kubernetes.core.k8s_info: + api_version: v1 + label_selectors: mas.ibm.com/appType=serverBundle + namespace: "{{ manage_namespace }}" + kind: Pod + register: managePodCheck + retries: 90 + delay: 90 + until: + - managePodCheck.resources | length > 0 + - managePodCheck.resources[0].spec.containers | length > 0 + - managePodCheck.resources[0].spec.containers[0].volumeMounts | selectattr('mountPath', 'contains', stitching_storage_mountpath) | length > 0 + +- name: Get stitching status + set_fact: + stitchingDeployed: "{{ managePodCheck.resources | length > 0 and managePodCheck.resources[0].spec.containers | length > 0 and managePodCheck.resources[0].spec.containers[0].volumeMounts | selectattr('mountPath', 'contains', stitching_storage_mountpath) | length > 0 }}" + +- name: Debug + debug: + msg: "Stitching deployed successfully: {{ stitchingDeployed }}" diff --git a/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/main.yml b/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/main.yml new file mode 100644 index 0000000000..49e1a6a734 --- /dev/null +++ b/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Retrieve Manage workspace CR + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_ws_cr_name }}" + namespace: "{{ manage_namespace }}" + kind: ManageWorkspace + register: workspaceCR + retries: 3 + delay: 10 + until: + - workspaceCR.resources | length > 0 + +- name: Check if Civil is enabled and image stitching is available + set_fact: + stitchingAvailable: "{{ workspaceCR.resources[0].status.components.civil is defined and workspaceCR.resources[0].status.components.civil.enabled and workspaceCR.resources[0].status.components.civil.deployimagestitching is defined }}" + +- name: Configure stitching + include_tasks: tasks/configure-stitching.yml + when: + - stitchingAvailable diff --git a/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/manage-set-system-property.yml b/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/manage-set-system-property.yml new file mode 100644 index 0000000000..0966a9cccc --- /dev/null +++ b/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/manage-set-system-property.yml @@ -0,0 +1,56 @@ +--- +- name: Set Manage API endpoint + set_fact: + manage_endpoint_get: "https://{{ mas_instance_id }}-{{ mas_workspace_id }}.mas-{{ mas_instance_id }}-manage.svc/maximo/api/service/system?action=wsmethod:getProperty&ccm=1&propName={{ property_to_set.name }}" + manage_endpoint_set: "https://{{ mas_instance_id }}-{{ mas_workspace_id }}.mas-{{ mas_instance_id }}-manage.svc/maximo/api/service/system?action=wsmethod:setProperty&ccm=1" + +- name: System Property to set + debug: + msg: + - "manage_endpoint_set: .............. {{ manage_endpoint_set }}" + - "System property: ........... {{ property_to_set }}" + +- name: Call the Manage API to get current system property value + ignore_errors: True + uri: + method: GET + url: "{{ manage_endpoint_get }}" + validate_certs: no + client_key: "{{ client_key_file }}" + client_cert: "{{ client_cert_file }}" + force_basic_auth: no + return_content: yes + status_code: + - 200 + - 404 + - -1 + timeout: 10 + register: get_system_prop_result + +- name: Set current 'current_property_value' Fact + set_fact: + current_property_value: "{{ get_system_prop_result.json.return | default('') }}" + +- name: Current property value + debug: + msg: + - "Value: .............. {{ current_property_value }}" + +- name: Call the Manage API to set system property if it was not set + when: + - property_to_set.value != '' and ( current_property_value == '' or current_property_value != property_to_set.value ) + ignore_errors: True + uri: + method: POST + url: "{{ manage_endpoint_set }}" + body: "{ 'serverName': 'COMMON', 'propName': '{{ property_to_set.name }}', 'propValue': '{{ property_to_set.value }}' }" + body_format: json + validate_certs: no + client_key: "{{ client_key_file }}" + client_cert: "{{ client_cert_file }}" + force_basic_auth: no + return_content: yes + status_code: + - 200 + - 404 + - -1 diff --git a/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/set-imagestitching-properties.yml b/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/set-imagestitching-properties.yml new file mode 100644 index 0000000000..2f6489bd6c --- /dev/null +++ b/ibm/mas_devops/roles/suite_manage_imagestitching_config/tasks/set-imagestitching-properties.yml @@ -0,0 +1,53 @@ +--- +- name: Stitching system properties to be set + set_fact: + system_properties_to_set: + - name: "mci.imagestitching.apiurl" + value: "http://{{ mas_instance_id }}-{{ mas_workspace_id }}-imagestitching.mas-{{ mas_instance_id }}-manage.svc:8080/maximo/stitching" + - name: "imagestitching.dataInputPath" + value: "/{{ stitching_storage_mountpath }}" + +- name: Stitching system Properties to set + debug: + msg: + - "Stitching system properties: ....... {{ system_properties_to_set }}" + +- name: Retrieve manage certificate secret + kubernetes.core.k8s_info: + api: v1 + kind: Secret + namespace: "{{ manage_namespace }}" + name: "{{ mas_instance_id }}-internal-manage-tls" + register: managecerts + +- name: Extract cert and key + when: + - managecerts is defined and managecerts.resources | length > 0 + set_fact: + manage_client_cert: "{{ managecerts.resources[0].data['tls.crt'] | b64decode }}" + manage_client_key: "{{ managecerts.resources[0].data['tls.key'] | b64decode }}" + +- name: Save key to temp file + when: + - manage_client_key is defined + ansible.builtin.copy: + dest: "/tmp/{{ mas_workspace_id }}-stitching-tls.key" + content: "{{ manage_client_key }}" + +- name: Save cert to temp file + when: + - manage_client_cert is defined + ansible.builtin.copy: + dest: "/tmp/{{ mas_workspace_id }}-stitching-tls.crt" + content: "{{ manage_client_cert }}" + +- name: Set System Properties in Manage + when: + - manage_client_key is defined + - manage_client_cert is defined + vars: + property_to_set: "{{ item }}" + client_key_file: "/tmp/{{ mas_workspace_id }}-stitching-tls.key" + client_cert_file: "/tmp/{{ mas_workspace_id }}-stitching-tls.crt" + include_tasks: manage-set-system-property.yml + with_items: "{{ system_properties_to_set }}" diff --git a/ibm/mas_devops/roles/suite_manage_imagestitching_config/templates/imagestitching-pvc.yml.j2 b/ibm/mas_devops/roles/suite_manage_imagestitching_config/templates/imagestitching-pvc.yml.j2 new file mode 100644 index 0000000000..7b591a8d8a --- /dev/null +++ b/ibm/mas_devops/roles/suite_manage_imagestitching_config/templates/imagestitching-pvc.yml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "{{ mas_instance_id }}-{{ mas_workspace_id }}-{{ stitching_pvcname }}" + namespace: "{{ manage_namespace }}" +spec: + resources: + requests: + storage: "{{ stitching_storage_size }}" + volumeMode: Filesystem + storageClassName: "{{ stitching_storage_class }}" + accessModes: + - "{{ stitching_storage_mode }}" + diff --git a/ibm/mas_devops/roles/suite_manage_pvc_config/README.md b/ibm/mas_devops/roles/suite_manage_pvc_config/README.md index d82ddb0825..3903004afd 100644 --- a/ibm/mas_devops/roles/suite_manage_pvc_config/README.md +++ b/ibm/mas_devops/roles/suite_manage_pvc_config/README.md @@ -59,6 +59,7 @@ Optional. Mouth path of the Persistent Volume in the Manage server container. ### mas_app_settings_custom_persistent_volume_sc_name Optional. Persistent Volume Claim Storage Class. If not set, it will be automatically defined accordingly to your cluster's available storage classes. + Both ReadWriteMany(RWX) and ReadWriteOnce(RWO) are supported. - Environment Variable: `MAS_APP_SETTINGS_CUSTOM_PV_STORAGE_CLASS` - Default Value: None. diff --git a/ibm/mas_devops/roles/suite_manage_pvc_config/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/suite_manage_pvc_config/tasks/determine-storage-classes.yml index d40e2e5d23..c5e15de25e 100644 --- a/ibm/mas_devops/roles/suite_manage_pvc_config/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/suite_manage_pvc_config/tasks/determine-storage-classes.yml @@ -1,29 +1,17 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "determine-storage-classes : Lookup storage classes" - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" -- name: "Debug available storage classes" - debug: - msg: "{{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" # 2. Set Primary Storage (Required) # ----------------------------------------------------------------------------- - name: "determine-storage-classes : Default Manage PVC Storage if not set by user" when: - mas_app_settings_custom_persistent_volume_sc_name is not defined or mas_app_settings_custom_persistent_volume_sc_name == "" - vars: - # ROKS, OCS, Azure - supported_storage_classes: [ibmc-file-gold-gid, ocs-storagecluster-cephfs, nfs-client, azurefiles-premium] set_fact: - mas_app_settings_custom_persistent_volume_sc_name: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(supported_storage_classes) }}" + mas_app_settings_custom_persistent_volume_sc_name: "{{ defaultStorageClasses.rwx }}" - name: "determine-storage-classes : Assert that Manage PVC storage class has been defined" assert: diff --git a/ibm/mas_devops/roles/suite_upgrade/tasks/upgrade.yml b/ibm/mas_devops/roles/suite_upgrade/tasks/upgrade.yml index 18a1f7cf1f..efa48eda89 100644 --- a/ibm/mas_devops/roles/suite_upgrade/tasks/upgrade.yml +++ b/ibm/mas_devops/roles/suite_upgrade/tasks/upgrade.yml @@ -32,6 +32,12 @@ debug: var: updated_suite_sub_info +# No easy way to determine the end of the installPlanGeneration as it depends on if we have a patch versions of the +# new version in the catalog. No patch versions means just one installPlanGeneration increase. Catalog has patches means +# two installPlanGenerateion increase. Wait for 5 minutes like we do for apps +- name: "Pause for 5 minutes before checking upgrade status..." + pause: + minutes: 5 # 3. Lookup the OperatorCondition # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/uds/README.md b/ibm/mas_devops/roles/uds/README.md index 5767104ecd..ddd0672989 100644 --- a/ibm/mas_devops/roles/uds/README.md +++ b/ibm/mas_devops/roles/uds/README.md @@ -20,7 +20,7 @@ Required only for ROSA cluster. This variable is required to extract the UDS cer - Default Value: None ### uds_storage_class -Required. Storage class where UDS will be installed. On IBM Cloud RedHat Openshift Kubernetes Service (ROKS) `ibmc-block-bronze` is the recommended value. +Required. Storage class where UDS will be installed. On IBM Cloud RedHat Openshift Kubernetes Service (ROKS) `ibmc-block-bronze` is the recommended value. The storage class must support ReadWriteOnce(RWO) access mode. - Environment Variable: `UDS_STORAGE_CLASS` - Default Value: None diff --git a/ibm/mas_devops/roles/uds/tasks/install/determine-storage-classes.yml b/ibm/mas_devops/roles/uds/tasks/install/determine-storage-classes.yml index f82c4cf7a8..2fced6b737 100644 --- a/ibm/mas_devops/roles/uds/tasks/install/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/uds/tasks/install/determine-storage-classes.yml @@ -1,22 +1,8 @@ --- - -# Provide intelligent storage class selection to minimize required user knowledge - # 1. Lookup storage class availabiity # ----------------------------------------------------------------------------- -- name: "Load default storage class information" - include_vars: "{{ role_path }}/../../common_vars/default_storage_classes.yml" - -- name: Lookup storage classes - kubernetes.core.k8s_info: - api_version: storage.k8s.io/v1 - kind: StorageClass - register: lookup_storageclasses - -- name: "Debug available storage classes" - debug: - msg: - - "Available storage classes .................... {{ lookup_storageclasses | ibm.mas_devops.getResourceNames }}" +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" # 2. Set Storage (Required) @@ -24,7 +10,7 @@ - name: Default Storage for ROKS if not set by user when: uds_storage_class is not defined or uds_storage_class == "" set_fact: - uds_storage_class: "{{ lookup_storageclasses | ibm.mas_devops.defaultStorageClass(default_storage_classes_rwo) }}" + uds_storage_class: "{{ defaultStorageClasses.rwo }}" - name: Assert that storage class has been defined assert: diff --git a/mkdocs.yml b/mkdocs.yml index 7c8f58e8a1..1fad7b80f4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,6 @@ # mkdocs serve -site_name: MAS Devops Collection +site_name: MAS DevOps Ansible Collection site_url: https://ibm-mas.github.io/ansible-devops repo_name: GitHub @@ -8,8 +8,9 @@ repo_url: https://github.com/ibm-mas/ansible-devops edit_uri: blob/master/docs/ nav: - - "Home": index.md - - "Execution Environment": execution-environment.md + - "General Information": + - "Home": index.md + - "Ansible Automation Platform": execution-environment.md - "Playbooks": - "OCP": playbooks/ocp.md - 'Install Cloud Pak For Data': playbooks/cp4d.md @@ -90,6 +91,7 @@ nav: - "suite_manage_birt_report_config": roles/suite_manage_birt_report_config.md - "suite_manage_bim_config": roles/suite_manage_bim_config.md - "suite_manage_customer_files_config": roles/suite_manage_customer_files_config.md + - "suite_manage_imagestitching_config": roles/suite_manage_imagestitching_config.md - "suite_manage_import_certs_config": roles/suite_manage_import_certs_config.md - "suite_manage_load_dbc_scripts": roles/suite_manage_load_dbc_scripts.md - "suite_manage_logging_config": roles/suite_manage_logging_config.md @@ -105,33 +107,40 @@ nav: - "gencfg_jdbc": roles/gencfg_jdbc.md - "gencfg_watsonstudio": roles/gencfg_watsonstudio.md - "gencfg_workspace": roles/gencfg_workspace.md - - "install_operator": roles/install_operator.md - "gencfg_mongo": roles/gencfg_mongo.md theme: - name: readthedocs - highlightjs: true - hljs_languages: - - yaml - prev_next_buttons_location: none -extra_css: - - extra.css + name: carbon + prefix: IBM + header_nav_items: + - title: Ansible + active: true + - title: CLI + url: https://ibm-mas.github.io/cli/ + markdown_extensions: - toc: - permalink:  + permalink: "¤" - admonition - # Add this to support expandable sections - #- pymdownx.details -# https://squidfunk.github.io/mkdocs-material/extensions/admonition/ -# Note -# Abstract -# Info -# Tip -# Success -# Question -# Warning -# Failure -# Danger -# Bug -# Example -# Quote + # https://squidfunk.github.io/mkdocs-material/extensions/admonition/ + # Note + # Abstract + # Info + # Tip + # Success + # Question + # Warning + # Failure + # Danger + # Bug + # Example + # Quote + +plugins: + - search: {} + - glightbox: + shadow: true + caption_position: bottom + auto_caption: true + touchNavigation: false + background: black From 6bbd273946417a1854ab93af5e24bccd62bbe96e Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 3 Dec 2024 16:42:54 +0000 Subject: [PATCH 035/169] [patch] update aibroker playbook --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index bf5633563e..b4eb8aed3f 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -16,7 +16,7 @@ ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" - # Control is saas release + # Control if saas release mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" # DB2 @@ -66,13 +66,11 @@ when: mas_aibroker_saas == true - include_role: - # install db2 name: ibm.mas_devops.db2 - when: mas_aibroker_saas == true + when: mas_aibroker_saas == true - include_role: - # config db2 name: ibm.mas_devops.suite_config - when: mas_aibroker_saas == true + when: mas_aibroker_saas == true roles: - ibm.mas_devops.odh From 71c451faa665ab3d91c48f7809c671036e69cccc Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 4 Dec 2024 10:27:41 +0000 Subject: [PATCH 036/169] [patch] add third params to create sls --- .../aibroker/files/alm_sample_sls_use.py | 39 +++++++++++-------- .../roles/aibroker/files/create_sls_secret.sh | 4 +- ibm/mas_devops/roles/aibroker/tasks/main.yml | 3 +- .../aibroker/tasks/tenant/install/main.yml | 6 +-- ibm/mas_devops/roles/aibroker/vars/main.yml | 1 + 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py index ad3d42b8e2..cd88322c75 100644 --- a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py +++ b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py @@ -15,8 +15,12 @@ slsUrl = sys.argv[1] slsRegistrationKey = sys.argv[2] -#print(slsUrl) -#print(slsRegistrationKey) + +caCertificateForSSL = sys.argv[3] +print('slsUrl='+slsUrl) +print(slsRegistrationKey) +print("caCertificateForSSL") +print(caCertificateForSSL) # -------------------------------------------------------------------------------------------------------------------------------- # - # - Configuration section - you need to complete steps 1-5 @@ -37,17 +41,17 @@ def generate_api_key(): # 2: Provide am identifier for your instance. This should be pseudounique, so chances of collisions are low. For example, based on the cluster # identity, or a pseudo unique UUID created at initial installation time -instanceIdentifier = generate_api_key() +instanceIdentifier = generate_api_key() #"xxxx" #print("id") print(instanceIdentifier) # 3: Provide the SLS instance URL (SRE will give you this) - +#slsUrl = "https://sls.ibm-sls.ibm-sls.apps.xxx.cp.fyre.ibm.com" # 4: Provide the SLS registration key (SRE will give you this) - +#slsRegistrationKey = "xxxx" # 5: Decide if you want the demonstration to tear down your client afterwards (as you would do in a deprovision). If not, the client # will remain registered for you to experiment with. @@ -90,11 +94,12 @@ def generate_api_key(): pass # Make initial registration request + print("will use certificate for client provisioning ") data = {'clientId': slsClientId, 'description': 'ALM sample client registration', 'role': 'manage'} # TODO: Rawa - Swagger docs incorrectly state 'id' rather than 'clientId' headers = {'X-Registration-Key': slsRegistrationKey} # TODO: Rawa - Swagger docs don't document the need for this critical header #print ("Making initial registration request for client "+slsClientId) - response = requests.post(slsUrl+"/api/registrations",verify=False,json=data, headers=headers ) + response = requests.post(slsUrl+"/api/registrations",verify=caCertificateForSSL,json=data, headers=headers ) response.raise_for_status() registrationId = response.json()['registrationId'] @@ -106,10 +111,10 @@ def generate_api_key(): provisioningWaiting = True while provisioningWaiting: headers = {'X-Registration-Key': slsRegistrationKey} - response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=False,headers=headers ) + response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=caCertificateForSSL,headers=headers ) response.raise_for_status() status = response.json()['state'] - #print("Waiting for client provisioning to be completed. Current status: %s" %status) + print("Waiting for client provisioning to be completed. Current status: %s" %status) if status=="AWAITING_CONFIRMATION": # We are now ready to proceed; client certs are available #print ("Extracting client certificates") @@ -128,12 +133,12 @@ def generate_api_key(): # The next step is to confirm the client. Poll status until complete. confirmationWaiting = True while confirmationWaiting: - response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=False, cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL, cert=(clientTlsCrtPath,clientTlsKeyPath)) #print (response) status = response.json()['state'] #print("Waiting for client registration to be confirmed. Current status: %s" %status) if status=="REGISTERED": - #print ("Client registration confirmed.") + print ("Client registration confirmed.") confirmationWaiting = False time.sleep(1) @@ -147,12 +152,12 @@ def generate_api_key(): def spendAppPoints(quantity): #print ("Let's spend "+str(quantity)+" AppPoints") data = {'validity': 86400, 'quantity': quantity} - response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() # String representation of AppPoint usage by your client. TODO: SLS will shortly be providing an update to correctly report usage for your client. At present, it returns the overall usage across all clients. def getUsage(): - response = requests.get(slsUrl+"/api/tokens/",verify=False,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner + response = requests.get(slsUrl+"/api/tokens/",verify=caCertificateForSSL,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner response.raise_for_status() j = json.loads(response.content.decode('utf-8')) return j[0]['used'] @@ -178,7 +183,7 @@ def getUsage(): # SIMPLE "single shot" # -------------------- # - Report AppPoint usage in one go. -# - This means a single, consistent licenseeId at the instance level - for example, licenseeId="", i.e. reusing the SLS clientId is convenient. +# - This means a single, consistent licenseeId at the instance level - for example, licenseeId="eis-126745fvabv", i.e. reusing the SLS clientId is convenient. # - Set the quantity to be 25+10 = 35 # # @@ -187,8 +192,8 @@ def getUsage(): # - Report AppPoint usage as a sequence of updates, per capability # - Enables you to use SLS as a way to understand unequivocally breakdown of AppPoint usage by capability # - Each capability has its own licenseeId - for example: -# - example: licenseeId="",quantity=25 -# licenseeId="",quantity=10 +# - example: licenseeId="eis-126745fvabv-users",quantity=25 +# licenseeId="eis-126745fvabv-iopoints",quantity=10 # # In this code example, we'll do a single shot for simplicity and clarity @@ -235,12 +240,12 @@ def getUsage(): # At deprovision, return your licenses and unregister the client print ("Preparing for deprovision: returning any AppPoints to the pool") - response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() print("Usage after deprovisioning: "+str(getUsage())) print ("Preparing for deprovision: unregistering the SLS client") - response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() #print ("Deleting redundant client certificates") os.remove(clientTlsCrtPath) diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index ebaad94acc..e3debc887b 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -6,7 +6,7 @@ instance_id=$2 if [ -z ${TENANT} ]; then #echo "❌ Missing tenant name" #exit 1 - echo "Usage ./create_sls_secret.sh tenant-name aibroker-instance-id-from-CustomResourceDefinitions sls-url-from-sls-routes slsRegistrationKey-from-sls-api-pod-Environment" + echo "Usage ./create_sls_secret.sh tenant-name aibroker-instance-id-from-CustomResourceDefinitions sls-url-from-sls-routes slsRegistrationKey-from-sls-api-pod-Environment path_to_the_ca_crt" echo "for example" echo "./create_sls_secret.sh aibroker-user aibdev https://sls.ibm-sls.xxx.ibm.com xxxx-xxxx-xxxx-xxx" exit 1 @@ -19,7 +19,7 @@ mkdir -p certs echo "creating SLS registration, please wait....." # instanceIdentifier=`python3 {{ role_path }}/files/alm_sample_sls_use.py $3 $4` -instanceIdentifier=$(python3 ../roles/aibroker/files/alm_sample_sls_use.py $3 $4) +instanceIdentifier=$(python3 ../roles/aibroker/files/alm_sample_sls_use.py $3 $4 $5) echo "SLS registration is created successfully." registrationKey=$4 diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 22a6f8c349..9a57222a1a 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -7,8 +7,7 @@ # run api calls for aibroker saas - include_tasks: tasks/saas/main.yml - when: - - mas_aibroker_saas == "true" + when: mas_aibroker_saas == true # create AI Broker tenant - include_tasks: tasks/tenant/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 49698e931e..60048317d9 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -92,7 +92,7 @@ register: sls_secret_info - name: Create SLS secret - script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }}" + script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} {{ mas_aibroker_path_ca_crt }}" when: - sls_secret_info.resources | length == 0 register: sls_secret_output @@ -142,7 +142,7 @@ timeout: 30 register: add_tenant_to_db2 -- name: Debug tenant #TODO: remove on finall commit +- name: Debug tenant #TODO: remove on final commit debug: msg: "{{ add_tenant_to_db2 }}" @@ -182,6 +182,6 @@ timeout: 30 register: add_tenant_entitlements_to_db2 -- name: Debug tenant entitlement #TODO: remove on finall commit +- name: Debug tenant entitlement #TODO: remove on final commit debug: msg: "{{ add_tenant_entitlements_to_db2 }}" diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index f344f5c625..98a1bc7935 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -48,3 +48,4 @@ mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', tr mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" +mas_aibroker_path_ca_crt: "./" From 3e6bdc0f9565a1eebf0c062378629f9a7758e849 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 12 Dec 2024 17:03:15 +0000 Subject: [PATCH 037/169] [patch] remove mult dash from name --- ibm/mas_devops/roles/aibroker/files/create_s3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_s3.sh b/ibm/mas_devops/roles/aibroker/files/create_s3.sh index fb94226c0b..4391404b8d 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_s3.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_s3.sh @@ -79,7 +79,7 @@ fi # exit 1 # fi -bucketpostfix='----training-bucket' +bucketpostfix='-training-bucket' if [[ $STORAGE_HOST == *"amazonaws"* ]]; then echo 'it is Amazon AWS' bucketname=$(python3 ../roles/aibroker/files/create_bucket.py ${TENANT} ${STORAGE_ACCESSKEY} ${STORAGE_SECRETKEY} ${STORAGE_REGION}) From e3f2eaaeb2c101c2c73a3e1017bdc76c550a15d6 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 13 Dec 2024 10:35:32 +0000 Subject: [PATCH 038/169] [patch] update with fix for saas --- .../roles/odh/tasks/odh-operator.yml | 36 +++++++++++-------- .../odh/templates/odh/subscription.yml.j2 | 12 +++++++ 2 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 ibm/mas_devops/roles/odh/templates/odh/subscription.yml.j2 diff --git a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml index cd75ca52a8..aa9f04f2d1 100644 --- a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml @@ -1,21 +1,27 @@ --- -# Install Operator & create entitlement openshift-odh -# ----------------------------------------------------------------------------- -- name: "Install Openshift odh Operator" - ibm.mas_devops.apply_subscription: - namespace: "{{ openshift_namespace }}" - package_name: "opendatahub-operator" - package_channel: "{{ odh_channel }}" - catalog_source: "{{ odh_catalog_source }}" - register: subscription +# # Install Operator & create entitlement openshift-odh +# # ----------------------------------------------------------------------------- +# - name: "Install Openshift odh Operator" +# ibm.mas_devops.apply_subscription: +# namespace: "{{ openshift_namespace }}" +# package_name: "opendatahub-operator" +# package_channel: "{{ odh_channel }}" +# catalog_source: "{{ odh_catalog_source }}" +# register: subscription + +# # Wait until the Opendata Hub CRD is available +# # ----------------------------------------------------------------------------- +# - name: "Wait until the Opendata Hub CRD is available" +# include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" +# vars: +# crd_name: knativeservings.operator.knative.dev -# Wait until the Opendata Hub CRD is available -# ----------------------------------------------------------------------------- -- name: "Wait until the Opendata Hub CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: knativeservings.operator.knative.dev +# enable following role temporary until find a way to use latest ODH versions and switch to apply_subscription role +- name: "Install Openshift odh Operator specific version" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/odh/subscription.yml.j2') }}" - name: Await availability of ODH Install Plans pause: diff --git a/ibm/mas_devops/roles/odh/templates/odh/subscription.yml.j2 b/ibm/mas_devops/roles/odh/templates/odh/subscription.yml.j2 new file mode 100644 index 0000000000..b50b76e040 --- /dev/null +++ b/ibm/mas_devops/roles/odh/templates/odh/subscription.yml.j2 @@ -0,0 +1,12 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: opendatahub-operator + namespace: openshift-operators +spec: + channel: "{{ odh_channel }}" + installPlanApproval: Manual + name: opendatahub-operator + source: "{{ odh_catalog_source }}" + sourceNamespace: openshift-marketplace + startingCSV: "{{ odh_operator_version }}" \ No newline at end of file From 5a180aa0a208b1940e89d00f59a7119f9252fa23 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 19 Dec 2024 15:27:24 +0000 Subject: [PATCH 039/169] [patch] update AI Broker documentation for saas --- docs/playbooks/oneclick-aibroker.md | 57 ++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index a54450b2fd..9695f0e1ec 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -39,6 +39,7 @@ AI Broker supports **AWS** and **Minio** storage providers. * `MAS_INSTANCE_ID` Declare the instance ID for the AI Broker install * `MAS_ENTITLEMENT_KEY` Your IBM Entitlement key to access the IBM Container Registry * `MAS_ENTITLEMENT_USERNAME` Your IBM Entitlement user to access the IBM Container Registry +* `MAS_APP_CHANNEL` Aibroker application channel * `MAS_AIBROKER_STORAGE_ACCESSKEY` Your strage provider access key * `MAS_AIBROKER_STORAGE_SECRETKEY` Your storage provider secret key * `MAS_AIBROKER_STORAGE_HOST` Your storage provider host @@ -58,9 +59,20 @@ AI Broker supports **AWS** and **Minio** storage providers. * `MAS_AIBROKER_DB_SECRET_NAME` Your database instance secret name * `MAS_AIBROKER_DB_SECRET_VALUE` Your database instance password +## Required environment variables when AI Broker deployed on SAAS + +* `MAS_AIBROKER_SAAS` specify if saas deployment (default value is: false) +* `MAS_CONFIG_DIR` specify config location, mandatory when `MAS_AIBROKER_SAAS=true` +* `MAS_AIBROKER_DOMAIN` specify cluster domain, mandatory when `MAS_AIBROKER_SAAS=true` +* `MAS_AIBROKER_SLS_URL` specify SLS url, mandatory when `MAS_AIBROKER_SAAS=true` +* `MAS_AIBROKER_SLS_REGISTRATION_KEY` specify sls registration key, mandatory when `MAS_AIBROKER_SAAS=true`, to get value: look in `ibm-sls` namespace, pod `sls-api-licensing-xxx` and in `Environment` tab check `REGISTRATION_KEY` value +* `MAS_AIBROKER_DRO_URL` specify DRO url, mandatory when `MAS_AIBROKER_SAAS=true` +* `MAS_AIBROKER_DRO_TOKEN` specify DRO token, mandatory when `MAS_AIBROKER_SAAS=true` to get value: go to `mas-{{ instance_id }}-core` and look in secret `dro-apikey` +* `DB2_INSTANCE_NAME` specify DB2 instance name (default value is: aibroker), mandatory when `MAS_AIBROKER_SAAS=true` +* `IBM_ENTITLEMENT_KEY` specify IBM Entitlement key, mandatory when `MAS_AIBROKER_SAAS=true` + ## Optional environment variables -* `MAS_AIBROKER_CHANNEL` Your custom AI broker application channel * `MAS_ICR_CP` Provide custom registry for AI Broker applications * `MAS_ICR_CPOPEN` Provide custom registry for AI Broker operator * `MAS_CATALOG_VERSION` Your custom AI broker catalog version @@ -70,6 +82,7 @@ AI Broker supports **AWS** and **Minio** storage providers. * `MAS_AIBROKER_APIKEY_ACTION` Whether to install or remove or update apikey (default value is: install) * `MAS_AIBROKER_WATSONX_ACTION` Whether to install or remove watsonx secret (default value is: install) * `MAS_AIBROKER_S3_ACTION` Whether to install or remove s3 (default value is: install) + ## Usage @@ -93,6 +106,7 @@ export ARTIFACTORY_TOKEN="" export MAS_ICR_CP="" export MAS_ICR_CPOPEN="" export MAS_INSTANCE_ID="" +export MAS_APP_CHANNEL="9.0.x" export MAS_AIBROKER_STORAGE_ACCESSKEY="" export MAS_AIBROKER_STORAGE_SECRETKEY="" export MAS_AIBROKER_STORAGE_HOST="" @@ -123,6 +137,7 @@ ansible-playbook playbooks/oneclick_add_aibroker.yml export MAS_ENTITLEMENT_USERNAME="" export MAS_ENTITLEMENT_KEY="" export MAS_INSTANCE_ID="" +export MAS_APP_CHANNEL="9.0.x" export MAS_AIBROKER_STORAGE_ACCESSKEY="" export MAS_AIBROKER_STORAGE_SECRETKEY="" export MAS_AIBROKER_STORAGE_HOST="" @@ -147,7 +162,45 @@ oc login --token=xxxx --server=https://myocpserver ansible-playbook playbooks/oneclick_add_aibroker.yml ``` -### +#### Run playbooks for deploy AI Broker on SAAS + +```bash +export MAS_ENTITLEMENT_USERNAME="" +export MAS_ENTITLEMENT_KEY="" +export IBM_ENTITLEMENT_KEY=${MAS_ENTITLEMENT_KEY} +export MAS_INSTANCE_ID="" +export MAS_APP_CHANNEL="9.1.x" +export MAS_CONFIG_DIR="config_path_location" +export MAS_AIBROKER_SAAS="true" +export MAS_AIBROKER_DOMAIN="apps.domain" +export MAS_AIBROKER_SLS_URL="https://sls.ibm-sls.ibm-sls."${MAS_AIBROKER_DOMAIN} +export MAS_AIBROKER_SLS_REGISTRATION_KEY="xxxxxxx" +export MAS_AIBROKER_DRO_URL="https://ibm-data-reporter-redhat-marketplace."${MAS_AIBROKER_DOMAIN} +export MAS_AIBROKER_DRO_TOKEN="xxx" +export DB2_INSTANCE_NAME="aibroker" +export MAS_AIBROKER_STORAGE_ACCESSKEY="" +export MAS_AIBROKER_STORAGE_SECRETKEY="" +export MAS_AIBROKER_STORAGE_HOST="" +export MAS_AIBROKER_STORAGE_SSL="true or false" +export MAS_AIBROKER_STORAGE_REGION="" +export MAS_AIBROKER_STORAGE_PROVIDER="" +export MAS_AIBROKER_STORAGE_PORT="" +export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" +export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" +export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" +export MAS_AIBROKER_WATSONXAI_APIKEY="" +export MAS_AIBROKER_WATSONXAI_URL="" +export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" +export MAS_AIBROKER_DB_HOST="" +export MAS_AIBROKER_DB_PORT="" +export MAS_AIBROKER_DB_USER="" +export MAS_AIBROKER_DB_DATABASE="" +export MAS_AIBROKER_DB_SECRET_NAME="" +export MAS_AIBROKER_DB_SECRET_VALUE="" + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook playbooks/oneclick_add_aibroker.yml +``` ## NOTICE: playbook oneclick_add_aibroker.yml will run three roles: From ef5b38c22efe9e318bb4532f205d9e89cc56204b Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 19 Dec 2024 15:39:58 +0000 Subject: [PATCH 040/169] Squashed commit of the following: commit 13770e9343341a5963d2918f1814006a6f326964 Author: sekharcvalluri <54737838+sekharcvalluri@users.noreply.github.com> Date: Thu Dec 19 19:19:48 2024 +0530 [patch] Fix cos_bucket picking incorrect resourcegroup (#1593) Co-authored-by: Harsh Tamakuwala Co-authored-by: leo-miran <105313348+leo-miran@users.noreply.github.com> commit 0a3b97bce5c3ed58d10995c773305eeb376067f2 Author: David Parker Date: Thu Dec 19 13:47:57 2024 +0000 [major] Replace ICSP with IDMS (#1594) commit b08cda4327ab30d03326dbc5c7b35bee86bf16ba Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Thu Dec 19 18:49:47 2024 +0530 [patch] Update supported kafka versions (#1608) Co-authored-by: Harsh Tamakuwala commit cf057173082d9a94137cf3aa3c63d5bb9aa7b3bd Author: David Parker Date: Thu Dec 19 11:39:55 2024 +0000 [patch] Lock mas-devops less than v2 (#1607) commit 6ff397e628583040bc4e7aad56ae5558d7796db2 Author: David Parker Date: Tue Dec 17 22:35:41 2024 +0000 [patch] Enable HMAC by default for IBM COS (#1606) commit 49b2099ff65f2322fb8c670311dc5ea7030ae2ea Author: Rawa <61942902+rawa-resul@users.noreply.github.com> Date: Thu Dec 12 16:28:49 2024 +0000 [patch] Fix subscription lookup in upgrade (#1589) Co-authored-by: Rawa Resul commit 79c5991d608369f36eed2c50ea2edb69225b5ac7 Author: leo-miran <105313348+leo-miran@users.noreply.github.com> Date: Thu Dec 12 13:26:06 2024 -0300 [patch] fix default booleans for suite_dns (#1590) commit 1e91ba5f6361f484e67c1ab6181f399eebd4269c Author: Andrew Whitfield Date: Thu Dec 12 07:34:31 2024 +0000 [patch] Increase rosa deprovision wait time (#1592) commit 91bd67beb1637d11677e8d835594eb26dbd07da3 Author: Ian Boden <82514609+IanBoden@users.noreply.github.com> Date: Mon Dec 9 10:32:54 2024 +0000 [patch] reinstate zenmetastore and couchdb workarounds for cpd 4.8 (#1580) commit aa05d8a589d8334149fcd53c66d5fcea01977d07 Author: Rawa <61942902+rawa-resul@users.noreply.github.com> Date: Fri Dec 6 16:32:24 2024 +0000 [patch] Fix label_selectors pattern (#1586) Co-authored-by: Rawa Resul commit e0ba0decf64579d7b021e16e68d866f9704c6a5e Author: Parveen Kumar Date: Fri Dec 6 19:23:46 2024 +0530 [minor] Support December catalog (#1562) Co-authored-by: Anil Prajapati Co-authored-by: Jon Levell Co-authored-by: Jon Levell Co-authored-by: Ian Boden <82514609+IanBoden@users.noreply.github.com> Co-authored-by: Ian Boden Co-authored-by: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Co-authored-by: Terence Quinn Co-authored-by: David Parker Co-authored-by: Rawa Resul Co-authored-by: Rawa <61942902+rawa-resul@users.noreply.github.com> commit 9206656d7384f91469b03841606d73c06682bd18 Author: David Parker Date: Tue Dec 3 14:36:31 2024 +0000 [major] Use new OLM functions in mas-devops (#1573) commit 3ccdbef982c83989942445ef077d4117d81fec48 Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Mon Dec 2 18:30:04 2024 +0530 [patch] Fixes ibmcloud_api_key issue in COS deprovision for ibmcloud (#1581) Co-authored-by: Harsh Tamakuwala commit 1283c03b65c60cfc0095969abc878264db98724f Author: Terence Quinn Date: Fri Nov 29 03:52:51 2024 -0600 [patch] create unique certificate for each db2 instance (#1579) commit e04138afb8c7ffdfe4be159323d98d404426bb88 Author: David Parker Date: Tue Nov 26 21:40:39 2024 +0000 [patch] get_default_storage_classes should always set failed=False (#1578) commit 7014f49fc63410d44808a7082e8a1ce922e93879 Author: Sanjay Prabhakar Date: Tue Nov 26 16:32:42 2024 +0000 [patch] Add estimates for cpd in readme (#1577) commit 9933bc23fb272ba7fe169ded7a54de71e952b55b Author: David Parker Date: Tue Nov 26 13:46:05 2024 +0000 [patch] Use correct variable for mongo extras (#1576) commit bb720a394502505f5c91849992fc276e0a4cc1c4 Author: David Parker Date: Tue Nov 26 13:32:30 2024 +0000 [patch] Set cpd_product_version from catalog metadata (#1575) commit ba23e83ae87fe889f6bf2fd6bd26da16d1034881 Author: Terence Quinn Date: Mon Nov 25 15:03:03 2024 -0600 [patch] Fix missing catalog_digest (#1574) Co-authored-by: David Parker commit 0f60c6150f2f4541673c22b8312113fe24ce67ac Author: Sanjay Prabhakar Date: Fri Nov 22 17:33:20 2024 +0000 [patch] Add back restore.yml.j2 (#1572) Co-authored-by: Sanjay Prabhakar commit 739c4d03781bca7c983d92e103e62e6542b72fa7 Author: David Parker Date: Fri Nov 22 16:02:24 2024 +0000 [major] Overhaul backup/restore capability (#1557) Co-authored-by: Sanjay Prabhakar commit d5ef4102841b56a9fa7045be01989a3056fed5ef Author: David Parker Date: Fri Nov 22 12:39:48 2024 +0000 [doc] Fix formatting commit 61272c972bc692df31bdd64104234257b1e41ef2 Author: Sanjay Prabhakar Date: Fri Nov 22 12:27:00 2024 +0000 [patch] fix restore for manage role (#1571) Co-authored-by: Sanjay Prabhakar commit 0a13abdc64bf2e458dcb42c9560965a5c107da9f Author: Sanjay Prabhakar Date: Thu Nov 21 10:40:05 2024 +0000 [patch] Db2 backup/restore: fix copy local to pod script (#1569) Co-authored-by: Sanjay Prabhakar commit f4745094e7a119e8fbf5eb2a341e856baaf757e3 Author: David Parker Date: Wed Nov 20 23:43:25 2024 +0000 [major] Move catalog & storage class metadata to python package (#1561) commit df4fe4c55d7a1f1d78723f1452b8a6c3f4dda3a9 Author: Lokesh <110647904+lokesh-sreedhara@users.noreply.github.com> Date: Wed Nov 20 08:49:28 2024 -0500 [patch] remove MIRRORLOGPATH folder creation (#1568) commit 5668f331e38307250c4bdb22e5e5dd2e1f687607 Author: Lokesh <110647904+lokesh-sreedhara@users.noreply.github.com> Date: Wed Nov 20 07:49:43 2024 -0500 [patch] fix conflicting DB config (#1567) commit 5c595c11aacfdb9a3044c257ba9042a08005f311 Author: Sanjay Prabhakar Date: Tue Nov 19 17:45:03 2024 +0000 [patch] can skip db2 in br_manage playbook (#1566) Co-authored-by: Sanjay Prabhakar commit 8c8f9933ce35e16862e3f328ee5fe79349a634ea Author: stonepd Date: Tue Nov 19 16:39:18 2024 +0000 [patch] update CP4D README, remove outdated fix code (#1565) Co-authored-by: Sanjay Prabhakar commit 5eb2cd4aa2ef8f9fcc12626e1f85b577b9219ef3 Author: Sanjay Prabhakar Date: Tue Nov 19 14:42:36 2024 +0000 [patch] fix restore for core and fix tar command in restore tasks (#1564) Co-authored-by: Sanjay Prabhakar commit e81d3a3e0ddd89ef0f7ed1289ff8a7f311599b33 Author: dclain Date: Sat Nov 16 15:39:54 2024 -0600 [patch] Address several issues w/ Nvidia playbook (#1558) commit 1792e98b77b32185c486da66a31c4c89fb11bfcf Author: Tremaine Hart Date: Thu Nov 14 13:19:26 2024 -0600 [patch] Fixes and updates for IBMCloud COS support (#1555) Co-authored-by: David Parker commit 495e26d52efbda11ccb94bbab990a00652e9cc0e Author: Daniel Istrate <16302386+istrate@users.noreply.github.com> Date: Thu Nov 14 13:02:17 2024 +0200 [doc] Document storage class access mode (#1554) Co-authored-by: Daniel Istrate commit 08c22d74515c90a46f19894a630cd2c8155bccef Author: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Date: Tue Nov 12 02:51:17 2024 +0530 [minor] Support November catalog (#1508) Co-authored-by: Parveen Kumar Co-authored-by: Jon Levell Co-authored-by: Jon Levell Co-authored-by: Ian Boden <82514609+IanBoden@users.noreply.github.com> Co-authored-by: Ian Boden commit 35017863e41ff139d567671a507800c49b4f1bc5 Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Thu Nov 7 09:53:24 2024 -0800 [patch] MAXMIS-897: check if attribute 'deployimagestitching' is defined (#1551) Co-authored-by: Brendan More O'Ferrall commit 00c50f15b3ba266f4ec42d0fd44bd81f95f9543c Author: caemar <44871769+caemar@users.noreply.github.com> Date: Thu Nov 7 11:47:38 2024 +0100 [minor] Add Kafka Cluster CA certificate validityDays (#1543) commit ee3a2272b4a3418339eb2c4df77598d56f1f0179 Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Wed Nov 6 11:57:00 2024 -0800 [patch] MAXMIS-886: disable stitching deploy when civil is not enabled2 (#1548) Co-authored-by: Brendan More O'Ferrall commit 3ee251228c962a156692391bceb063c22ebac459 Author: Tremaine Hart Date: Wed Nov 6 11:41:34 2024 -0600 [patch] [MASCORE-4344] Grafana role overwriting critical k8s configuration resource (#1549) commit 482b3868b31468f220da901d5d23458eb47d8afd Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Tue Nov 5 21:51:50 2024 -0800 [minor] #MAXMIS-748: add role to auto configure image stitching (#1530) Co-authored-by: Brendan More O'Ferrall Co-authored-by: Alexandre Quinteiro Co-authored-by: unnati-solanki-git <131964221+unnati-solanki-git@users.noreply.github.com> commit dd13898e98f238b724da63f476d59312c494f139 Author: Tremaine Hart Date: Tue Nov 5 14:45:45 2024 -0600 [patch] [MASCORE-4344] Grafana role overwriting critical k8s configuration resource (#1547) commit ade2a0ea54989b2b2ce55a47890f5d249d13484a Author: Andrew Whitfield Date: Tue Nov 5 17:18:24 2024 +0000 [patch] wait for olm to update operatorcondition (#1545) commit c267990a26a599d405fc78fe248181ab13ad258a Author: Sanjay Prabhakar Date: Tue Nov 5 16:33:07 2024 +0000 [patch] prevent mongo upgrade when featureCompatibilityVersion is not defined or set incorrectly (#1544) Co-authored-by: Sanjay Prabhakar commit f4c04d569cbad2268ad763b344fd297399c5e19a Author: Ian Boden <82514609+IanBoden@users.noreply.github.com> Date: Tue Nov 5 11:21:08 2024 +0000 [patch] Fix gpu-certified-operator default channel (#1542) commit 733b595cebbd00232c415a1d62d456266936ed58 Author: David Parker Date: Mon Nov 4 15:49:12 2024 +0000 [patch] Fix oc exec commands (#1541) commit 224a9a8707befe26612523397a2564a1f31b4071 Author: David Parker Date: Thu Oct 31 18:10:39 2024 +0000 [patch] Reduce useless logging (#1540) --- .github/workflows/ansible-publish.yml | 7 +- .github/workflows/ansible.yml | 5 + build/bin/build-collection.sh | 4 +- build/bin/copy-role-docs.sh | 2 +- docs/playbooks/ocp.md | 6 +- docs/playbooks/oneclick-predict.md | 2 +- ibm/mas_devops/common_tasks/detect_airgap.yml | 28 ++- .../common_tasks/get_signed_ingress_cert.yml | 8 +- .../common_vars/compatibility_matrix.yml | 13 ++ .../playbooks/mirror_add_assist.yml | 8 +- ibm/mas_devops/playbooks/mirror_add_iot.yml | 8 +- .../playbooks/mirror_add_manage.yml | 4 +- .../playbooks/mirror_add_monitor.yml | 8 +- .../playbooks/mirror_add_optimizer.yml | 4 +- .../playbooks/mirror_add_predict.yml | 4 +- .../playbooks/mirror_add_visualinspection.yml | 4 +- .../playbooks/mirror_dependencies.yml | 65 +++++- .../playbooks/ocp_convert_to_disconnected.yml | 2 +- .../playbooks/ocp_fyre_provision.yml | 2 +- .../playbooks/ocp_roks_provision.yml | 2 +- ibm/mas_devops/plugins/filter/filters.py | 11 + .../aibroker/tasks/apikey/install/main.yml | 2 +- .../roles/aibroker/tasks/s3/install/main.yml | 2 +- .../roles/aibroker/tasks/s3/remove/main.yml | 2 +- .../aibroker/tasks/tenant/install/main.yml | 2 +- .../aibroker/tasks/watsonx/install/main.yml | 2 +- .../arcgis/tasks/lookup_trusted_certs.yml | 2 +- ibm/mas_devops/roles/cos/README.md | 23 +- ibm/mas_devops/roles/cos/defaults/main.yml | 2 +- .../cos_bucket/tasks/providers/ibm/create.yml | 4 +- .../cos_bucket/tasks/providers/ibm/delete.yml | 4 +- .../tasks/prereq-check/check-cpfs-version.yml | 2 +- .../roles/cp4d/tasks/prereqs/install-cpfs.yml | 2 +- .../tasks/prereqs/install-ibm-licensing.yml | 2 +- .../prereqs/install-ibm-namespace-scope.yml | 2 +- .../cp4d/tasks/wait/wait-zenmetastore-edb.yml | 112 ++++++++++ .../roles/cp4d_service/tasks/main.yml | 2 +- .../cp4d_service/tasks/wait/wait-ccs.yml | 11 +- .../cp4d_service/tasks/wait/wait-couchdb.yml | 108 +++++++++ .../roles/dro/tasks/gencfg/main.yml | 6 +- ibm/mas_devops/roles/kafka/defaults/main.yaml | 4 +- .../lookup-supported-kafka-versions.yml | 3 +- ibm/mas_devops/roles/kmodels/tasks/main.yml | 2 +- .../mirror_case_prepare/defaults/main.yml | 7 +- .../roles/mirror_case_prepare/tasks/main.yml | 98 +++------ .../tasks/uds-2010-fix.yml | 16 -- .../mirror_case_prepare/tasks/uds-208-fix.yml | 16 -- .../mirror_case_prepare/tasks/uds-209-fix.yml | 16 -- .../templates/iot860fix/direct.txt.j2 | 206 +++++++++--------- .../iot860fix/from-filesystem.txt.j2 | 206 +++++++++--------- .../templates/iot867fix/direct.txt.j2 | 204 ++++++++--------- .../iot867fix/from-filesystem.txt.j2 | 204 ++++++++--------- .../templates/sls350fix/direct.txt.j2 | 6 +- .../sls350fix/from-filesystem.txt.j2 | 6 +- .../templates/uds2010fix/direct.txt.j2 | 15 -- .../uds2010fix/from-filesystem.txt.j2 | 15 -- .../templates/uds2010fix/to-filesystem.txt.j2 | 15 -- .../templates/uds208fix/direct.txt.j2 | 22 -- .../uds208fix/from-filesystem.txt.j2 | 22 -- .../templates/uds208fix/to-filesystem.txt.j2 | 22 -- .../templates/uds209fix/direct.txt.j2 | 15 -- .../uds209fix/from-filesystem.txt.j2 | 15 -- .../templates/uds209fix/to-filesystem.txt.j2 | 15 -- .../roles/mirror_extras_prepare/README.md | 1 - .../mirror_extras_prepare/defaults/main.yml | 9 +- .../templates/direct.txt.j2 | 2 +- .../templates/from-filesystem.txt.j2 | 2 +- .../vars/spark_3.3.4.yml | 16 ++ .../vars/spark_3.4.2.yml | 16 ++ .../mirror_extras_prepare/vars/uds_1.0.0.yml | 16 -- .../mirror_extras_prepare/vars/uds_1.1.0.yml | 11 - .../mirror_extras_prepare/vars/uds_1.2.0.yml | 11 - .../mirror_extras_prepare/vars/uds_1.3.0.yml | 11 - .../mirror_extras_prepare/vars/uds_1.4.0.yml | 11 - .../mirror_extras_prepare/vars/uds_1.5.0.yml | 11 - .../roles/mirror_images/defaults/main.yml | 7 +- ibm/mas_devops/roles/mirror_ocp/README.md | 26 +-- .../roles/mirror_ocp/defaults/main.yml | 8 +- .../roles/mirror_ocp/tasks/actions/direct.yml | 6 +- .../tasks/actions/from-filesystem.yml | 6 +- .../templates/imagesetconfiguration.yml.j2 | 5 +- .../roles/nvidia_gpu/tasks/main.yml | 1 - .../roles/ocp_cluster_monitoring/README.md | 4 +- .../ocp_contentsourcepolicy/defaults/main.yml | 26 --- .../ocp_contentsourcepolicy/tasks/main.yml | 66 ------ .../templates/redhat-catalogs.yml.j2 | 127 ----------- .../templates/redhat-release.yml.j2 | 12 - .../ocp_deprovision/tasks/providers/rosa.yml | 4 +- .../README.md | 36 +-- .../roles/ocp_idms/defaults/main.yml | 40 ++++ .../meta/main.yml | 0 ibm/mas_devops/roles/ocp_idms/tasks/main.yml | 135 ++++++++++++ .../tasks/trust.yml | 0 .../tasks/update-pull-secret-dev.yml | 0 .../tasks/update-pull-secret.yml | 0 .../templates/configmap.yml.j2 | 7 + .../templates/idms/mas-ibm-catalog.yml.j2} | 48 ++-- .../templates/idms/mas-redhat-catalogs.yml.j2 | 161 ++++++++++++++ .../templates/idms/mas-redhat-release.yml.j2 | 21 ++ .../roles/ocp_provision/defaults/main.yml | 2 +- .../roles/ocp_provision/tasks/main.yml | 4 +- .../roles/odh/tasks/odh-operator.yml | 21 +- .../roles/odh/tasks/pipeline-operator.yml | 2 +- .../tasks/check_app_compatibility.yml | 11 +- .../tasks/check_core_compatibility.yml | 9 +- .../roles/suite_app_upgrade/tasks/main.yml | 10 +- .../roles/suite_app_upgrade/tasks/upgrade.yml | 12 +- .../roles/suite_dns/defaults/main.yaml | 10 +- .../roles/suite_uninstall/tasks/main.yml | 2 +- .../tasks/check_app_compatibility.yml | 9 +- .../tasks/check_core_compatibility.yml | 9 +- .../roles/suite_upgrade/tasks/main.yml | 5 +- .../roles/suite_upgrade/tasks/upgrade.yml | 12 +- .../roles/uds/tasks/install/udscfg.yml | 4 +- mkdocs.yml | 2 +- 115 files changed, 1436 insertions(+), 1190 deletions(-) create mode 100644 ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml create mode 100644 ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-2010-fix.yml delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-208-fix.yml delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-209-fix.yml delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/direct.txt.j2 delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/from-filesystem.txt.j2 delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/to-filesystem.txt.j2 delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/direct.txt.j2 delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/from-filesystem.txt.j2 delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/to-filesystem.txt.j2 delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/direct.txt.j2 delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/from-filesystem.txt.j2 delete mode 100644 ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/to-filesystem.txt.j2 create mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/spark_3.3.4.yml create mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/spark_3.4.2.yml delete mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.0.0.yml delete mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.1.0.yml delete mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.2.0.yml delete mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.3.0.yml delete mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.4.0.yml delete mode 100644 ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.5.0.yml delete mode 100644 ibm/mas_devops/roles/ocp_contentsourcepolicy/defaults/main.yml delete mode 100644 ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/main.yml delete mode 100644 ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/redhat-catalogs.yml.j2 delete mode 100644 ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/redhat-release.yml.j2 rename ibm/mas_devops/roles/{ocp_contentsourcepolicy => ocp_idms}/README.md (67%) create mode 100644 ibm/mas_devops/roles/ocp_idms/defaults/main.yml rename ibm/mas_devops/roles/{ocp_contentsourcepolicy => ocp_idms}/meta/main.yml (100%) create mode 100644 ibm/mas_devops/roles/ocp_idms/tasks/main.yml rename ibm/mas_devops/roles/{ocp_contentsourcepolicy => ocp_idms}/tasks/trust.yml (100%) rename ibm/mas_devops/roles/{ocp_contentsourcepolicy => ocp_idms}/tasks/update-pull-secret-dev.yml (100%) rename ibm/mas_devops/roles/{ocp_contentsourcepolicy => ocp_idms}/tasks/update-pull-secret.yml (100%) rename ibm/mas_devops/roles/{ocp_contentsourcepolicy => ocp_idms}/templates/configmap.yml.j2 (59%) rename ibm/mas_devops/roles/{ocp_contentsourcepolicy/templates/imagecontentsourcepolicy.yml.j2 => ocp_idms/templates/idms/mas-ibm-catalog.yml.j2} (51%) create mode 100644 ibm/mas_devops/roles/ocp_idms/templates/idms/mas-redhat-catalogs.yml.j2 create mode 100644 ibm/mas_devops/roles/ocp_idms/templates/idms/mas-redhat-release.yml.j2 diff --git a/.github/workflows/ansible-publish.yml b/.github/workflows/ansible-publish.yml index fc5ec1d344..cc9829d318 100644 --- a/.github/workflows/ansible-publish.yml +++ b/.github/workflows/ansible-publish.yml @@ -6,6 +6,11 @@ jobs: ansible-publish: runs-on: ubuntu-latest steps: + - name: Install Python v3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Checkout uses: actions/checkout@v2.3.1 @@ -64,7 +69,7 @@ jobs: podman images podman login --username "${{ secrets.QUAYIO_USERNAME }}" --password "${{ secrets.QUAYIO_PASSWORD }}" quay.io podman push quay.io/ibmmas/ansible-devops-ee:latest - podman push quay.io/ibmmas/ansible-devops-ee:${{ env.DOCKER_TAG }} + podman push quay.io/ibmmas/ansible-devops-ee:${{ env.DOCKER_TAG }} - name: Trigger ibm-mas/cli rebuild on Ansible Collection release run: | diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 2644d7c2ab..1b3cfc5ba2 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -9,6 +9,11 @@ jobs: ansible-build: runs-on: ubuntu-latest steps: + - name: Install Python v3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Checkout uses: actions/checkout@v2.3.1 # Without this option, we don't get the tag information diff --git a/build/bin/build-collection.sh b/build/bin/build-collection.sh index b09fadd02e..84a599cfb2 100644 --- a/build/bin/build-collection.sh +++ b/build/bin/build-collection.sh @@ -13,8 +13,8 @@ cat $GITHUB_WORKSPACE/ibm/mas_devops/galaxy.yml # Update this when we have new catalog -MAS_PREVIOUS_CATALOG='v9-241003-amd64' -MAS_LATEST_CATALOG='v9-241107-amd64' +MAS_PREVIOUS_CATALOG='v9-241107-amd64' +MAS_LATEST_CATALOG='v9-241205-amd64' # Update all the placeholders in the playbooks diff --git a/build/bin/copy-role-docs.sh b/build/bin/copy-role-docs.sh index 018ae96846..ab9a096eb3 100644 --- a/build/bin/copy-role-docs.sh +++ b/build/bin/copy-role-docs.sh @@ -51,7 +51,7 @@ copyDoc mongodb copyDoc nvidia_gpu copyDoc ocp_cluster_monitoring copyDoc ocp_config -copyDoc ocp_contentsourcepolicy +copyDoc ocp_idms copyDoc ocp_deprovision copyDoc ocp_efs copyDoc ocp_github_oauth diff --git a/docs/playbooks/ocp.md b/docs/playbooks/ocp.md index 5af2bfb6aa..0efb9732e8 100644 --- a/docs/playbooks/ocp.md +++ b/docs/playbooks/ocp.md @@ -16,7 +16,7 @@ export AWS_SECRET_ACCESS_KEY=xxx export ROSA_TOKEN=xxx export CLUSTER_NAME=masonrosa -export OCP_VERSION=4.15 +export OCP_VERSION=4.16 export ROSA_COMPUTE_NODES=5 export ROSA_CLUSTER_ADMIN_PASSWORD=xxx ansible-playbook ibm.mas_devops.ocp_rosa_provision @@ -31,7 +31,7 @@ This also supports upgrading the storage volume used for the cluster's internal ```bash export CLUSTER_NAME=masinst1 -export OCP_VERSION=4.15_openshift +export OCP_VERSION=4.16_openshift export IBMCLOUD_APIKEY=xxx export REBOOT_WORKER_NODES=true export CPD_ENTITLEMENT_KEY=xxx @@ -44,7 +44,7 @@ This playbook will provision a QuickBurn OCP cluster in IBM DevIT Fyre service, ```bash export CLUSTER_NAME=masinst1 -export OCP_VERSION=4.15 +export OCP_VERSION=4.16 export FYRE_USERNAME=xxx export FYRE_APIKEY=xxx export FYRE_PRODUCT_ID=xxx diff --git a/docs/playbooks/oneclick-predict.md b/docs/playbooks/oneclick-predict.md index 549672fbf4..b1068667e6 100644 --- a/docs/playbooks/oneclick-predict.md +++ b/docs/playbooks/oneclick-predict.md @@ -2,7 +2,7 @@ ## Prerequisites -You will need a RedHat OpenShift v4.12 cluster with IBM Maximo Application Suite Core v8.11 already be installed, the [oneclick-core](oneclick-core.md) playbook can be used to set this up. +You will need a RedHat OpenShift cluster with IBM Maximo Application Suite Core v8.11 already be installed, the [oneclick-core](oneclick-core.md) playbook can be used to set this up. ## Overview diff --git a/ibm/mas_devops/common_tasks/detect_airgap.yml b/ibm/mas_devops/common_tasks/detect_airgap.yml index 9bbd2a0ad6..ecbf958465 100644 --- a/ibm/mas_devops/common_tasks/detect_airgap.yml +++ b/ibm/mas_devops/common_tasks/detect_airgap.yml @@ -1,21 +1,35 @@ --- -# 1. Look for the airgap config - +# 1. Look for the legacy ICSP +# ----------------------------------------------------------------------------- - name: "detect-airgap : Look for the MAS ImageContentSourcePolicy" kubernetes.core.k8s_info: api_version: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy name: ibm-mas-and-dependencies - register: content_source_policy_lookup + register: icsp_lookup + + +# 2. Look for the new IDMS +# ----------------------------------------------------------------------------- +- name: "detect-airgap : Look for the MAS ImageDigestMirrorSet" + kubernetes.core.k8s_info: + api_version: config.openshift.io/v1 + kind: ImageDigestMirrorSet + label_selectors: + - mas.ibm.com/idmsContent=ibm + register: idms_lookup -- debug: - var: content_source_policy_lookup +# 3. Set the airgap boolena and print debug +# ----------------------------------------------------------------------------- - name: "detect-airgap : Set airgap_install property" set_fact: - airgap_install: "{{ content_source_policy_lookup.resources | length == 1 | bool }}" + airgap_install: "{{ idms_lookup.resources | length == 1 | bool }}" - name: "detect-airgap : Debug Airgap detection" debug: - msg: "Airgap environment .................... {{ airgap_install }}" + msg: + - "Legacy ICSP resources .................. {{ icsp_lookup.resources | length }}" + - "Current IDMS resources ................. {{ idms_lookup.resources | length }}" + - "Airgap environment ..................... {{ airgap_install }}" diff --git a/ibm/mas_devops/common_tasks/get_signed_ingress_cert.yml b/ibm/mas_devops/common_tasks/get_signed_ingress_cert.yml index 2f82bd22eb..7d5a948f96 100644 --- a/ibm/mas_devops/common_tasks/get_signed_ingress_cert.yml +++ b/ibm/mas_devops/common_tasks/get_signed_ingress_cert.yml @@ -8,7 +8,7 @@ - name: "1st attempt : Lookup for {{ ocp_ingress_tls_secret_name }} secret" no_log: true - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: "{{ ocp_ingress_tls_secret_name }}" @@ -47,7 +47,7 @@ - cluster_subdomain.resources is defined - cluster_subdomain.resources | length > 0 no_log: true - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: "{{ cluster_subdomain.resources[0].spec.domain | regex_search('[^.]*') }}" @@ -91,7 +91,7 @@ - name: Lookup default secret name containing cluster's ingress certificate based on IngressController default when: cluster_ingress_secret_name is defined and cluster_ingress_secret_name | length > 0 - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: "{{ cluster_ingress_secret_name }}" @@ -118,7 +118,7 @@ cluster_ingress_tls_crt is not defined block: - name: Get all TLS secrets - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret namespace: openshift-ingress diff --git a/ibm/mas_devops/common_vars/compatibility_matrix.yml b/ibm/mas_devops/common_vars/compatibility_matrix.yml index 53365effd1..b52044e900 100644 --- a/ibm/mas_devops/common_vars/compatibility_matrix.yml +++ b/ibm/mas_devops/common_vars/compatibility_matrix.yml @@ -1,5 +1,13 @@ --- compatibility_matrix: + 9.1.x-feature: + assist: [9.0.x] + iot: [9.0.x] + manage: [9.0.x, 9.1.x-feature] + monitor: [9.0.x] + optimizer: [9.0.x, 9.1.x-feature] + predict: [9.0.x] + visualinspection: [9.0.x, 9.1.x-feature] 9.0.x: assist: [8.8.x, 9.0.x] iot: [8.8.x, 9.0.x] @@ -39,6 +47,7 @@ compatibility_matrix: # There is probably a better way to do this by manipulating the channel string, but this is fast and cheap! upgrade_requirement: core: + 9.1.x-feature: 9.0.x 9.0.x: 8.11.x 8.11.x: 8.10.x 8.10.x: 8.9.x @@ -51,6 +60,7 @@ upgrade_requirement: 8.8.x: 8.7.x 8.7.x: 8.6.x manage: + 9.1.x-feature: 9.0.x 9.0.x: 8.7.x 8.7.x: 8.6.x 8.6.x: 8.5.x @@ -59,6 +69,7 @@ upgrade_requirement: 8.11.x: 8.10.x 8.10.x: 8.9.x optimizer: + 9.1.x-feature: 9.0.x 9.0.x: 8.5.x 8.5.x: 8.4.x 8.4.x: 8.3.x @@ -67,6 +78,7 @@ upgrade_requirement: 8.9.x: 8.8.x 8.8.x: 8.7.x visualinspection: + 9.1.x-feature: 9.0.x 9.0.x: 8.9.x 8.9.x: 8.8.x 8.8.x: 8.7.x @@ -74,6 +86,7 @@ upgrade_requirement: # There is probably a better way to do this by manipulating the channel string, but this is fast and cheap! upgrade_path: + 9.0.x: 9.1.x-feature 8.11.x: 9.0.x 8.10.x: 8.11.x 8.9.x: 8.10.x diff --git a/ibm/mas_devops/playbooks/mirror_add_assist.yml b/ibm/mas_devops/playbooks/mirror_add_assist.yml index d5a16cd9ac..0459f39a05 100644 --- a/ibm/mas_devops/playbooks/mirror_add_assist.yml +++ b/ibm/mas_devops/playbooks/mirror_add_assist.yml @@ -24,20 +24,20 @@ when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-assist - case_version: "{{ lookup('env', 'MAS_ASSIST_VERSION') | default (mas_catalog_metadata.mas_assist_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_ASSIST_VERSION') | default (mas_catalog_metadata.mas_assist_version[mas_channel], True) | replace('_', '.') }}" exclude_images: [] ibmpak_skip_dependencies: false - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-assist - manifest_version: "{{ lookup('env', 'MAS_ASSIST_VERSION') | default (mas_catalog_metadata.mas_assist_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_ASSIST_VERSION') | default (mas_catalog_metadata.mas_assist_version[mas_channel], True) | replace('_', '.') }}" # 2. IBM MAS Assist dependency from 9.0 - ibm-couchdb # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare when: - - couchdb_version is defined + - mas_catalog_metadata.couchdb_version is defined - mirror_mode != "from-filesystem" vars: case_name: ibm-couchdb @@ -46,7 +46,7 @@ ibmpak_skip_dependencies: false - role: ibm.mas_devops.mirror_images - when: couchdb_version is defined + when: mas_catalog_metadata.couchdb_version is defined vars: manifest_name: ibm-couchdb manifest_version: "{{ mas_catalog_metadata.couchdb_version }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_iot.yml b/ibm/mas_devops/playbooks/mirror_add_iot.yml index 11459a790e..17cbd83a4b 100644 --- a/ibm/mas_devops/playbooks/mirror_add_iot.yml +++ b/ibm/mas_devops/playbooks/mirror_add_iot.yml @@ -24,27 +24,27 @@ when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-iot - case_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_catalog_metadata.mas_iot_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_catalog_metadata.mas_iot_version[mas_channel], True) | replace('_', '.') }}" exclude_images: [] - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-iot - manifest_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_catalog_metadata.mas_iot_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_catalog_metadata.mas_iot_version[mas_channel], True) | replace('_', '.') }}" # 2. Eclipse Amlen # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_extras_prepare when: - - amlen_extras_version is defined + - mas_catalog_metadata.amlen_extras_version is defined - mirror_mode != "from-filesystem" vars: extras_name: amlen extras_version: "{{ mas_catalog_metadata.amlen_extras_version }}" - role: ibm.mas_devops.mirror_images - when: amlen_extras_version is defined + when: mas_catalog_metadata.amlen_extras_version is defined vars: manifest_name: extras_amlen manifest_version: "{{ mas_catalog_metadata.amlen_extras_version }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_manage.yml b/ibm/mas_devops/playbooks/mirror_add_manage.yml index cb307155ef..5c970bfeb8 100644 --- a/ibm/mas_devops/playbooks/mirror_add_manage.yml +++ b/ibm/mas_devops/playbooks/mirror_add_manage.yml @@ -25,11 +25,11 @@ when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-manage - case_version: "{{ lookup('env', 'MAS_MANAGE_VERSION') | default (mas_catalog_metadata.mas_manage_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_MANAGE_VERSION') | default (mas_catalog_metadata.mas_manage_version[mas_channel], True) | replace('_', '.') }}" exclude_images: [] image_group_filter: "{{ (mirror_icd == 'true') | ternary ('ibmmasManage,ibmMasManage,ibmmasMaximoIT', 'ibmmasManage,ibmMasManage') }}" - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-manage - manifest_version: "{{ lookup('env', 'MAS_MANAGE_VERSION') | default (mas_catalog_metadata.mas_manage_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_MANAGE_VERSION') | default (mas_catalog_metadata.mas_manage_version[mas_channel], True) | replace('_', '.') }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_monitor.yml b/ibm/mas_devops/playbooks/mirror_add_monitor.yml index c2655022d4..3454e3a0c4 100644 --- a/ibm/mas_devops/playbooks/mirror_add_monitor.yml +++ b/ibm/mas_devops/playbooks/mirror_add_monitor.yml @@ -24,13 +24,13 @@ when: mirror_mode != "from-filesystem" vars: case_name: ibm-data-dictionary - case_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (mas_catalog_metadata.dd_version, True) }}" + case_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (mas_catalog_metadata.dd_version, True) | replace('_', '.') }}" exclude_images: [] - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-data-dictionary - manifest_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (mas_catalog_metadata.dd_version, True) }}" + manifest_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (mas_catalog_metadata.dd_version, True) | replace('_', '.') }}" # 2. IBM Maximo Monitor @@ -39,10 +39,10 @@ when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-monitor - case_version: "{{ lookup('env', 'MAS_MONITOR_VERSION') | default (mas_catalog_metadata.mas_monitor_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_MONITOR_VERSION') | default (mas_catalog_metadata.mas_monitor_version[mas_channel], True) | replace('_', '.') }}" exclude_images: [] - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-monitor - manifest_version: "{{ lookup('env', 'MAS_MONITOR_VERSION') | default (mas_catalog_metadata.mas_monitor_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_MONITOR_VERSION') | default (mas_catalog_metadata.mas_monitor_version[mas_channel], True) | replace('_', '.') }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_optimizer.yml b/ibm/mas_devops/playbooks/mirror_add_optimizer.yml index eaa51b0436..00b5c64cac 100644 --- a/ibm/mas_devops/playbooks/mirror_add_optimizer.yml +++ b/ibm/mas_devops/playbooks/mirror_add_optimizer.yml @@ -24,10 +24,10 @@ when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-optimizer - case_version: "{{ lookup('env', 'MAS_OPTIMIZER_VERSION') | default (mas_catalog_metadata.mas_optimizer_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_OPTIMIZER_VERSION') | default (mas_catalog_metadata.mas_optimizer_version[mas_channel], True) | replace('_', '.') }}" exclude_images: [] - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-optimizer - manifest_version: "{{ lookup('env', 'MAS_OPTIMIZER_VERSION') | default (mas_catalog_metadata.mas_optimizer_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_OPTIMIZER_VERSION') | default (mas_catalog_metadata.mas_optimizer_version[mas_channel], True) | replace('_', '.') }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_predict.yml b/ibm/mas_devops/playbooks/mirror_add_predict.yml index d9d7a53af3..c94fcf1f94 100644 --- a/ibm/mas_devops/playbooks/mirror_add_predict.yml +++ b/ibm/mas_devops/playbooks/mirror_add_predict.yml @@ -24,11 +24,11 @@ when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-predict - case_version: "{{ lookup('env', 'MAS_PREDICT_VERSION') | default (mas_catalog_metadata.mas_predict_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_PREDICT_VERSION') | default (mas_catalog_metadata.mas_predict_version[mas_channel], True) | replace('_', '.') }}" exclude_images: [] ibmpak_skip_dependencies: false - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-predict - manifest_version: "{{ lookup('env', 'MAS_PREDICT_VERSION') | default (mas_catalog_metadata.mas_predict_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_PREDICT_VERSION') | default (mas_catalog_metadata.mas_predict_version[mas_channel], True) | replace('_', '.') }}" diff --git a/ibm/mas_devops/playbooks/mirror_add_visualinspection.yml b/ibm/mas_devops/playbooks/mirror_add_visualinspection.yml index 6fadf429eb..d4ed63486e 100644 --- a/ibm/mas_devops/playbooks/mirror_add_visualinspection.yml +++ b/ibm/mas_devops/playbooks/mirror_add_visualinspection.yml @@ -30,10 +30,10 @@ when: mirror_mode != "from-filesystem" vars: case_name: ibm-mas-visualinspection - case_version: "{{ lookup('env', 'MAS_VISUALINSPECTION_VERSION') | default (mas_catalog_metadata.mas_visualinspection_version[mas_channel], True) }}" + case_version: "{{ lookup('env', 'MAS_VISUALINSPECTION_VERSION') | default (mas_catalog_metadata.mas_visualinspection_version[mas_channel], True) | replace('_', '.') }}" exclude_images: [] - role: ibm.mas_devops.mirror_images vars: manifest_name: ibm-mas-visualinspection - manifest_version: "{{ lookup('env', 'MAS_VISUALINSPECTION_VERSION') | default (mas_catalog_metadata.mas_visualinspection_version[mas_channel], True) }}" + manifest_version: "{{ lookup('env', 'MAS_VISUALINSPECTION_VERSION') | default (mas_catalog_metadata.mas_visualinspection_version[mas_channel], True) | replace('_', '.') }}" diff --git a/ibm/mas_devops/playbooks/mirror_dependencies.yml b/ibm/mas_devops/playbooks/mirror_dependencies.yml index 3d009b7ef9..2c6c3843c7 100644 --- a/ibm/mas_devops/playbooks/mirror_dependencies.yml +++ b/ibm/mas_devops/playbooks/mirror_dependencies.yml @@ -36,6 +36,7 @@ mirror_cp4d: "{{ lookup('env', 'MIRROR_CP4D') | default ('False', True) | bool }}" cpd_48_or_higher: "{{ cpd_product_version is defined and cpd_product_version is version('4.8.0','>=') | bool }}" cpd_48: "{{ cpd_product_version is defined and cpd_product_version is version('4.8.0','==') | bool }}" + cpd_50: "{{ cpd_product_version is defined and cpd_product_version is version('5.0.0','==') | bool }}" # 7. Watson Studio Local # ------------------------------------------------------------------------- @@ -227,7 +228,7 @@ # Mirroring another version for Cloud Pak Foundation Services - role: ibm.mas_devops.mirror_case_prepare when: - - mirror_common_svcs + - mirror_common_svcs or (cpd_48_or_higher and mirror_cp4d) - mirror_mode != "from-filesystem" - mas_catalog_metadata.common_svcs_version_1 is defined - mas_catalog_metadata.common_svcs_version_1 != "" @@ -244,13 +245,39 @@ - role: ibm.mas_devops.mirror_images when: - - mirror_common_svcs + - mirror_common_svcs or (cpd_48_or_higher and mirror_cp4d) - mas_catalog_metadata.common_svcs_version_1 is defined - mas_catalog_metadata.common_svcs_version_1 != "" vars: manifest_name: ibm-cp-common-services manifest_version: "{{ mas_catalog_metadata.common_svcs_version_1 }}" + - role: ibm.mas_devops.mirror_case_prepare + when: + - mirror_common_svcs or (cpd_48_or_higher and mirror_cp4d) + - mirror_mode != "from-filesystem" + - mas_catalog_metadata.common_svcs_version_2 is defined + - mas_catalog_metadata.common_svcs_version_2 != "" + vars: + case_name: ibm-cp-common-services + case_version: "{{ mas_catalog_metadata.common_svcs_version_2 }}" + exclude_images: + - ibm-auditlogging + - ibm-cpp + - ibm-cs-healthcheck + - ibm-cs-monitoring + - ibm-events-operator + ibmpak_skip_dependencies: false + + - role: ibm.mas_devops.mirror_images + when: + - mirror_common_svcs or (cpd_48_or_higher and mirror_cp4d) + - mas_catalog_metadata.common_svcs_version_1 is defined + - mas_catalog_metadata.common_svcs_version_1 != "" + vars: + manifest_name: ibm-cp-common-services + manifest_version: "{{ mas_catalog_metadata.common_svcs_version_2 }}" + # 4. IBM Suite License Service # ------------------------------------------------------------------------- @@ -325,6 +352,40 @@ manifest_name: ibm-cp-datacore manifest_version: "{{ mas_catalog_metadata.cp4d_platform_version }}" + - role: ibm.mas_devops.mirror_extras_prepare + when: + - mirror_cp4d + - cpd_50 + - mirror_mode != "from-filesystem" + vars: + extras_name: spark + extras_version: "3.3.4" + + - role: ibm.mas_devops.mirror_images + when: + - mirror_cp4d + - cpd_50 + vars: + manifest_name: extras_spark + manifest_version: "3.3.4" + + - role: ibm.mas_devops.mirror_extras_prepare + when: + - mirror_cp4d + - cpd_50 + - mirror_mode != "from-filesystem" + vars: + extras_name: spark + extras_version: "3.4.2" + + - role: ibm.mas_devops.mirror_images + when: + - mirror_cp4d + - cpd_50 + vars: + manifest_name: extras_spark + manifest_version: "3.4.2" + # 6.2 CP4D Platform - IBM Licensing dependency # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare diff --git a/ibm/mas_devops/playbooks/ocp_convert_to_disconnected.yml b/ibm/mas_devops/playbooks/ocp_convert_to_disconnected.yml index fac4463f68..fe063536db 100644 --- a/ibm/mas_devops/playbooks/ocp_convert_to_disconnected.yml +++ b/ibm/mas_devops/playbooks/ocp_convert_to_disconnected.yml @@ -5,7 +5,7 @@ vars: ocp_operatorhub_disable_redhat_sources: true - ocp_release: "{{ lookup('env', 'OCP_RELEASE') | default('4.12', true) }}" + ocp_release: "{{ lookup('env', 'OCP_RELEASE') | default('4.16', true) }}" setup_redhat_release: true setup_redhat_catalogs: true diff --git a/ibm/mas_devops/playbooks/ocp_fyre_provision.yml b/ibm/mas_devops/playbooks/ocp_fyre_provision.yml index 21efc2d7cd..9a351e6b7e 100644 --- a/ibm/mas_devops/playbooks/ocp_fyre_provision.yml +++ b/ibm/mas_devops/playbooks/ocp_fyre_provision.yml @@ -2,7 +2,7 @@ - hosts: localhost vars: cluster_type: fyre - ocp_version: "{{ lookup('env', 'OCP_VERSION') | default('4.15', True) }}" + ocp_version: "{{ lookup('env', 'OCP_VERSION') | default('4.16', True) }}" # We update the cipher support on all installs, even though it's only technically # requires for FIPS clusters diff --git a/ibm/mas_devops/playbooks/ocp_roks_provision.yml b/ibm/mas_devops/playbooks/ocp_roks_provision.yml index e3303d196c..c6d0480f84 100644 --- a/ibm/mas_devops/playbooks/ocp_roks_provision.yml +++ b/ibm/mas_devops/playbooks/ocp_roks_provision.yml @@ -3,7 +3,7 @@ any_errors_fatal: true vars: cluster_type: roks - ocp_version: "{{ lookup('env', 'OCP_VERSION') | default('4.15_openshift', True) }}" + ocp_version: "{{ lookup('env', 'OCP_VERSION') | default('4.16_openshift', True) }}" prometheus_storage_class: ibmc-block-gold prometheus_alertmgr_storage_class: ibmc-file-gold-gid diff --git a/ibm/mas_devops/plugins/filter/filters.py b/ibm/mas_devops/plugins/filter/filters.py index bac70ec613..e2b7769d28 100644 --- a/ibm/mas_devops/plugins/filter/filters.py +++ b/ibm/mas_devops/plugins/filter/filters.py @@ -378,6 +378,16 @@ def format_pre_version_without_buildid(data): return data return data[:data.rfind("-")] +def format_pre_version_with_buildid(data): + """ + Versions in format 9.0.0-pre.stable-3757 cannot be used to compare with the version + reconciled by application operators, which is in format 9.0.0-pre.stable+3757. This function is to + format version to make it comparable + """ + if "pre" not in data or data.count("-") < 2: + return data + build_id_idx = data.rfind('-') + return f"{data[:build_id_idx]}{(data[build_id_idx:]).replace('-', '+')}" def get_db2_instance_name(binding_scope, mas_instance_id, mas_workspace_id, mas_application_id): if binding_scope == "": @@ -409,5 +419,6 @@ def filters(self): 'setSystemProperties': _setSystemProperties, 'format_pre_version_with_plus': format_pre_version_with_plus, 'format_pre_version_without_buildid': format_pre_version_without_buildid, + 'format_pre_version_with_buildid': format_pre_version_with_buildid, 'get_db2_instance_name': get_db2_instance_name } diff --git a/ibm/mas_devops/roles/aibroker/tasks/apikey/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/apikey/install/main.yml index 819131c758..a45f748433 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/apikey/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/apikey/install/main.yml @@ -1,6 +1,6 @@ --- - name: "Check if API key secret exists in namespace: {{ aibroker_namespace }}" - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: "{{ tenantNamespace }}----apikey-secret" diff --git a/ibm/mas_devops/roles/aibroker/tasks/s3/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/s3/install/main.yml index 80d7613d66..a5662de1ad 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/s3/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/s3/install/main.yml @@ -1,6 +1,6 @@ --- - name: "Check if S3 key secret exists in namespace: {{ aibroker_namespace }}" - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: "{{ tenantNamespace }}----s3-secret" diff --git a/ibm/mas_devops/roles/aibroker/tasks/s3/remove/main.yml b/ibm/mas_devops/roles/aibroker/tasks/s3/remove/main.yml index 5917c23a10..ddbd3f49f2 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/s3/remove/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/s3/remove/main.yml @@ -1,6 +1,6 @@ --- - name: "Check if S3 key secret exists in namespace: {{ aibroker_namespace }}" - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: "{{ tenantNamespace }}----s3-secret" diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 60048317d9..d930f93408 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -1,6 +1,6 @@ --- - name: 'Check if tenant namespace: {{ tenantNamespace }} exists' - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Namespace name: '{{ tenantNamespace }}' diff --git a/ibm/mas_devops/roles/aibroker/tasks/watsonx/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/watsonx/install/main.yml index 466818b286..3476be827b 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/watsonx/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/watsonx/install/main.yml @@ -1,6 +1,6 @@ --- - name: "Check if WatsonX AI API key secret exists in namespace: {{ aibroker_namespace }}" - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: "{{ tenantNamespace }}----wx-secret" diff --git a/ibm/mas_devops/roles/arcgis/tasks/lookup_trusted_certs.yml b/ibm/mas_devops/roles/arcgis/tasks/lookup_trusted_certs.yml index 1ef131c2c2..53b2719bf0 100644 --- a/ibm/mas_devops/roles/arcgis/tasks/lookup_trusted_certs.yml +++ b/ibm/mas_devops/roles/arcgis/tasks/lookup_trusted_certs.yml @@ -2,7 +2,7 @@ # For now, it will be required to include ca.crt and tls.crt into ArcGIS Trusted Certificates # in order for it to work well, thus we're using this automation to generate the tls files - name: "Lookup {{ mas_instance_id }}-internal-arcgis-tls" - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: "{{ mas_instance_id }}-internal-arcgis-tls" diff --git a/ibm/mas_devops/roles/cos/README.md b/ibm/mas_devops/roles/cos/README.md index 1bb0fdb1e2..5fdd2d45e9 100644 --- a/ibm/mas_devops/roles/cos/README.md +++ b/ibm/mas_devops/roles/cos/README.md @@ -14,25 +14,21 @@ Currently this role only supports generating a system-scoped ObjectStorageCfg re Role Variables - General ------------------------------------------------------------------------------- ### cos_type -Required. Which COS provider to use; can be set to either `ibm` for IBM Cloud Object Storage or `ocs` for OpenShift Container Storage +Which COS provider to use; can be set to either `ibm` for IBM Cloud Object Storage or `ocs` for OpenShift Container Storage +- **Required** - Environment Variable: `COS_TYPE` - Default Value: None ### cos_action -Required. Which action you want to run for the COS instance. You can either `provision` or `deprovision` a COS instance in your IBM Cloud account. +Which action you want to run for the COS instance. You can either `provision` or `deprovision` a COS instance in your IBM Cloud account. +- **Required** - Environment Variable: `COS_ACTION` - Default Value: `provision` -### cos_use_hmac -Supported values are true and false, this is used when ibm cloud-cos to be setup with hmac encrypted credentials. - -- Environment Variable: COS_USE_HMAC -- Default: false - -### cluster ingres tls secret name -Specify the name of the cluster's ingres tls secret which contains the default router certificate. +### ocp_ingress_tls_secret_name +This can be set to the name of the cluster default router certificate's secret, for use in rare cases where the role is unable to determine the secret name automatically. - Optional - Environment Variable: `OCP_INGRESS_TLS_SECRET_NAME` @@ -74,6 +70,13 @@ Provide an optional role when cos service credential is getting created during C - Environment Variable: `COS_RESOURCE_KEY_IAM_ROLE` - Default Value: `Manager` +### cos_use_hmac +Set to false to disable the use of HMAC encrypted credentials, however doing so will prevent use of this COS instance with Maximo Application Suite. + +- Optional +- Environment Variable: `COS_USE_HMAC` +- Default: `true` + ### cos_apikey Required if cos_type is set to `ibm`. Provide your less priviledged IBM Cloud API Key for COS only diff --git a/ibm/mas_devops/roles/cos/defaults/main.yml b/ibm/mas_devops/roles/cos/defaults/main.yml index 74c7e019d7..89756a6632 100644 --- a/ibm/mas_devops/roles/cos/defaults/main.yml +++ b/ibm/mas_devops/roles/cos/defaults/main.yml @@ -8,7 +8,7 @@ cos_service: "cloud-object-storage" # --------------------------------------------------------------------------------------------------------------------- mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" -cos_use_hmac: "{{ lookup('env', 'COS_USE_HMAC') | default(false, true) }}" +cos_use_hmac: "{{ lookup('env', 'COS_USE_HMAC') | default(true, true) }}" # OpenShift Container Storage Object Storage (ocs) # --------------------------------------------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create.yml index 1a71800ed6..5296d90db7 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create.yml @@ -33,7 +33,7 @@ - name: "Create IBM COS Bucket: Debug general info" debug: msg: - - "IBM Cloud Resource Group ........... {{ ibmcloud_resourcegroup }}" + - "IBM Cloud Resource Group ........... {{ cos_resourcegroup }}" - "IBM COS url ........................ {{ cos_url }}" - "IBM COS instance name .............. {{ cos_instance_name }}" - "IBM COS location info .............. {{ cos_location_info }}" @@ -52,7 +52,7 @@ # --------------------------------------------------------------------------------------------------------------------- - name: "ibm : Retrieve resource group guid" ibm.cloudcollection.ibm_resource_group_info: - name: "{{ ibmcloud_resourcegroup }}" + name: "{{ cos_resourcegroup }}" ibmcloud_api_key: "{{ cos_apikey }}" register: rg_info diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/delete.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/delete.yml index 1144bcbccb..1eb50afd30 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/delete.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/delete.yml @@ -16,7 +16,7 @@ - name: "Delete IBM COS Bucket: Debug general info" debug: msg: - - "IBM Cloud Resource Group ........... {{ ibmcloud_resourcegroup }}" + - "IBM Cloud Resource Group ........... {{ cos_resourcegroup }}" - "IBM COS url ........................ {{ cos_url }}" - "IBM COS instance name .............. {{ cos_instance_name }}" - "IBM COS location info .............. {{ cos_location_info }}" @@ -28,7 +28,7 @@ # --------------------------------------------------------------------------------------------------------------------- - name: "ibm : Retrieve resource group guid" ibm.cloudcollection.ibm_resource_group_info: - name: "{{ ibmcloud_resourcegroup }}" + name: "{{ cos_resourcegroup }}" ibmcloud_api_key: "{{ cos_apikey }}" register: rg_info diff --git a/ibm/mas_devops/roles/cp4d/tasks/prereq-check/check-cpfs-version.yml b/ibm/mas_devops/roles/cp4d/tasks/prereq-check/check-cpfs-version.yml index 6effabf527..a37f658824 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/prereq-check/check-cpfs-version.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/prereq-check/check-cpfs-version.yml @@ -12,7 +12,7 @@ block: - name: Lookup ibm-common-service-operator packagemanifest - k8s_info: + kubernetes.core.k8s_info: api_version: packages.operators.coreos.com/v1 kind: PackageManifest name: ibm-common-service-operator diff --git a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-cpfs.yml b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-cpfs.yml index fd7b04b050..1016973e4e 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-cpfs.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-cpfs.yml @@ -2,7 +2,7 @@ # This won't install CPFS v3 as we will continue to have separate role for that (common_service). --- - name: Lookup ibm-common-service packagemanifest - k8s_info: + kubernetes.core.k8s_info: api_version: packages.operators.coreos.com/v1 kind: PackageManifest name: ibm-common-service-operator diff --git a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-licensing.yml b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-licensing.yml index 7d3ab55b7e..28a70a8a2f 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-licensing.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-licensing.yml @@ -7,7 +7,7 @@ - "Catalog source namespace .................. {{ cpd_ibm_licensing_catalog_source_namespace }}" - name: Lookup ibm-licensing-operator-app packagemanifest - k8s_info: + kubernetes.core.k8s_info: api_version: packages.operators.coreos.com/v1 kind: PackageManifest namespace: "{{ cpd_ibm_licensing_catalog_source_namespace }}" diff --git a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-namespace-scope.yml b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-namespace-scope.yml index 0b3d628669..6cae4833ce 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-namespace-scope.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-namespace-scope.yml @@ -7,7 +7,7 @@ register: og_cpd_operators_info - name: "Lookup ibm-namespace-scope-operator packagemanifest" - k8s_info: + kubernetes.core.k8s_info: api_version: packages.operators.coreos.com/v1 kind: PackageManifest name: ibm-namespace-scope-operator diff --git a/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml b/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml new file mode 100644 index 0000000000..5df2b778da --- /dev/null +++ b/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml @@ -0,0 +1,112 @@ +--- +# 1. Wait for zen metastore cluster to start +# ----------------------------------------------------------------------------- +- name: "wait-zenmetastore-edb : Wait for Zen Metastore EDB Cluster to be created" + kubernetes.core.k8s_info: + kind: Cluster + namespace: "{{ cpd_instance_namespace }}" + name: "zen-metastore-edb" + register: zenmetastoreCluster + retries: 120 # Give 60 minutes for the zenService to start Zen Metastore Pods (Logs show this taking ~20 minutes in a good run) + delay: 30 + until: zenmetastoreCluster.resources[0].status is defined + +# 2. For V4.8, We need to patch the postgres licensing job acccording to https://www.ibm.com/support/pages/node/7158524 +- name: "wait-zenmetastore-edb : Check and display the create-postgres-license-config license expiry date" + when: cpd_48 + block: + - name: "wait-zenmetastore-edb : Fetch the license expiry date" + kubernetes.core.k8s_info: + kind: Cluster + namespace: "{{ cpd_instance_namespace }}" + name: "zen-metastore-edb" + register: zenmetastoreCluster + until: + zenmetastoreCluster.resources[0].spec.instances is defined + and zenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus is defined + retries: 40 # Give 20 minutes + delay: 30 + - debug: + msg: + - "License Expiration .. {{ zenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration | default ('') }}" + - "License Status ...... {{ zenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus}}" + +- name: "wait-zenmetastore-edb : Update create-postgres-license-config job" + when: + - cpd_48 + - zenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration is not defined + block: + - name: "wait-zenmetastore-edb : Wait for create-postgres-license-config Job to be Completed or Failed (10s delay)" + kubernetes.core.k8s_info: + api_version: batch/v1 + kind: Job + name: "create-postgres-license-config" + namespace: "{{ cpd_operators_namespace }}" + register: _job_info + until: + - _job_info.resources is defined + - _job_info.resources | length > 0 + - (_job_info.resources | json_query('[*].status.conditions[?type==`Complete`][].status') | select ('match','True') | list | length == 1) or + (_job_info.resources | json_query('[*].status.conditions[?type==`Failed`][].status') | select ('match','True') | list | length == 1) + retries: 30 + delay: 10 + + - name: "wait-zenmetastore-edb : Recreate the job with up to date license image" + shell: >- + oc get job create-postgres-license-config -n {{ cpd_operators_namespace }} -o yaml | \ + sed -e 's/operator.ibm.com\/opreq-control: "true"/operator.ibm.com\/opreq-control: "false"/' \ + -e 's|\(image: \).*|\1"cp.icr.io/cp/cpd/edb-postgres-license-provider@sha256:c1670e7dd93c1e65a6659ece644e44aa5c2150809ac1089e2fd6be37dceae4ce"|' \ + -e '/controller-uid:/d' | \ + oc replace --force -f - + register: _job_recreate_output + + - debug: + msg: "Recreate the job: {{ _job_recreate_output }}" + + - name: "wait-zenmetastore-edb : Wait for create-postgres-license-config Job to be Completed or Failed (10s delay)" + kubernetes.core.k8s_info: + api_version: batch/v1 + kind: Job + name: "create-postgres-license-config" + namespace: "{{ cpd_operators_namespace }}" + register: _job_info + until: + - _job_info.resources is defined + - _job_info.resources | length > 0 + - (_job_info.resources | json_query('[*].status.conditions[?type==`Complete`][].status') | select ('match','True') | list | length == 1) or + (_job_info.resources | json_query('[*].status.conditions[?type==`Failed`][].status') | select ('match','True') | list | length == 1) + retries: 30 + delay: 10 + + - name: "wait-zenmetastore-edb : Check and display the license expiry date" + kubernetes.core.k8s_info: + kind: Cluster + namespace: "{{ cpd_instance_namespace }}" + name: "zen-metastore-edb" + register: newzenmetastoreCluster + until: + newzenmetastoreCluster.resources[0].spec.instances is defined + and newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus is defined + and "Valid license" in newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus + retries: 40 # Give 20 minutes + delay: 30 + + - debug: + msg: + - "License Expiration .. {{ newzenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration | default ('') }}" + - "License Status ...... {{ newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus | default ('')}}" + +# 3. Wait for zen metastore replica pods to become ready +# ----------------------------------------------------------------------------- +- name: "wait-zenmetastore-edb : Wait for ZenMetastore pods to be become ready" + kubernetes.core.k8s_info: + kind: Cluster + namespace: "{{ cpd_instance_namespace }}" + name: "zen-metastore-edb" + register: zenmetastoreCluster + retries: 40 # Give 20 minutes for the pods to become ready + delay: 30 + until: >- + zenmetastoreCluster.resources[0].spec.instances is defined + and zenmetastoreCluster.resources[0].status.readyInstances is defined + and zenmetastoreCluster.resources[0].spec.instances == zenmetastoreCluster.resources[0].status.readyInstances diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/main.yml b/ibm/mas_devops/roles/cp4d_service/tasks/main.yml index a1df07cb5c..f2018fdf82 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/main.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/main.yml @@ -48,7 +48,7 @@ cpd_platform_cr_name: "ibmcpd" - name: "Lookup Cloud Pak for Data CR named '{{ cpd_platform_cr_name }}'" - k8s_info: + kubernetes.core.k8s_info: api_version: cpd.ibm.com/v1 kind: Ibmcpd name: "{{ cpd_platform_cr_name }}" diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml index f304fa5bef..9645398c7e 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml @@ -109,7 +109,16 @@ - cpd_48_or_higher # elastic search operator was just introduced with cpd 4.8 - not skip_ibm_entitlement_injection # eventually we hope to be able to skip patching the elastic search cr with image pull secret, but not for now -# 5. Wait for CCS CR to be ready +# 5. Wait for CouchDB Stateful Set to be ready +# ----------------------------------------------------------------------------- +# There have been issues with CouchDB not starting due to Persistent Storage, +# This task restarts any failing pods +- include_tasks: "tasks/wait/wait-couchdb.yml" + when: + - cpd_48 + + +# 6. Wait for CCS CR to be ready # ----------------------------------------------------------------------------- # Note: We can't fail early when we see Failed status, as the operator will # report failed multiple times during initial reconcile. diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml new file mode 100644 index 0000000000..732b04b342 --- /dev/null +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml @@ -0,0 +1,108 @@ +--- +# 1. Wait for couch-db stateful set to start all the replica pods +# ----------------------------------------------------------------------------- +- name: "wait-couchdb: Wait for CouchDB pods to be created" + kubernetes.core.k8s_info: + kind: StatefulSet + namespace: "{{ cpd_instance_namespace }}" + name: "wdp-couchdb" + register: couchdbStatefulSet + retries: 40 # Give 20 minutes for the ccs Operator to start CouchDB Pods (Logs show this taking ~7 minutes in a good run) + delay: 30 + until: >- + (( couchdbStatefulSet.resources[0].status is defined + and couchdbStatefulSet.resources[0].status.replicas is defined + and couchdbStatefulSet.resources[0].status.replicas == 0 ) + or ( couchdbStatefulSet.resources[0].status is defined + and couchdbStatefulSet.resources[0].status.updatedReplicas is defined + and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.updatedReplicas )) + + +# 2. Wait for couchdb replica pods to become ready +# ----------------------------------------------------------------------------- +- name: "wait-couchdb: Wait for CouchDB pods to be become ready" + kubernetes.core.k8s_info: + kind: StatefulSet + namespace: "{{ cpd_instance_namespace }}" + name: "wdp-couchdb" + register: couchdbStatefulSet + retries: 10 # Give 5 minutes for the pods to become ready + delay: 30 + until: >- + couchdbStatefulSet.resources[0].status.readyReplicas is defined + and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.readyReplicas + #ignore-errors: true # If this fails then we restart pending pods below + failed_when: false + +# 2. Restart any couchDB pods that are still Pending +# ----------------------------------------------------------------------------- +- set_fact: + is_couchdb_ready: true + when: + couchdbStatefulSet.resources[0].status.readyReplicas is defined + and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.readyReplicas + +- name: "wait-couchdb: Detecting and restarting pending CouchDB Pods" + block: + - name: "install-cp4d : Get pending CouchDB Pods" + kubernetes.core.k8s_info: + api_version: v1 + kind: Pod + label_selectors: + - "app=couchdb" + field_selectors: + - "status.phase=Pending" + namespace: "{{ cpd_instance_namespace }}" + register: pending_pod_lookup + + - set_fact: + pending_pod_names: "{{ pending_pod_lookup.resources | map(attribute='metadata.name') }}" + + - debug: + msg: "Restarting pending CouchDB Pods: {{ pending_pod_names }}" + + - name: "wait-couchdb: Restarting pending CouchDB Pods" + kubernetes.core.k8s: + state: absent + api_version: v1 + kind: Pod + namespace: "{{ cpd_instance_namespace }}" + name: "{{ item }}" + loop: "{{ pending_pod_names }}" + + # 3. Wait again couchdb replica pods to become ready + # ----------------------------------------------------------------------------- + - name: "wait-couchdb: Wait for CouchDB pods to be become ready" + kubernetes.core.k8s_info: + kind: StatefulSet + namespace: "{{ cpd_instance_namespace }}" + name: "wdp-couchdb" + register: couchdbStatefulSet + retries: 10 # Give another 5 minutes for the pods to become ready + delay: 30 + until: >- + couchdbStatefulSet.resources[0].status.readyReplicas is defined + and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.readyReplicas + failed_when: false # We handle and log the failure below. + + - name: "wait-couchdb: Fail if CouchDB pods are not ready" + block: + - name: "install-cp4d : Get Pending CouchDB Pods" + kubernetes.core.k8s_info: + api_version: v1 + kind: Pod + label_selectors: + - "app=couchdb" + field_selectors: + - "status.phase=Pending" + namespace: "{{ cpd_instance_namespace }}" + register: pending_pod_lookup + + - fail: + msg: + - "CouchDB pods are not ready {{ couchdbStatefulSet.resources[0].status }}" + - "Pending CouchDB Pods: {{ pending_pod_lookup.resources | map(attribute='metadata.name') }}" + when: + couchdbStatefulSet.resources[0].status.replicas != couchdbStatefulSet.resources[0].status.readyReplicas + + when: is_couchdb_ready is not defined diff --git a/ibm/mas_devops/roles/dro/tasks/gencfg/main.yml b/ibm/mas_devops/roles/dro/tasks/gencfg/main.yml index 00454baa02..96615e2aa0 100644 --- a/ibm/mas_devops/roles/dro/tasks/gencfg/main.yml +++ b/ibm/mas_devops/roles/dro/tasks/gencfg/main.yml @@ -16,7 +16,7 @@ # 1. Lookup the endpoint route # ----------------------------------------------------------------------------- - name: "drocfg : Get ibm-data-reporter route" - k8s_info: + kubernetes.core.k8s_info: api_version: route.openshift.io/v1 kind: Route name: ibm-data-reporter @@ -53,7 +53,7 @@ type: kubernetes.io/service-account-token - name: "drocfg : Get ibm-data-reporter-operator-api secret" - k8s_info: + kubernetes.core.k8s_info: kind: Secret name: ibm-data-reporter-operator-api-token namespace: "{{ dro_namespace }}" @@ -115,7 +115,7 @@ # 6. DRO Migration: Clean BASCfg of UDS # ----------------------------------------------------------------------------- - name: "gencfg: get existing UDS BASCfg" - k8s_info: + kubernetes.core.k8s_info: api_version: config.mas.ibm.com/v1 kind: BasCfg register: udscfgs diff --git a/ibm/mas_devops/roles/kafka/defaults/main.yaml b/ibm/mas_devops/roles/kafka/defaults/main.yaml index 35eb153bf7..a379192a7d 100644 --- a/ibm/mas_devops/roles/kafka/defaults/main.yaml +++ b/ibm/mas_devops/roles/kafka/defaults/main.yaml @@ -6,12 +6,12 @@ kafka_action: "{{ lookup('env', 'KAFKA_ACTION') | default('install', true) }}" amq_exists: false # initialize variable kafka_defaults: strimzi: - version: "3.7.0" # Supported versions for strimzi-cluster-operator.v0.40.0 are: [3.6.0, 3.6.1, 3.7.0] + version: "3.9.0" # Supported versions for strimzi-cluster-operator.v0.45.0 are: ['3.8.0', '3.8.1', '3.9.0'] namespace: "strimzi" operator_name: "strimzi-kafka-operator" alias_name: "Strimzi" redhat: - version: "3.7.0" + version: "3.9.0" namespace: "amq-streams" operator_name: "amq-streams" alias_name: "Red Hat AMQ Streams" diff --git a/ibm/mas_devops/roles/kafka/tasks/provider/redhat/lookup-supported-kafka-versions.yml b/ibm/mas_devops/roles/kafka/tasks/provider/redhat/lookup-supported-kafka-versions.yml index 7ba48f1ba1..70b54bb6a7 100644 --- a/ibm/mas_devops/roles/kafka/tasks/provider/redhat/lookup-supported-kafka-versions.yml +++ b/ibm/mas_devops/roles/kafka/tasks/provider/redhat/lookup-supported-kafka-versions.yml @@ -59,10 +59,11 @@ - name: "Creating dictionary while looping over page content" ignore_errors: true vars: + url_line: 'td class="description">','') | replace('','') | regex_replace(' 0 %}/{{ env_prefix }}{% endif %}" # If the prefix is not empty then add a slash -registry_public_url: "{{ registry_public_host }}:{{ registry_public_port }}{{ registry_prefix }}" +registry_public_url: "{{ registry_public_host }}{% if registry_public_port != '' %}:{{ registry_public_port }}{% endif %}" + +registry_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') | default('', true) }}" +registry_public_url_with_path: "{{ registry_public_url }}{% if registry_prefix | length > 0 %}/{{ registry_prefix }}{% endif %}" # Development config # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mirror_case_prepare/tasks/main.yml b/ibm/mas_devops/roles/mirror_case_prepare/tasks/main.yml index 197fe4c3d5..44c85b3ca4 100644 --- a/ibm/mas_devops/roles/mirror_case_prepare/tasks/main.yml +++ b/ibm/mas_devops/roles/mirror_case_prepare/tasks/main.yml @@ -8,7 +8,7 @@ - case_name is defined and case_name != "" - case_version is defined and case_version != "" - mirror_working_dir is defined and mirror_working_dir != "" - - registry_public_url is defined and registry_public_url != ":" + - registry_public_url_with_path is defined and registry_public_url_with_path != ":" fail_msg: "One or more required properties are missing" # 2. Check for required software @@ -51,13 +51,15 @@ shell: oc ibm-pak get {{ case_name }} --version {{ case_version }} {{ ibmpak_flag_insecure }} {{ ibmpak_flag_skip_verify }} {{ ibmpak_flag_skip_dependencies }} register: ibmpak_get_result -# Team messed up the release and the version we will get back is 8.7.0+20230925.114420 rather than 8.7.0 -- name: "Workaround for ibm-mas-manage 8.7.0" - when: - - case_name == "ibm-mas-manage" - - case_version == "8.7.0" +- name: "{{ case_name }} : Get Build from output" set_fact: - case_version: "8.7.0+20230925.114420" + build_version: "{{ ibmpak_get_result.stdout | regex_search(regex,'\\1') | first }}" + vars: + regex: 'version: ([a-zA-Z0-9\+\.\-]*) is complete' + +- name: "{{ case_name }} : Debug build version" + debug: + var: build_version # 5. Remove excluded images # ----------------------------------------------------------------------------- @@ -66,38 +68,14 @@ when: - exclude_images is defined - exclude_images | length > 0 - shell: rm ~/.ibm-pak/data/cases/{{ case_name }}/{{ case_version }}/{{ item }}-*-images.csv + shell: rm ~/.ibm-pak/data/cases/{{ case_name }}/{{ build_version }}/{{ item }}-*-images.csv # We ignore errors, because if we can't delete something that isn't there it # doesn't really matter, e.g. removing ibm-mas-safety which isn't present since MAS v8.8 ignore_errors: true register: excludeImagesResult loop: "{{ exclude_images }}" -# 6. Generate mirror-manifest -# ----------------------------------------------------------------------------- -- name: Generate mirror-manifest with image group filter - block: - - name: "{{ case_name }} : Generate the mirror manifest from the CASE bundle (direct)" - shell: "oc ibm-pak generate mirror-manifests {{ case_name }} {{ registry_public_url }} --version {{ case_version }} --filter {{ image_group_filter }}" - register: ibmpak_gen1_result - - - name: "{{ case_name }} : Generate the mirror manifest from the CASE bundle (indirect)" - shell: "oc ibm-pak generate mirror-manifests {{ case_name }} file:// --version {{ case_version }} --final-registry {{ registry_public_url }} --filter {{ image_group_filter }}" - register: ibmpak_gen2_result - when: image_group_filter is defined - -- name: Generate mirror-manifest without image group filter - block: - - name: "{{ case_name }} : Generate the mirror manifest from the CASE bundle (direct)" - shell: "oc ibm-pak generate mirror-manifests {{ case_name }} {{ registry_public_url }} --version {{ case_version }}" - register: ibmpak_gen1_result - - - name: "{{ case_name }} : Generate the mirror manifest from the CASE bundle (indirect)" - shell: "oc ibm-pak generate mirror-manifests {{ case_name }} file:// --version {{ case_version }} --final-registry {{ registry_public_url }}" - register: ibmpak_gen2_result - when: image_group_filter is not defined - -# 7. Collect generated files +# 6. Prepare for mirror-manifest # ----------------------------------------------------------------------------- - name: "{{ case_name }} : Create target directories (direct)" file: @@ -114,38 +92,37 @@ path: "{{ mirror_working_dir }}/manifests/from-filesystem" state: directory -# 8. Save the manifests to our working directory -# ----------------------------------------------------------------------------- -# Team messed up the release and the version we will get back is 8.7.0+20230925.114420 rather than 8.7.0 -# -# Note: We use _manifest_version to avoid conflict with manifest_version that is used as an input to -# the mirror_images role. If we set a manifest_version fact it will override the value passed into all -# mirror_image roles invoked in a playbook. - name: "Set manifest version" set_fact: _manifest_version: "{{ case_version }}" -- name: "Workaround for ibm-mas-manage 8.7.0" - when: - - case_name == "ibm-mas-manage" - - case_version == "8.7.0+20230925.114420" - set_fact: - _manifest_version: "8.7.0" +# 7. Generate mirror-manifest direct +# ----------------------------------------------------------------------------- +- name: "{{ case_name }} : Generate the mirror manifest from the CASE bundle (direct)" + shell: "oc ibm-pak generate mirror-manifests {{ case_name }} {{ registry_public_url_with_path }} --version {{ build_version }} {% if image_group_filter is defined %} --filter {{ image_group_filter }} {% endif %}" + register: ibmpak_gen1_result + +- name: "{{ case_name }} : Copy images-mapping" + ansible.builtin.copy: + src: ~/.ibm-pak/data/mirror/{{ case_name }}/{{ build_version }}/images-mapping.txt + dest: "{{ mirror_working_dir }}/manifests/direct/{{ case_name }}_{{ _manifest_version }}.txt" + +# 8. Generate mirror-manifest indirect +# ----------------------------------------------------------------------------- +- name: "{{ case_name }} : Generate the mirror manifest from the CASE bundle (indirect)" + shell: "oc ibm-pak generate mirror-manifests {{ case_name }} file:// --version {{ build_version }} --final-registry {{ registry_public_url_with_path }} {% if image_group_filter is defined %} --filter {{ image_group_filter }} {% endif %}" + register: ibmpak_gen2_result - name: "{{ case_name }} : Copy images-mapping-to-filesystem" ansible.builtin.copy: - src: ~/.ibm-pak/data/mirror/{{ case_name }}/{{ case_version }}/images-mapping-to-filesystem.txt + src: ~/.ibm-pak/data/mirror/{{ case_name }}/{{ build_version }}/images-mapping-to-filesystem.txt dest: "{{ mirror_working_dir }}/manifests/to-filesystem/{{ case_name }}_{{ _manifest_version }}.txt" - name: "{{ case_name }} : Copy images-mapping-from-filesystem" ansible.builtin.copy: - src: ~/.ibm-pak/data/mirror/{{ case_name }}/{{ case_version }}/images-mapping-from-filesystem.txt + src: ~/.ibm-pak/data/mirror/{{ case_name }}/{{ build_version }}/images-mapping-from-filesystem.txt dest: "{{ mirror_working_dir }}/manifests/from-filesystem/{{ case_name }}_{{ _manifest_version }}.txt" -- name: "{{ case_name }} : Copy images-mapping" - ansible.builtin.copy: - src: ~/.ibm-pak/data/mirror/{{ case_name }}/{{ case_version }}/images-mapping.txt - dest: "{{ mirror_working_dir }}/manifests/direct/{{ case_name }}_{{ _manifest_version }}.txt" # 9. IBM SLS 3.5.0 Bad Digest Hack # ----------------------------------------------------------------------------- @@ -154,23 +131,6 @@ when: case_name == "ibm-sls" and case_version == "3.5.0" include_tasks: "tasks/sls-350-fix.yml" -# 10. IBM UDS Entitled Image Hacks -# ----------------------------------------------------------------------------- -# The UDS CASE bundle includes one entitled image - cp/uds/uds-submodule:2.0.8 -- name: "IBM UDS 2.0.8 workaround" - when: case_name == "ibm-uds" and case_version == "2.0.8" - include_tasks: "tasks/uds-208-fix.yml" - -# The UDS CASE bundle includes one entitled image - cp/uds/uds-submodule:2.0.9 -- name: "IBM UDS 2.0.9 workaround" - when: case_name == "ibm-uds" and case_version == "2.0.9" - include_tasks: "tasks/uds-209-fix.yml" - -# The UDS CASE bundle includes one entitled image - cp/uds/uds-submodule:2.0.10 -- name: "IBM UDS 2.0.10 workaround" - when: case_name == "ibm-uds" and case_version == "2.0.10" - include_tasks: "tasks/uds-2010-fix.yml" - # 11. IBM Maximo IoT Hacks # ----------------------------------------------------------------------------- # The IoT CASE bundle for 8.6.0 has an incorrect image digest in it diff --git a/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-2010-fix.yml b/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-2010-fix.yml deleted file mode 100644 index 43b0c358cd..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-2010-fix.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# IBM UDS 2.0.10 Entitled Image Hack -# ----------------------------------------------------------------------------- -# The UDS CASE bundle includes one entitled image - cp/uds/uds-submodule:2.0.10 - -- template: - src: uds2010fix/direct.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/direct/{{ case_name }}_{{ case_version }}.txt" - -- template: - src: uds2010fix/to-filesystem.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/to-filesystem/{{ case_name }}_{{ case_version }}.txt" - -- template: - src: uds2010fix/from-filesystem.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/from-filesystem/{{ case_name }}_{{ case_version }}.txt" diff --git a/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-208-fix.yml b/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-208-fix.yml deleted file mode 100644 index 6191ba1d4e..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-208-fix.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# IBM UDS 2.0.8 Entitled Image Hack -# ----------------------------------------------------------------------------- -# The UDS CASE bundle includes one entitled image - cp/uds/uds-submodule:2.0.8 - -- template: - src: uds208fix/direct.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/direct/{{ case_name }}_{{ case_version }}.txt" - -- template: - src: uds208fix/to-filesystem.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/to-filesystem/{{ case_name }}_{{ case_version }}.txt" - -- template: - src: uds208fix/from-filesystem.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/from-filesystem/{{ case_name }}_{{ case_version }}.txt" diff --git a/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-209-fix.yml b/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-209-fix.yml deleted file mode 100644 index 2c933425f4..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/tasks/uds-209-fix.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# IBM UDS 2.0.9 Entitled Image Hack -# ----------------------------------------------------------------------------- -# The UDS CASE bundle includes one entitled image - cp/uds/uds-submodule:2.0.9 - -- template: - src: uds209fix/direct.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/direct/{{ case_name }}_{{ case_version }}.txt" - -- template: - src: uds209fix/to-filesystem.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/to-filesystem/{{ case_name }}_{{ case_version }}.txt" - -- template: - src: uds209fix/from-filesystem.txt.j2 - dest: "{{ mirror_working_dir }}/manifests/from-filesystem/{{ case_name }}_{{ case_version }}.txt" diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/iot860fix/direct.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/iot860fix/direct.txt.j2 index ca0c51e03a..f3c0d0f39c 100644 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/iot860fix/direct.txt.j2 +++ b/ibm/mas_devops/roles/mirror_case_prepare/templates/iot860fix/direct.txt.j2 @@ -1,103 +1,103 @@ -cp.icr.io/cp/mas/ltpakeys-generator@sha256:0fabd61f1cc6313e57b74ed393fff1eb7445001c7421efde93c0a755ec70133d={{ registry_public_url }}/cp/mas/ltpakeys-generator:2.3.35 -cp.icr.io/cp/wiotp/action-mgr@sha256:989c76823d3ef446ef5e367d0fdfa44a9cf73f5658051553b43f04a14a1093f6={{ registry_public_url }}/cp/wiotp/action-mgr:2.0.53 -cp.icr.io/cp/wiotp/api-actions@sha256:4dba1876b2aeaffb3fb1d24091c62ecebc0954cbdf624e55fb9ed8549f4bf02f={{ registry_public_url }}/cp/wiotp/api-actions:2.1.47 -cp.icr.io/cp/wiotp/api-authentication@sha256:158351ddf133c2bb982df9afe947c2fd842dc9c89e7d17d3f2f598a1d5bb7a5f={{ registry_public_url }}/cp/wiotp/api-authentication:3.3.49 -cp.icr.io/cp/wiotp/api-authorization@sha256:04762e32d7b1d1705c24c56bf567f3e1cbd6a72104dd20fe4758c96a17d9aa9b={{ registry_public_url }}/cp/wiotp/api-authorization:5.3.46 -cp.icr.io/cp/wiotp/api-connectors@sha256:62942ed423bcb793ba872b133b810c3211c626dcce820bc8232601bfc0a3c7b0={{ registry_public_url }}/cp/wiotp/api-connectors:4.1.3 -cp.icr.io/cp/wiotp/api-dashboard@sha256:dd98151e83eada6c97fd0c51a4cfcf561c7af383fe653170f777036ab7278863={{ registry_public_url }}/cp/wiotp/api-dashboard:3.0.82 -cp.icr.io/cp/wiotp/api-devicemgmt@sha256:ed3bb00f6e823052faeb643cbb2a6e02a3772981a9146aae64f830a81755823f={{ registry_public_url }}/cp/wiotp/api-devicemgmt:3.2.37 -cp.icr.io/cp/wiotp/api-devops@sha256:fae4ba84a2b7af5b036906bd6d430e4b086e672bb1d2fca3073cb9001007b174={{ registry_public_url }}/cp/wiotp/api-devops:3.7.49 -cp.icr.io/cp/wiotp/api-importconnectors@sha256:73b0c2ce525ed53ed4426487c25f0f1579fd51a09104b67eb15f489724d3c9c2={{ registry_public_url }}/cp/wiotp/api-importconnectors:2.0.30 -cp.icr.io/cp/wiotp/api-mbgadmin@sha256:f60f533237cffcb760fb29c93e2faebddefa97be78a03d7997f4d177ece14fd2={{ registry_public_url }}/cp/wiotp/api-mbgadmin:2.4.74 -cp.icr.io/cp/wiotp/api-messagesight@sha256:7f37bfd840122de1f3aba6d84adf426a0cf26a89b367ad056ee06d79f9ddffe6={{ registry_public_url }}/cp/wiotp/api-messagesight:3.1.41 -cp.icr.io/cp/wiotp/api-org@sha256:307f7023dc004eb6c5fd579e17f1bc47cbcf9fc454c4b819e56d6d8463553544={{ registry_public_url }}/cp/wiotp/api-org:8.2.29 -cp.icr.io/cp/wiotp/api-organizations@sha256:e4cb0a5f4acb347ff27696a82b96e46ff3044d92904716597bbc37f393ffbbe3={{ registry_public_url }}/cp/wiotp/api-organizations:10.0.39 -cp.icr.io/cp/wiotp/api-pipeline@sha256:4e926b536b7713aa8a2ee166aaeebaac5d9e74a78fbf03abeeac484387054917={{ registry_public_url }}/cp/wiotp/api-pipeline:1.1.0 -cp.icr.io/cp/wiotp/api-resourcecontroller@sha256:5b8b4ef64ff65e727e2c7a7134785a673ba2cdb134655800d84753107c90a8ce={{ registry_public_url }}/cp/wiotp/api-resourcecontroller:3.0.72 -cp.icr.io/cp/wiotp/api-riskmgmt-secguardian@sha256:ef80d0b8fbdd629b3d47ffd9fc00b0cd63e69e5b51133a9816e09117646463bd={{ registry_public_url }}/cp/wiotp/api-riskmgmt-secguardian:5.1.47 -cp.icr.io/cp/wiotp/api-s2s@sha256:476966d25e6086e36b4212cebfe8a8e6146ad80021a64eeb3dad0ea55a1850cd={{ registry_public_url }}/cp/wiotp/api-s2s:10.1.29 -cp.icr.io/cp/wiotp/api-schemas@sha256:2885df674610b01e98daff4a676c03b178fbf49e5696fd872148a3f4d16602a5={{ registry_public_url }}/cp/wiotp/api-schemas:4.0.31 -cp.icr.io/cp/wiotp/api-state@sha256:10b9f206b48235dfc5187496fada9077fb7d6e0082d760b4f1299d5afe4f9447={{ registry_public_url }}/cp/wiotp/api-state:5.0.32 -cp.icr.io/cp/wiotp/api-status@sha256:4164d2d2b5cf0f6314b4fdfa11e780003f4632d9a2545f7fb041990fb45bcad2={{ registry_public_url }}/cp/wiotp/api-status:3.4.8 -cp.icr.io/cp/wiotp/auth-store@sha256:1725ed33844394a808d6b154e4eff2016d0a21dac73fb397aa2c772dc533cc3f={{ registry_public_url }}/cp/wiotp/auth-store:2.4.51 -cp.icr.io/cp/wiotp/config-store@sha256:c14d7489d9ca23441076e57758c673422e7abb00785570ed8159c4caece9cd35={{ registry_public_url }}/cp/wiotp/config-store:4.0.46 -cp.icr.io/cp/wiotp/connectors-store@sha256:b6f4a3078bef6ec614db4d8114592e7768b46e294ce215ec34dee4bd62f34a9b={{ registry_public_url }}/cp/wiotp/connectors-store:8.0.34 -cp.icr.io/cp/wiotp/data-import-connector@sha256:fa0ce13bb49df74d3fb4aeaa3211f6dcdb9d311e1cb3f3b979bc33dbc04b1e61={{ registry_public_url }}/cp/wiotp/data-import-connector:2.0.66 -cp.icr.io/cp/wiotp/datapower-config@sha256:547f74ac50f1724c037ca8d2a15f34263cf21309d3eb3d6e37e85c1a750d5126={{ registry_public_url }}/cp/wiotp/datapower-config:10.3.1 -cp.icr.io/cp/wiotp/datapower@sha256:d4122be5bb21f1333032a446849124ae1d769195576ad0cc4d5ea1d76d7c460c={{ registry_public_url }}/cp/wiotp/datapower:7.2.1 -cp.icr.io/cp/wiotp/deprovagent-actions@sha256:a9a7cb86f430e9f8a1b52bb312d8a96b8f424d128007d7dab5ab42aa6a3306ca={{ registry_public_url }}/cp/wiotp/deprovagent-actions:1.1.65 -cp.icr.io/cp/wiotp/deprovagent-auth@sha256:d621f386046b9bd942db9aabf8b1b2da0225ff7f4de716ad5207f87654edb4cd={{ registry_public_url }}/cp/wiotp/deprovagent-auth:2.3.61 -cp.icr.io/cp/wiotp/deprovagent-dm@sha256:22ba1a9fb466c609a08db49adba23715c7874e0caf119683b74a7efbec87fc6f={{ registry_public_url }}/cp/wiotp/deprovagent-dm:2.1.61 -cp.icr.io/cp/wiotp/deprovagent-fpl@sha256:59963eef65e070034c8642772b804f63d670fe9f14a06d91c60e7c3d220e3073={{ registry_public_url }}/cp/wiotp/deprovagent-fpl:1.0.14 -cp.icr.io/cp/wiotp/deprovagent-guardian@sha256:e1cea24868cbea17c5415b2fe445cdc80a3ee824c3e100e0f9239b5d66814bff={{ registry_public_url }}/cp/wiotp/deprovagent-guardian:3.1.61 -cp.icr.io/cp/wiotp/deprovagent-orgmgmt@sha256:28927e185b80ad8ec971f4e1f8746c0a4434590f91bc481f2ce3ed8fddec9c5d={{ registry_public_url }}/cp/wiotp/deprovagent-orgmgmt:2.1.63 -cp.icr.io/cp/wiotp/deprovagent-provision@sha256:64caf0bf3b94f875b0c92a2a9f9567a4f6f6cdb68f869d84f91700d5043edef9={{ registry_public_url }}/cp/wiotp/deprovagent-provision:4.4.57 -cp.icr.io/cp/wiotp/deprovagent-registry@sha256:a57a0e64c81a771cf4ef6960489ca1ac7980783c2001b435e74de0129b782690={{ registry_public_url }}/cp/wiotp/deprovagent-registry:2.2.57 -cp.icr.io/cp/wiotp/deprovagent-state@sha256:37055b0230d4372e47b4981dd8c858adcf387254c81d7be5fa58a6b51e5100e8={{ registry_public_url }}/cp/wiotp/deprovagent-state:1.2.62 -cp.icr.io/cp/wiotp/deprovagent-webui@sha256:2de7621297e2e8306cb5f5e1475007058194083a33e97d0b9042e81602632323={{ registry_public_url }}/cp/wiotp/deprovagent-webui:1.2.64 -cp.icr.io/cp/wiotp/device-store@sha256:86879b7fccf9f984406633f7e4e8edceed03e3ff51d2e0705de149b9e06515da={{ registry_public_url }}/cp/wiotp/device-store:2.2.39 -cp.icr.io/cp/wiotp/devicemgmt-server@sha256:15252f496515254daa8747f3e39026f2a03eefed99062afcd2b5cbfbbda236a3={{ registry_public_url }}/cp/wiotp/devicemgmt-server:3.2.35 -cp.icr.io/cp/wiotp/entity-connector@sha256:1dc81de46abbad063e6db41badf74758fad0f0ab7236833329e4b1b9f437d3c7={{ registry_public_url }}/cp/wiotp/entity-connector:1.1.85 -cp.icr.io/cp/wiotp/entity-manager@sha256:7a72775b4c2058387b5298d6ddeabc0add47ccad0b41e0879b4bf6c081f3baa5={{ registry_public_url }}/cp/wiotp/entity-manager:2.1.93 -cp.icr.io/cp/wiotp/functionsexecutor@sha256:4b5cb0daaff9ff8366a715927a265ee6ca87e4ea49b55e8b5df19a7ad7c70871={{ registry_public_url }}/cp/wiotp/functionsexecutor:1.1.0 -cp.icr.io/cp/wiotp/graphite-exporter@sha256:afb2d4af1cd0bb8e52de3e1da92b4e853aa490b1e2580f0a434ba8ceeba12fbd={{ registry_public_url }}/cp/wiotp/graphite-exporter:2.1.79 -cp.icr.io/cp/wiotp/historian-configuration-manager@sha256:374d51451948d28367486f9582c2facb0232e53561608d41b95fa6e61daeaeab={{ registry_public_url }}/cp/wiotp/historian-configuration-manager:7.1.2 -cp.icr.io/cp/wiotp/historian-connector@sha256:2d57ac856c1dc34096bc294d80e3184c0f0d121598ae58a66aa82e59875bf1fb={{ registry_public_url }}/cp/wiotp/historian-connector:8.1.5 -cp.icr.io/cp/wiotp/ibm-iot-actions@sha256:d150aaa38ed8f3dc32685b248b4422c1848a2d87a2b168a92b50558cdfeb03e2={{ registry_public_url }}/cp/wiotp/ibm-iot-actions:2.3.0 -cp.icr.io/cp/wiotp/ibm-iot-auth@sha256:1ee7931a46ae4c0a257e9587305d2a427c2a1cf140aebf5866a69a4f58fdc789={{ registry_public_url }}/cp/wiotp/ibm-iot-auth:6.3.1 -cp.icr.io/cp/wiotp/ibm-iot-datapower@sha256:9c7874c1088091fc1d88a3882785ad80b9d54b639a988d7be32b8b16b5120131={{ registry_public_url }}/cp/wiotp/ibm-iot-datapower:7.2.1 -cp.icr.io/cp/wiotp/ibm-iot-devops@sha256:253b6274c2d87bb0ba12df129000dd34e8081a37b04bd653ab66ef6287576086={{ registry_public_url }}/cp/wiotp/ibm-iot-devops:10.2.1 -cp.icr.io/cp/wiotp/ibm-iot-dm@sha256:46a83b53e8ec51d1db6c6c86f877e72f31af4ba06b002bf9fb1165bd77658678={{ registry_public_url }}/cp/wiotp/ibm-iot-dm:2.2.1 -cp.icr.io/cp/wiotp/ibm-iot-dsc@sha256:72e8c6b0ce0f772a60c415b095494a0f744de638288bcf1a3aebd2b28e049bdc={{ registry_public_url }}/cp/wiotp/ibm-iot-dsc:6.2.1 -cp.icr.io/cp/wiotp/ibm-iot-fpl@sha256:b21d37ff6414e0ac423c1bf95e38b747b345eebe5152901fa09ad6954788a02e={{ registry_public_url }}/cp/wiotp/ibm-iot-fpl:1.1.0 -cp.icr.io/cp/wiotp/ibm-iot-guardian@sha256:3af0f8a7ce6011e27988d2ec779d567e729bd6bdc584112d78cfea1f546d8a69={{ registry_public_url }}/cp/wiotp/ibm-iot-guardian:4.2.1 -cp.icr.io/cp/wiotp/ibm-iot-mbgx@sha256:5adf9cefbfdee1cc3a30598e6c55794ada9d577a975161891303eb1e2826dd67={{ registry_public_url }}/cp/wiotp/ibm-iot-mbgx:3.3.1 -cp.icr.io/cp/wiotp/ibm-iot-mfgx@sha256:f1c5a8bb66c2fd879763c8981dd5a9ab0d2de34426eb59310816049f3cd5f92f={{ registry_public_url }}/cp/wiotp/ibm-iot-mfgx:6.3.1 -cp.icr.io/cp/wiotp/ibm-iot-monitor@sha256:5241962c295cfe71a6d4f4c90306fdc765a7486f81ec4674caf64c3d476da194={{ registry_public_url }}/cp/wiotp/ibm-iot-monitor:2.3.0 -cp.icr.io/cp/wiotp/ibm-iot-orgmgmt@sha256:4f575f552e11599d35c941c3d53d968201ea9019eec7e601244bb891cc2b5cdd={{ registry_public_url }}/cp/wiotp/ibm-iot-orgmgmt:6.3.0 -cp.icr.io/cp/wiotp/ibm-iot-provision@sha256:06f28fa79b0bd9184c1b0c5c43f0e07e1cdd70d06d503b20e7dc1d43b8ce71a7={{ registry_public_url }}/cp/wiotp/ibm-iot-provision:10.2.1 -cp.icr.io/cp/wiotp/ibm-iot-registry@sha256:730fc97a9e72eebac635aea847e567c5e1f89708c585221e1453be062c97c0ac={{ registry_public_url }}/cp/wiotp/ibm-iot-registry:8.2.1 -cp.icr.io/cp/wiotp/ibm-iot-state@sha256:fe9ded36f0bcaeeacb9d270760baaffb7a7081fd54717cc30b7473f5154226ce={{ registry_public_url }}/cp/wiotp/ibm-iot-state:5.2.1 -cp.icr.io/cp/wiotp/ibm-iot-webui@sha256:b1c3e07ab5bd302c24f93753ae1c1395a14da9879dede6eaecd1e3a327090270={{ registry_public_url }}/cp/wiotp/ibm-iot-webui:6.2.1 -cp.icr.io/cp/wiotp/ibm-iot-workspace@sha256:c8ae641bbe8b46352dce21495a600a407d7bdaa88dda23c1ca87b89d5368293e={{ registry_public_url }}/cp/wiotp/ibm-iot-workspace:2.5.1 -cp.icr.io/cp/wiotp/kafkamodel-actions@sha256:da423d4fbb7d8dbada9b1ddb6d7d55b63f6c95e97e58e61ff69aaa1dc6d02b5b={{ registry_public_url }}/cp/wiotp/kafkamodel-actions:1.4.6 -cp.icr.io/cp/wiotp/kafkamodel-auth@sha256:ad1e3090cbec3dc68dbed33baf4466bad053c14a3439db6766a497aced8e572e={{ registry_public_url }}/cp/wiotp/kafkamodel-auth:1.7.6 -cp.icr.io/cp/wiotp/kafkamodel-fpl@sha256:29ac84c25d04af5828a2af92420904ebbeb594e67ee093c80f4801bdcbf2b0d8={{ registry_public_url }}/cp/wiotp/kafkamodel-fpl:1.1.6 -cp.icr.io/cp/wiotp/kafkamodel-mfgx@sha256:f950837e92d28ac2904b2ce39d30817e2ae4348a003d0b83898b87c412813a2c={{ registry_public_url }}/cp/wiotp/kafkamodel-mfgx:1.3.6 -cp.icr.io/cp/wiotp/kafkamodel-orgmgmt@sha256:d8f7be2c19042439dea0ecc487c2d8a6e94860af0642c52af0a651c9f48bc1d5={{ registry_public_url }}/cp/wiotp/kafkamodel-orgmgmt:2.2.6 -cp.icr.io/cp/wiotp/kafkamodel-registry@sha256:de865fff88e4d25811287de01d3e998d7f246340f97004e7535fed11a8ee6eb8={{ registry_public_url }}/cp/wiotp/kafkamodel-registry:1.4.6 -cp.icr.io/cp/wiotp/kafkamodel-state@sha256:0a8c7645ccf1ccecff108d1fbc993161e099e72becad085dcb0e7b837cfcabfb={{ registry_public_url }}/cp/wiotp/kafkamodel-state:1.4.6 -cp.icr.io/cp/wiotp/masuseragent@sha256:06a65a089e71b9cb07bf64da7bdd75faa265970de868309e2da460cdc5aed1d3={{ registry_public_url }}/cp/wiotp/masuseragent:1.4.8 -cp.icr.io/cp/wiotp/messagesight-configurator@sha256:e90f01c84e4a84fefafdcf02731b8f79c08d23d3501c3cdccb1f532514c346bf={{ registry_public_url }}/cp/wiotp/messagesight-configurator:5.14.35 -cp.icr.io/cp/wiotp/messagesight@sha256:24597be9d139ba12c4573fff74a41f1323d77be9443469d8633fbd4bde8e6744={{ registry_public_url }}/cp/wiotp/messagesight:5.14.35 -cp.icr.io/cp/wiotp/monagent-iot@sha256:da292da546da4acdc27f9865b0040384dcb79ca16eb9455ea907ffdcd5c37b0c={{ registry_public_url }}/cp/wiotp/monagent-iot:1.1.0 -cp.icr.io/cp/wiotp/monagent-msserver@sha256:9067b5c576333dcd272883cb20ccd45950cdd5f30e2d566c8af7501e2079b1be={{ registry_public_url }}/cp/wiotp/monagent-msserver:4.3.70 -cp.icr.io/cp/wiotp/monagent-org@sha256:0ac2638d4d4c590fa7a3566aa6b8a902de54e8ecbd93c61c851a0cbe461cedf3={{ registry_public_url }}/cp/wiotp/monagent-org:2.1.83 -cp.icr.io/cp/wiotp/mongomodel-actions@sha256:b30714e01aedcf2654f718308a29591157937290301cb2c157ea1536975bdd79={{ registry_public_url }}/cp/wiotp/mongomodel-actions:1.2.76 -cp.icr.io/cp/wiotp/mongomodel-dm@sha256:e9eb535d9b156c1e6f7c6fc3a3ddf6de346fadc0e30aef6a7f32354cba4bc9df={{ registry_public_url }}/cp/wiotp/mongomodel-dm:1.3.80 -cp.icr.io/cp/wiotp/mongomodel-dsc@sha256:f58651c00dd8c5f1f87ca323c521fe02dbd1787449a79f860e4a635c5c3cf667={{ registry_public_url }}/cp/wiotp/mongomodel-dsc:1.2.78 -cp.icr.io/cp/wiotp/mongomodel-guardian@sha256:db8fbf2d5a1cc7b453dc262954e9de782d0d074bacc67b8bdf90c18154813349={{ registry_public_url }}/cp/wiotp/mongomodel-guardian:1.3.75 -cp.icr.io/cp/wiotp/mongomodel-orgmgmt@sha256:c277dfa1c54e33aab55b195745bd74c967bb7febdf92f20d6f612eb36eb1372a={{ registry_public_url }}/cp/wiotp/mongomodel-orgmgmt:2.0.75 -cp.icr.io/cp/wiotp/mongomodel-provision@sha256:76f16025df17d94ea32047624123b967186c95d74514e344f4dd892efd7f84fc={{ registry_public_url }}/cp/wiotp/mongomodel-provision:2.5.65 -cp.icr.io/cp/wiotp/mongomodel-registry@sha256:bb49e23af6c150de2bcfdced8c617c7354814694c8137caacfabbb7bf78d7032={{ registry_public_url }}/cp/wiotp/mongomodel-registry:2.0.71 -cp.icr.io/cp/wiotp/mongomodel-state@sha256:c3df1e9b8d2e895b92088a733745e683d1dd16d6b77b4a768c3df21cfcb30c9f={{ registry_public_url }}/cp/wiotp/mongomodel-state:1.2.72 -cp.icr.io/cp/wiotp/mongomodel-webui@sha256:9cab9ba25341778564f5b98567a429b9e12f6cc4da55d92c935f7561c2a0ab28={{ registry_public_url }}/cp/wiotp/mongomodel-webui:2.3.79 -cp.icr.io/cp/wiotp/mqtt-connector@sha256:b51922a2283c4581b910878e7d542b4ab2de6e5d670305657eedd7f0a15239b4={{ registry_public_url }}/cp/wiotp/mqtt-connector:2.0.73 -cp.icr.io/cp/wiotp/msproxy-monitor@sha256:b53dfac38f002814c2a33c1c13778b55954a095a2132308543624dd61d27b05f={{ registry_public_url }}/cp/wiotp/msproxy-monitor:8.5.42 -cp.icr.io/cp/wiotp/msproxy@sha256:417a3d8f94a19ededfe6b53d4ecd3d8622abd7bbbf990d3ac3ae303b92dbc1f8={{ registry_public_url }}/cp/wiotp/msproxy:8.5.42 -cp.icr.io/cp/wiotp/org-store@sha256:72796a6a6c92299f68b3d5581b7325db9d3600cadd22e67133c43eb275295873={{ registry_public_url }}/cp/wiotp/org-store:4.0.46 -cp.icr.io/cp/wiotp/orgdeprovmgr@sha256:78bc78197b119c0f92fa299d9a9473d4dc221d1b1c851f304c394e960a0c6baa={{ registry_public_url }}/cp/wiotp/orgdeprovmgr:2.2.66 -cp.icr.io/cp/wiotp/orgpoolmgr@sha256:f4424d91a381116815b5a2448c7a606bb2a0ff6c45f74ab92bf5d31f7153bb0c={{ registry_public_url }}/cp/wiotp/orgpoolmgr:3.1.78 -cp.icr.io/cp/wiotp/orgprovmgr@sha256:f71ede6293c82e2c85e89e0d331f3674541effcabfc4aeb851ae9f6551ec4611={{ registry_public_url }}/cp/wiotp/orgprovmgr:2.1.64 -cp.icr.io/cp/wiotp/pipelinerouter@sha256:1c0047681cf5a8eb6c867c3581d613a64a03c326e23c7a9b292cd55573ca2171={{ registry_public_url }}/cp/wiotp/pipelinerouter:1.1.0 -cp.icr.io/cp/wiotp/provagent-guardian@sha256:37b9d3a251c2597c0eb4e5a744b9fb871a0436691649d434e42ad8877a7b2d05={{ registry_public_url }}/cp/wiotp/provagent-guardian:2.1.65 -cp.icr.io/cp/wiotp/s2s-store@sha256:4bbf05141df77240f76ca90b183b3200822b60d3d13df05e5be39c232a8b1c6a={{ registry_public_url }}/cp/wiotp/s2s-store:6.1.31 -cp.icr.io/cp/wiotp/sqlmodel-auth@sha256:6ec3d1dc0ab7b04be6b3d948606e030f2ceaff91271650b37fee7b87b60643f1={{ registry_public_url }}/cp/wiotp/sqlmodel-auth:1.1.17 -cp.icr.io/cp/wiotp/sqlmodel-fpl@sha256:c65076e777fa55fb94625cc228cea0e45b2e7397e1ed4aa1e9d82ea0d823e215={{ registry_public_url }}/cp/wiotp/sqlmodel-fpl:1.0.15 -cp.icr.io/cp/wiotp/sqlmodel-state@sha256:9a1551a240c5b925f272e0c43cfa949b099d12136d98dfb9c213608e0446a27b={{ registry_public_url }}/cp/wiotp/sqlmodel-state:1.0.62 -cp.icr.io/cp/wiotp/state-updater@sha256:0fdffec079afe9277c6ce656b270cc1ef1b2aaca80e0b425ddc4c1c596d72aab={{ registry_public_url }}/cp/wiotp/state-updater:2.0.35 -cp.icr.io/cp/wiotp/statsd@sha256:5cb1c6005fa136d1c1cb0c0b04ae4cfb5a6588d98d74bf046784725d1a50aa3d={{ registry_public_url }}/cp/wiotp/statsd:1.3.70 -cp.icr.io/cp/wiotp/ui-dashboard@sha256:d0ccf2e8175c83a77aa8136ceebe46188800eb82db6a745ef455e2812b068916={{ registry_public_url }}/cp/wiotp/ui-dashboard:7.1.67 -icr.io/cpopen/ibm-mas-iot-operator-bundle@sha256:e576f5731f052b99d95bba2f1491c37349259de1696ae46124ff16bdc253bb83={{ registry_public_url }}/cpopen/ibm-mas-iot-operator-bundle:8.6.0 -icr.io/cpopen/ibm-mas-iot-operator-catalog@sha256:09b6560684c9f651a9c837b080498e46162a334fb02ff5bb7a5b8faf75734e70={{ registry_public_url }}/cpopen/ibm-mas-iot-operator-catalog:latest -icr.io/cpopen/ibm-mas-iot@sha256:870d956908cec2bbc7d6fd5ffa9863fe02d643c183910e7e6fd211d391027564={{ registry_public_url }}/cpopen/ibm-mas-iot:8.6.0 \ No newline at end of file +cp.icr.io/cp/mas/ltpakeys-generator@sha256:0fabd61f1cc6313e57b74ed393fff1eb7445001c7421efde93c0a755ec70133d={{ registry_public_url_with_path }}/cp/mas/ltpakeys-generator:2.3.35 +cp.icr.io/cp/wiotp/action-mgr@sha256:989c76823d3ef446ef5e367d0fdfa44a9cf73f5658051553b43f04a14a1093f6={{ registry_public_url_with_path }}/cp/wiotp/action-mgr:2.0.53 +cp.icr.io/cp/wiotp/api-actions@sha256:4dba1876b2aeaffb3fb1d24091c62ecebc0954cbdf624e55fb9ed8549f4bf02f={{ registry_public_url_with_path }}/cp/wiotp/api-actions:2.1.47 +cp.icr.io/cp/wiotp/api-authentication@sha256:158351ddf133c2bb982df9afe947c2fd842dc9c89e7d17d3f2f598a1d5bb7a5f={{ registry_public_url_with_path }}/cp/wiotp/api-authentication:3.3.49 +cp.icr.io/cp/wiotp/api-authorization@sha256:04762e32d7b1d1705c24c56bf567f3e1cbd6a72104dd20fe4758c96a17d9aa9b={{ registry_public_url_with_path }}/cp/wiotp/api-authorization:5.3.46 +cp.icr.io/cp/wiotp/api-connectors@sha256:62942ed423bcb793ba872b133b810c3211c626dcce820bc8232601bfc0a3c7b0={{ registry_public_url_with_path }}/cp/wiotp/api-connectors:4.1.3 +cp.icr.io/cp/wiotp/api-dashboard@sha256:dd98151e83eada6c97fd0c51a4cfcf561c7af383fe653170f777036ab7278863={{ registry_public_url_with_path }}/cp/wiotp/api-dashboard:3.0.82 +cp.icr.io/cp/wiotp/api-devicemgmt@sha256:ed3bb00f6e823052faeb643cbb2a6e02a3772981a9146aae64f830a81755823f={{ registry_public_url_with_path }}/cp/wiotp/api-devicemgmt:3.2.37 +cp.icr.io/cp/wiotp/api-devops@sha256:fae4ba84a2b7af5b036906bd6d430e4b086e672bb1d2fca3073cb9001007b174={{ registry_public_url_with_path }}/cp/wiotp/api-devops:3.7.49 +cp.icr.io/cp/wiotp/api-importconnectors@sha256:73b0c2ce525ed53ed4426487c25f0f1579fd51a09104b67eb15f489724d3c9c2={{ registry_public_url_with_path }}/cp/wiotp/api-importconnectors:2.0.30 +cp.icr.io/cp/wiotp/api-mbgadmin@sha256:f60f533237cffcb760fb29c93e2faebddefa97be78a03d7997f4d177ece14fd2={{ registry_public_url_with_path }}/cp/wiotp/api-mbgadmin:2.4.74 +cp.icr.io/cp/wiotp/api-messagesight@sha256:7f37bfd840122de1f3aba6d84adf426a0cf26a89b367ad056ee06d79f9ddffe6={{ registry_public_url_with_path }}/cp/wiotp/api-messagesight:3.1.41 +cp.icr.io/cp/wiotp/api-org@sha256:307f7023dc004eb6c5fd579e17f1bc47cbcf9fc454c4b819e56d6d8463553544={{ registry_public_url_with_path }}/cp/wiotp/api-org:8.2.29 +cp.icr.io/cp/wiotp/api-organizations@sha256:e4cb0a5f4acb347ff27696a82b96e46ff3044d92904716597bbc37f393ffbbe3={{ registry_public_url_with_path }}/cp/wiotp/api-organizations:10.0.39 +cp.icr.io/cp/wiotp/api-pipeline@sha256:4e926b536b7713aa8a2ee166aaeebaac5d9e74a78fbf03abeeac484387054917={{ registry_public_url_with_path }}/cp/wiotp/api-pipeline:1.1.0 +cp.icr.io/cp/wiotp/api-resourcecontroller@sha256:5b8b4ef64ff65e727e2c7a7134785a673ba2cdb134655800d84753107c90a8ce={{ registry_public_url_with_path }}/cp/wiotp/api-resourcecontroller:3.0.72 +cp.icr.io/cp/wiotp/api-riskmgmt-secguardian@sha256:ef80d0b8fbdd629b3d47ffd9fc00b0cd63e69e5b51133a9816e09117646463bd={{ registry_public_url_with_path }}/cp/wiotp/api-riskmgmt-secguardian:5.1.47 +cp.icr.io/cp/wiotp/api-s2s@sha256:476966d25e6086e36b4212cebfe8a8e6146ad80021a64eeb3dad0ea55a1850cd={{ registry_public_url_with_path }}/cp/wiotp/api-s2s:10.1.29 +cp.icr.io/cp/wiotp/api-schemas@sha256:2885df674610b01e98daff4a676c03b178fbf49e5696fd872148a3f4d16602a5={{ registry_public_url_with_path }}/cp/wiotp/api-schemas:4.0.31 +cp.icr.io/cp/wiotp/api-state@sha256:10b9f206b48235dfc5187496fada9077fb7d6e0082d760b4f1299d5afe4f9447={{ registry_public_url_with_path }}/cp/wiotp/api-state:5.0.32 +cp.icr.io/cp/wiotp/api-status@sha256:4164d2d2b5cf0f6314b4fdfa11e780003f4632d9a2545f7fb041990fb45bcad2={{ registry_public_url_with_path }}/cp/wiotp/api-status:3.4.8 +cp.icr.io/cp/wiotp/auth-store@sha256:1725ed33844394a808d6b154e4eff2016d0a21dac73fb397aa2c772dc533cc3f={{ registry_public_url_with_path }}/cp/wiotp/auth-store:2.4.51 +cp.icr.io/cp/wiotp/config-store@sha256:c14d7489d9ca23441076e57758c673422e7abb00785570ed8159c4caece9cd35={{ registry_public_url_with_path }}/cp/wiotp/config-store:4.0.46 +cp.icr.io/cp/wiotp/connectors-store@sha256:b6f4a3078bef6ec614db4d8114592e7768b46e294ce215ec34dee4bd62f34a9b={{ registry_public_url_with_path }}/cp/wiotp/connectors-store:8.0.34 +cp.icr.io/cp/wiotp/data-import-connector@sha256:fa0ce13bb49df74d3fb4aeaa3211f6dcdb9d311e1cb3f3b979bc33dbc04b1e61={{ registry_public_url_with_path }}/cp/wiotp/data-import-connector:2.0.66 +cp.icr.io/cp/wiotp/datapower-config@sha256:547f74ac50f1724c037ca8d2a15f34263cf21309d3eb3d6e37e85c1a750d5126={{ registry_public_url_with_path }}/cp/wiotp/datapower-config:10.3.1 +cp.icr.io/cp/wiotp/datapower@sha256:d4122be5bb21f1333032a446849124ae1d769195576ad0cc4d5ea1d76d7c460c={{ registry_public_url_with_path }}/cp/wiotp/datapower:7.2.1 +cp.icr.io/cp/wiotp/deprovagent-actions@sha256:a9a7cb86f430e9f8a1b52bb312d8a96b8f424d128007d7dab5ab42aa6a3306ca={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-actions:1.1.65 +cp.icr.io/cp/wiotp/deprovagent-auth@sha256:d621f386046b9bd942db9aabf8b1b2da0225ff7f4de716ad5207f87654edb4cd={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-auth:2.3.61 +cp.icr.io/cp/wiotp/deprovagent-dm@sha256:22ba1a9fb466c609a08db49adba23715c7874e0caf119683b74a7efbec87fc6f={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-dm:2.1.61 +cp.icr.io/cp/wiotp/deprovagent-fpl@sha256:59963eef65e070034c8642772b804f63d670fe9f14a06d91c60e7c3d220e3073={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-fpl:1.0.14 +cp.icr.io/cp/wiotp/deprovagent-guardian@sha256:e1cea24868cbea17c5415b2fe445cdc80a3ee824c3e100e0f9239b5d66814bff={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-guardian:3.1.61 +cp.icr.io/cp/wiotp/deprovagent-orgmgmt@sha256:28927e185b80ad8ec971f4e1f8746c0a4434590f91bc481f2ce3ed8fddec9c5d={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-orgmgmt:2.1.63 +cp.icr.io/cp/wiotp/deprovagent-provision@sha256:64caf0bf3b94f875b0c92a2a9f9567a4f6f6cdb68f869d84f91700d5043edef9={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-provision:4.4.57 +cp.icr.io/cp/wiotp/deprovagent-registry@sha256:a57a0e64c81a771cf4ef6960489ca1ac7980783c2001b435e74de0129b782690={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-registry:2.2.57 +cp.icr.io/cp/wiotp/deprovagent-state@sha256:37055b0230d4372e47b4981dd8c858adcf387254c81d7be5fa58a6b51e5100e8={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-state:1.2.62 +cp.icr.io/cp/wiotp/deprovagent-webui@sha256:2de7621297e2e8306cb5f5e1475007058194083a33e97d0b9042e81602632323={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-webui:1.2.64 +cp.icr.io/cp/wiotp/device-store@sha256:86879b7fccf9f984406633f7e4e8edceed03e3ff51d2e0705de149b9e06515da={{ registry_public_url_with_path }}/cp/wiotp/device-store:2.2.39 +cp.icr.io/cp/wiotp/devicemgmt-server@sha256:15252f496515254daa8747f3e39026f2a03eefed99062afcd2b5cbfbbda236a3={{ registry_public_url_with_path }}/cp/wiotp/devicemgmt-server:3.2.35 +cp.icr.io/cp/wiotp/entity-connector@sha256:1dc81de46abbad063e6db41badf74758fad0f0ab7236833329e4b1b9f437d3c7={{ registry_public_url_with_path }}/cp/wiotp/entity-connector:1.1.85 +cp.icr.io/cp/wiotp/entity-manager@sha256:7a72775b4c2058387b5298d6ddeabc0add47ccad0b41e0879b4bf6c081f3baa5={{ registry_public_url_with_path }}/cp/wiotp/entity-manager:2.1.93 +cp.icr.io/cp/wiotp/functionsexecutor@sha256:4b5cb0daaff9ff8366a715927a265ee6ca87e4ea49b55e8b5df19a7ad7c70871={{ registry_public_url_with_path }}/cp/wiotp/functionsexecutor:1.1.0 +cp.icr.io/cp/wiotp/graphite-exporter@sha256:afb2d4af1cd0bb8e52de3e1da92b4e853aa490b1e2580f0a434ba8ceeba12fbd={{ registry_public_url_with_path }}/cp/wiotp/graphite-exporter:2.1.79 +cp.icr.io/cp/wiotp/historian-configuration-manager@sha256:374d51451948d28367486f9582c2facb0232e53561608d41b95fa6e61daeaeab={{ registry_public_url_with_path }}/cp/wiotp/historian-configuration-manager:7.1.2 +cp.icr.io/cp/wiotp/historian-connector@sha256:2d57ac856c1dc34096bc294d80e3184c0f0d121598ae58a66aa82e59875bf1fb={{ registry_public_url_with_path }}/cp/wiotp/historian-connector:8.1.5 +cp.icr.io/cp/wiotp/ibm-iot-actions@sha256:d150aaa38ed8f3dc32685b248b4422c1848a2d87a2b168a92b50558cdfeb03e2={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-actions:2.3.0 +cp.icr.io/cp/wiotp/ibm-iot-auth@sha256:1ee7931a46ae4c0a257e9587305d2a427c2a1cf140aebf5866a69a4f58fdc789={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-auth:6.3.1 +cp.icr.io/cp/wiotp/ibm-iot-datapower@sha256:9c7874c1088091fc1d88a3882785ad80b9d54b639a988d7be32b8b16b5120131={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-datapower:7.2.1 +cp.icr.io/cp/wiotp/ibm-iot-devops@sha256:253b6274c2d87bb0ba12df129000dd34e8081a37b04bd653ab66ef6287576086={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-devops:10.2.1 +cp.icr.io/cp/wiotp/ibm-iot-dm@sha256:46a83b53e8ec51d1db6c6c86f877e72f31af4ba06b002bf9fb1165bd77658678={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-dm:2.2.1 +cp.icr.io/cp/wiotp/ibm-iot-dsc@sha256:72e8c6b0ce0f772a60c415b095494a0f744de638288bcf1a3aebd2b28e049bdc={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-dsc:6.2.1 +cp.icr.io/cp/wiotp/ibm-iot-fpl@sha256:b21d37ff6414e0ac423c1bf95e38b747b345eebe5152901fa09ad6954788a02e={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-fpl:1.1.0 +cp.icr.io/cp/wiotp/ibm-iot-guardian@sha256:3af0f8a7ce6011e27988d2ec779d567e729bd6bdc584112d78cfea1f546d8a69={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-guardian:4.2.1 +cp.icr.io/cp/wiotp/ibm-iot-mbgx@sha256:5adf9cefbfdee1cc3a30598e6c55794ada9d577a975161891303eb1e2826dd67={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-mbgx:3.3.1 +cp.icr.io/cp/wiotp/ibm-iot-mfgx@sha256:f1c5a8bb66c2fd879763c8981dd5a9ab0d2de34426eb59310816049f3cd5f92f={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-mfgx:6.3.1 +cp.icr.io/cp/wiotp/ibm-iot-monitor@sha256:5241962c295cfe71a6d4f4c90306fdc765a7486f81ec4674caf64c3d476da194={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-monitor:2.3.0 +cp.icr.io/cp/wiotp/ibm-iot-orgmgmt@sha256:4f575f552e11599d35c941c3d53d968201ea9019eec7e601244bb891cc2b5cdd={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-orgmgmt:6.3.0 +cp.icr.io/cp/wiotp/ibm-iot-provision@sha256:06f28fa79b0bd9184c1b0c5c43f0e07e1cdd70d06d503b20e7dc1d43b8ce71a7={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-provision:10.2.1 +cp.icr.io/cp/wiotp/ibm-iot-registry@sha256:730fc97a9e72eebac635aea847e567c5e1f89708c585221e1453be062c97c0ac={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-registry:8.2.1 +cp.icr.io/cp/wiotp/ibm-iot-state@sha256:fe9ded36f0bcaeeacb9d270760baaffb7a7081fd54717cc30b7473f5154226ce={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-state:5.2.1 +cp.icr.io/cp/wiotp/ibm-iot-webui@sha256:b1c3e07ab5bd302c24f93753ae1c1395a14da9879dede6eaecd1e3a327090270={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-webui:6.2.1 +cp.icr.io/cp/wiotp/ibm-iot-workspace@sha256:c8ae641bbe8b46352dce21495a600a407d7bdaa88dda23c1ca87b89d5368293e={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-workspace:2.5.1 +cp.icr.io/cp/wiotp/kafkamodel-actions@sha256:da423d4fbb7d8dbada9b1ddb6d7d55b63f6c95e97e58e61ff69aaa1dc6d02b5b={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-actions:1.4.6 +cp.icr.io/cp/wiotp/kafkamodel-auth@sha256:ad1e3090cbec3dc68dbed33baf4466bad053c14a3439db6766a497aced8e572e={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-auth:1.7.6 +cp.icr.io/cp/wiotp/kafkamodel-fpl@sha256:29ac84c25d04af5828a2af92420904ebbeb594e67ee093c80f4801bdcbf2b0d8={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-fpl:1.1.6 +cp.icr.io/cp/wiotp/kafkamodel-mfgx@sha256:f950837e92d28ac2904b2ce39d30817e2ae4348a003d0b83898b87c412813a2c={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-mfgx:1.3.6 +cp.icr.io/cp/wiotp/kafkamodel-orgmgmt@sha256:d8f7be2c19042439dea0ecc487c2d8a6e94860af0642c52af0a651c9f48bc1d5={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-orgmgmt:2.2.6 +cp.icr.io/cp/wiotp/kafkamodel-registry@sha256:de865fff88e4d25811287de01d3e998d7f246340f97004e7535fed11a8ee6eb8={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-registry:1.4.6 +cp.icr.io/cp/wiotp/kafkamodel-state@sha256:0a8c7645ccf1ccecff108d1fbc993161e099e72becad085dcb0e7b837cfcabfb={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-state:1.4.6 +cp.icr.io/cp/wiotp/masuseragent@sha256:06a65a089e71b9cb07bf64da7bdd75faa265970de868309e2da460cdc5aed1d3={{ registry_public_url_with_path }}/cp/wiotp/masuseragent:1.4.8 +cp.icr.io/cp/wiotp/messagesight-configurator@sha256:e90f01c84e4a84fefafdcf02731b8f79c08d23d3501c3cdccb1f532514c346bf={{ registry_public_url_with_path }}/cp/wiotp/messagesight-configurator:5.14.35 +cp.icr.io/cp/wiotp/messagesight@sha256:24597be9d139ba12c4573fff74a41f1323d77be9443469d8633fbd4bde8e6744={{ registry_public_url_with_path }}/cp/wiotp/messagesight:5.14.35 +cp.icr.io/cp/wiotp/monagent-iot@sha256:da292da546da4acdc27f9865b0040384dcb79ca16eb9455ea907ffdcd5c37b0c={{ registry_public_url_with_path }}/cp/wiotp/monagent-iot:1.1.0 +cp.icr.io/cp/wiotp/monagent-msserver@sha256:9067b5c576333dcd272883cb20ccd45950cdd5f30e2d566c8af7501e2079b1be={{ registry_public_url_with_path }}/cp/wiotp/monagent-msserver:4.3.70 +cp.icr.io/cp/wiotp/monagent-org@sha256:0ac2638d4d4c590fa7a3566aa6b8a902de54e8ecbd93c61c851a0cbe461cedf3={{ registry_public_url_with_path }}/cp/wiotp/monagent-org:2.1.83 +cp.icr.io/cp/wiotp/mongomodel-actions@sha256:b30714e01aedcf2654f718308a29591157937290301cb2c157ea1536975bdd79={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-actions:1.2.76 +cp.icr.io/cp/wiotp/mongomodel-dm@sha256:e9eb535d9b156c1e6f7c6fc3a3ddf6de346fadc0e30aef6a7f32354cba4bc9df={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-dm:1.3.80 +cp.icr.io/cp/wiotp/mongomodel-dsc@sha256:f58651c00dd8c5f1f87ca323c521fe02dbd1787449a79f860e4a635c5c3cf667={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-dsc:1.2.78 +cp.icr.io/cp/wiotp/mongomodel-guardian@sha256:db8fbf2d5a1cc7b453dc262954e9de782d0d074bacc67b8bdf90c18154813349={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-guardian:1.3.75 +cp.icr.io/cp/wiotp/mongomodel-orgmgmt@sha256:c277dfa1c54e33aab55b195745bd74c967bb7febdf92f20d6f612eb36eb1372a={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-orgmgmt:2.0.75 +cp.icr.io/cp/wiotp/mongomodel-provision@sha256:76f16025df17d94ea32047624123b967186c95d74514e344f4dd892efd7f84fc={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-provision:2.5.65 +cp.icr.io/cp/wiotp/mongomodel-registry@sha256:bb49e23af6c150de2bcfdced8c617c7354814694c8137caacfabbb7bf78d7032={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-registry:2.0.71 +cp.icr.io/cp/wiotp/mongomodel-state@sha256:c3df1e9b8d2e895b92088a733745e683d1dd16d6b77b4a768c3df21cfcb30c9f={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-state:1.2.72 +cp.icr.io/cp/wiotp/mongomodel-webui@sha256:9cab9ba25341778564f5b98567a429b9e12f6cc4da55d92c935f7561c2a0ab28={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-webui:2.3.79 +cp.icr.io/cp/wiotp/mqtt-connector@sha256:b51922a2283c4581b910878e7d542b4ab2de6e5d670305657eedd7f0a15239b4={{ registry_public_url_with_path }}/cp/wiotp/mqtt-connector:2.0.73 +cp.icr.io/cp/wiotp/msproxy-monitor@sha256:b53dfac38f002814c2a33c1c13778b55954a095a2132308543624dd61d27b05f={{ registry_public_url_with_path }}/cp/wiotp/msproxy-monitor:8.5.42 +cp.icr.io/cp/wiotp/msproxy@sha256:417a3d8f94a19ededfe6b53d4ecd3d8622abd7bbbf990d3ac3ae303b92dbc1f8={{ registry_public_url_with_path }}/cp/wiotp/msproxy:8.5.42 +cp.icr.io/cp/wiotp/org-store@sha256:72796a6a6c92299f68b3d5581b7325db9d3600cadd22e67133c43eb275295873={{ registry_public_url_with_path }}/cp/wiotp/org-store:4.0.46 +cp.icr.io/cp/wiotp/orgdeprovmgr@sha256:78bc78197b119c0f92fa299d9a9473d4dc221d1b1c851f304c394e960a0c6baa={{ registry_public_url_with_path }}/cp/wiotp/orgdeprovmgr:2.2.66 +cp.icr.io/cp/wiotp/orgpoolmgr@sha256:f4424d91a381116815b5a2448c7a606bb2a0ff6c45f74ab92bf5d31f7153bb0c={{ registry_public_url_with_path }}/cp/wiotp/orgpoolmgr:3.1.78 +cp.icr.io/cp/wiotp/orgprovmgr@sha256:f71ede6293c82e2c85e89e0d331f3674541effcabfc4aeb851ae9f6551ec4611={{ registry_public_url_with_path }}/cp/wiotp/orgprovmgr:2.1.64 +cp.icr.io/cp/wiotp/pipelinerouter@sha256:1c0047681cf5a8eb6c867c3581d613a64a03c326e23c7a9b292cd55573ca2171={{ registry_public_url_with_path }}/cp/wiotp/pipelinerouter:1.1.0 +cp.icr.io/cp/wiotp/provagent-guardian@sha256:37b9d3a251c2597c0eb4e5a744b9fb871a0436691649d434e42ad8877a7b2d05={{ registry_public_url_with_path }}/cp/wiotp/provagent-guardian:2.1.65 +cp.icr.io/cp/wiotp/s2s-store@sha256:4bbf05141df77240f76ca90b183b3200822b60d3d13df05e5be39c232a8b1c6a={{ registry_public_url_with_path }}/cp/wiotp/s2s-store:6.1.31 +cp.icr.io/cp/wiotp/sqlmodel-auth@sha256:6ec3d1dc0ab7b04be6b3d948606e030f2ceaff91271650b37fee7b87b60643f1={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-auth:1.1.17 +cp.icr.io/cp/wiotp/sqlmodel-fpl@sha256:c65076e777fa55fb94625cc228cea0e45b2e7397e1ed4aa1e9d82ea0d823e215={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-fpl:1.0.15 +cp.icr.io/cp/wiotp/sqlmodel-state@sha256:9a1551a240c5b925f272e0c43cfa949b099d12136d98dfb9c213608e0446a27b={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-state:1.0.62 +cp.icr.io/cp/wiotp/state-updater@sha256:0fdffec079afe9277c6ce656b270cc1ef1b2aaca80e0b425ddc4c1c596d72aab={{ registry_public_url_with_path }}/cp/wiotp/state-updater:2.0.35 +cp.icr.io/cp/wiotp/statsd@sha256:5cb1c6005fa136d1c1cb0c0b04ae4cfb5a6588d98d74bf046784725d1a50aa3d={{ registry_public_url_with_path }}/cp/wiotp/statsd:1.3.70 +cp.icr.io/cp/wiotp/ui-dashboard@sha256:d0ccf2e8175c83a77aa8136ceebe46188800eb82db6a745ef455e2812b068916={{ registry_public_url_with_path }}/cp/wiotp/ui-dashboard:7.1.67 +icr.io/cpopen/ibm-mas-iot-operator-bundle@sha256:e576f5731f052b99d95bba2f1491c37349259de1696ae46124ff16bdc253bb83={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot-operator-bundle:8.6.0 +icr.io/cpopen/ibm-mas-iot-operator-catalog@sha256:09b6560684c9f651a9c837b080498e46162a334fb02ff5bb7a5b8faf75734e70={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot-operator-catalog:latest +icr.io/cpopen/ibm-mas-iot@sha256:870d956908cec2bbc7d6fd5ffa9863fe02d643c183910e7e6fd211d391027564={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot:8.6.0 \ No newline at end of file diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/iot860fix/from-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/iot860fix/from-filesystem.txt.j2 index 4eac320d7c..7403562def 100644 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/iot860fix/from-filesystem.txt.j2 +++ b/ibm/mas_devops/roles/mirror_case_prepare/templates/iot860fix/from-filesystem.txt.j2 @@ -1,103 +1,103 @@ -file:///cp/mas/ltpakeys-generator@sha256:0fabd61f1cc6313e57b74ed393fff1eb7445001c7421efde93c0a755ec70133d={{ registry_public_url }}/cp/mas/ltpakeys-generator:2.3.35 -file:///cp/wiotp/action-mgr@sha256:989c76823d3ef446ef5e367d0fdfa44a9cf73f5658051553b43f04a14a1093f6={{ registry_public_url }}/cp/wiotp/action-mgr:2.0.53 -file:///cp/wiotp/api-actions@sha256:4dba1876b2aeaffb3fb1d24091c62ecebc0954cbdf624e55fb9ed8549f4bf02f={{ registry_public_url }}/cp/wiotp/api-actions:2.1.47 -file:///cp/wiotp/api-authentication@sha256:158351ddf133c2bb982df9afe947c2fd842dc9c89e7d17d3f2f598a1d5bb7a5f={{ registry_public_url }}/cp/wiotp/api-authentication:3.3.49 -file:///cp/wiotp/api-authorization@sha256:04762e32d7b1d1705c24c56bf567f3e1cbd6a72104dd20fe4758c96a17d9aa9b={{ registry_public_url }}/cp/wiotp/api-authorization:5.3.46 -file:///cp/wiotp/api-connectors@sha256:62942ed423bcb793ba872b133b810c3211c626dcce820bc8232601bfc0a3c7b0={{ registry_public_url }}/cp/wiotp/api-connectors:4.1.3 -file:///cp/wiotp/api-dashboard@sha256:dd98151e83eada6c97fd0c51a4cfcf561c7af383fe653170f777036ab7278863={{ registry_public_url }}/cp/wiotp/api-dashboard:3.0.82 -file:///cp/wiotp/api-devicemgmt@sha256:ed3bb00f6e823052faeb643cbb2a6e02a3772981a9146aae64f830a81755823f={{ registry_public_url }}/cp/wiotp/api-devicemgmt:3.2.37 -file:///cp/wiotp/api-devops@sha256:fae4ba84a2b7af5b036906bd6d430e4b086e672bb1d2fca3073cb9001007b174={{ registry_public_url }}/cp/wiotp/api-devops:3.7.49 -file:///cp/wiotp/api-importconnectors@sha256:73b0c2ce525ed53ed4426487c25f0f1579fd51a09104b67eb15f489724d3c9c2={{ registry_public_url }}/cp/wiotp/api-importconnectors:2.0.30 -file:///cp/wiotp/api-mbgadmin@sha256:f60f533237cffcb760fb29c93e2faebddefa97be78a03d7997f4d177ece14fd2={{ registry_public_url }}/cp/wiotp/api-mbgadmin:2.4.74 -file:///cp/wiotp/api-messagesight@sha256:7f37bfd840122de1f3aba6d84adf426a0cf26a89b367ad056ee06d79f9ddffe6={{ registry_public_url }}/cp/wiotp/api-messagesight:3.1.41 -file:///cp/wiotp/api-org@sha256:307f7023dc004eb6c5fd579e17f1bc47cbcf9fc454c4b819e56d6d8463553544={{ registry_public_url }}/cp/wiotp/api-org:8.2.29 -file:///cp/wiotp/api-organizations@sha256:e4cb0a5f4acb347ff27696a82b96e46ff3044d92904716597bbc37f393ffbbe3={{ registry_public_url }}/cp/wiotp/api-organizations:10.0.39 -file:///cp/wiotp/api-pipeline@sha256:4e926b536b7713aa8a2ee166aaeebaac5d9e74a78fbf03abeeac484387054917={{ registry_public_url }}/cp/wiotp/api-pipeline:1.1.0 -file:///cp/wiotp/api-resourcecontroller@sha256:5b8b4ef64ff65e727e2c7a7134785a673ba2cdb134655800d84753107c90a8ce={{ registry_public_url }}/cp/wiotp/api-resourcecontroller:3.0.72 -file:///cp/wiotp/api-riskmgmt-secguardian@sha256:ef80d0b8fbdd629b3d47ffd9fc00b0cd63e69e5b51133a9816e09117646463bd={{ registry_public_url }}/cp/wiotp/api-riskmgmt-secguardian:5.1.47 -file:///cp/wiotp/api-s2s@sha256:476966d25e6086e36b4212cebfe8a8e6146ad80021a64eeb3dad0ea55a1850cd={{ registry_public_url }}/cp/wiotp/api-s2s:10.1.29 -file:///cp/wiotp/api-schemas@sha256:2885df674610b01e98daff4a676c03b178fbf49e5696fd872148a3f4d16602a5={{ registry_public_url }}/cp/wiotp/api-schemas:4.0.31 -file:///cp/wiotp/api-state@sha256:10b9f206b48235dfc5187496fada9077fb7d6e0082d760b4f1299d5afe4f9447={{ registry_public_url }}/cp/wiotp/api-state:5.0.32 -file:///cp/wiotp/api-status@sha256:4164d2d2b5cf0f6314b4fdfa11e780003f4632d9a2545f7fb041990fb45bcad2={{ registry_public_url }}/cp/wiotp/api-status:3.4.8 -file:///cp/wiotp/auth-store@sha256:1725ed33844394a808d6b154e4eff2016d0a21dac73fb397aa2c772dc533cc3f={{ registry_public_url }}/cp/wiotp/auth-store:2.4.51 -file:///cp/wiotp/config-store@sha256:c14d7489d9ca23441076e57758c673422e7abb00785570ed8159c4caece9cd35={{ registry_public_url }}/cp/wiotp/config-store:4.0.46 -file:///cp/wiotp/connectors-store@sha256:b6f4a3078bef6ec614db4d8114592e7768b46e294ce215ec34dee4bd62f34a9b={{ registry_public_url }}/cp/wiotp/connectors-store:8.0.34 -file:///cp/wiotp/data-import-connector@sha256:fa0ce13bb49df74d3fb4aeaa3211f6dcdb9d311e1cb3f3b979bc33dbc04b1e61={{ registry_public_url }}/cp/wiotp/data-import-connector:2.0.66 -file:///cp/wiotp/datapower-config@sha256:547f74ac50f1724c037ca8d2a15f34263cf21309d3eb3d6e37e85c1a750d5126={{ registry_public_url }}/cp/wiotp/datapower-config:10.3.1 -file:///cp/wiotp/datapower@sha256:d4122be5bb21f1333032a446849124ae1d769195576ad0cc4d5ea1d76d7c460c={{ registry_public_url }}/cp/wiotp/datapower:7.2.1 -file:///cp/wiotp/deprovagent-actions@sha256:a9a7cb86f430e9f8a1b52bb312d8a96b8f424d128007d7dab5ab42aa6a3306ca={{ registry_public_url }}/cp/wiotp/deprovagent-actions:1.1.65 -file:///cp/wiotp/deprovagent-auth@sha256:d621f386046b9bd942db9aabf8b1b2da0225ff7f4de716ad5207f87654edb4cd={{ registry_public_url }}/cp/wiotp/deprovagent-auth:2.3.61 -file:///cp/wiotp/deprovagent-dm@sha256:22ba1a9fb466c609a08db49adba23715c7874e0caf119683b74a7efbec87fc6f={{ registry_public_url }}/cp/wiotp/deprovagent-dm:2.1.61 -file:///cp/wiotp/deprovagent-fpl@sha256:59963eef65e070034c8642772b804f63d670fe9f14a06d91c60e7c3d220e3073={{ registry_public_url }}/cp/wiotp/deprovagent-fpl:1.0.14 -file:///cp/wiotp/deprovagent-guardian@sha256:e1cea24868cbea17c5415b2fe445cdc80a3ee824c3e100e0f9239b5d66814bff={{ registry_public_url }}/cp/wiotp/deprovagent-guardian:3.1.61 -file:///cp/wiotp/deprovagent-orgmgmt@sha256:28927e185b80ad8ec971f4e1f8746c0a4434590f91bc481f2ce3ed8fddec9c5d={{ registry_public_url }}/cp/wiotp/deprovagent-orgmgmt:2.1.63 -file:///cp/wiotp/deprovagent-provision@sha256:64caf0bf3b94f875b0c92a2a9f9567a4f6f6cdb68f869d84f91700d5043edef9={{ registry_public_url }}/cp/wiotp/deprovagent-provision:4.4.57 -file:///cp/wiotp/deprovagent-registry@sha256:a57a0e64c81a771cf4ef6960489ca1ac7980783c2001b435e74de0129b782690={{ registry_public_url }}/cp/wiotp/deprovagent-registry:2.2.57 -file:///cp/wiotp/deprovagent-state@sha256:37055b0230d4372e47b4981dd8c858adcf387254c81d7be5fa58a6b51e5100e8={{ registry_public_url }}/cp/wiotp/deprovagent-state:1.2.62 -file:///cp/wiotp/deprovagent-webui@sha256:2de7621297e2e8306cb5f5e1475007058194083a33e97d0b9042e81602632323={{ registry_public_url }}/cp/wiotp/deprovagent-webui:1.2.64 -file:///cp/wiotp/device-store@sha256:86879b7fccf9f984406633f7e4e8edceed03e3ff51d2e0705de149b9e06515da={{ registry_public_url }}/cp/wiotp/device-store:2.2.39 -file:///cp/wiotp/devicemgmt-server@sha256:15252f496515254daa8747f3e39026f2a03eefed99062afcd2b5cbfbbda236a3={{ registry_public_url }}/cp/wiotp/devicemgmt-server:3.2.35 -file:///cp/wiotp/entity-connector@sha256:1dc81de46abbad063e6db41badf74758fad0f0ab7236833329e4b1b9f437d3c7={{ registry_public_url }}/cp/wiotp/entity-connector:1.1.85 -file:///cp/wiotp/entity-manager@sha256:7a72775b4c2058387b5298d6ddeabc0add47ccad0b41e0879b4bf6c081f3baa5={{ registry_public_url }}/cp/wiotp/entity-manager:2.1.93 -file:///cp/wiotp/functionsexecutor@sha256:4b5cb0daaff9ff8366a715927a265ee6ca87e4ea49b55e8b5df19a7ad7c70871={{ registry_public_url }}/cp/wiotp/functionsexecutor:1.1.0 -file:///cp/wiotp/graphite-exporter@sha256:afb2d4af1cd0bb8e52de3e1da92b4e853aa490b1e2580f0a434ba8ceeba12fbd={{ registry_public_url }}/cp/wiotp/graphite-exporter:2.1.79 -file:///cp/wiotp/historian-configuration-manager@sha256:374d51451948d28367486f9582c2facb0232e53561608d41b95fa6e61daeaeab={{ registry_public_url }}/cp/wiotp/historian-configuration-manager:7.1.2 -file:///cp/wiotp/historian-connector@sha256:2d57ac856c1dc34096bc294d80e3184c0f0d121598ae58a66aa82e59875bf1fb={{ registry_public_url }}/cp/wiotp/historian-connector:8.1.5 -file:///cp/wiotp/ibm-iot-actions@sha256:d150aaa38ed8f3dc32685b248b4422c1848a2d87a2b168a92b50558cdfeb03e2={{ registry_public_url }}/cp/wiotp/ibm-iot-actions:2.3.0 -file:///cp/wiotp/ibm-iot-auth@sha256:1ee7931a46ae4c0a257e9587305d2a427c2a1cf140aebf5866a69a4f58fdc789={{ registry_public_url }}/cp/wiotp/ibm-iot-auth:6.3.1 -file:///cp/wiotp/ibm-iot-datapower@sha256:9c7874c1088091fc1d88a3882785ad80b9d54b639a988d7be32b8b16b5120131={{ registry_public_url }}/cp/wiotp/ibm-iot-datapower:7.2.1 -file:///cp/wiotp/ibm-iot-devops@sha256:253b6274c2d87bb0ba12df129000dd34e8081a37b04bd653ab66ef6287576086={{ registry_public_url }}/cp/wiotp/ibm-iot-devops:10.2.1 -file:///cp/wiotp/ibm-iot-dm@sha256:46a83b53e8ec51d1db6c6c86f877e72f31af4ba06b002bf9fb1165bd77658678={{ registry_public_url }}/cp/wiotp/ibm-iot-dm:2.2.1 -file:///cp/wiotp/ibm-iot-dsc@sha256:72e8c6b0ce0f772a60c415b095494a0f744de638288bcf1a3aebd2b28e049bdc={{ registry_public_url }}/cp/wiotp/ibm-iot-dsc:6.2.1 -file:///cp/wiotp/ibm-iot-fpl@sha256:b21d37ff6414e0ac423c1bf95e38b747b345eebe5152901fa09ad6954788a02e={{ registry_public_url }}/cp/wiotp/ibm-iot-fpl:1.1.0 -file:///cp/wiotp/ibm-iot-guardian@sha256:3af0f8a7ce6011e27988d2ec779d567e729bd6bdc584112d78cfea1f546d8a69={{ registry_public_url }}/cp/wiotp/ibm-iot-guardian:4.2.1 -file:///cp/wiotp/ibm-iot-mbgx@sha256:5adf9cefbfdee1cc3a30598e6c55794ada9d577a975161891303eb1e2826dd67={{ registry_public_url }}/cp/wiotp/ibm-iot-mbgx:3.3.1 -file:///cp/wiotp/ibm-iot-mfgx@sha256:f1c5a8bb66c2fd879763c8981dd5a9ab0d2de34426eb59310816049f3cd5f92f={{ registry_public_url }}/cp/wiotp/ibm-iot-mfgx:6.3.1 -file:///cp/wiotp/ibm-iot-monitor@sha256:5241962c295cfe71a6d4f4c90306fdc765a7486f81ec4674caf64c3d476da194={{ registry_public_url }}/cp/wiotp/ibm-iot-monitor:2.3.0 -file:///cp/wiotp/ibm-iot-orgmgmt@sha256:4f575f552e11599d35c941c3d53d968201ea9019eec7e601244bb891cc2b5cdd={{ registry_public_url }}/cp/wiotp/ibm-iot-orgmgmt:6.3.0 -file:///cp/wiotp/ibm-iot-provision@sha256:06f28fa79b0bd9184c1b0c5c43f0e07e1cdd70d06d503b20e7dc1d43b8ce71a7={{ registry_public_url }}/cp/wiotp/ibm-iot-provision:10.2.1 -file:///cp/wiotp/ibm-iot-registry@sha256:730fc97a9e72eebac635aea847e567c5e1f89708c585221e1453be062c97c0ac={{ registry_public_url }}/cp/wiotp/ibm-iot-registry:8.2.1 -file:///cp/wiotp/ibm-iot-state@sha256:fe9ded36f0bcaeeacb9d270760baaffb7a7081fd54717cc30b7473f5154226ce={{ registry_public_url }}/cp/wiotp/ibm-iot-state:5.2.1 -file:///cp/wiotp/ibm-iot-webui@sha256:b1c3e07ab5bd302c24f93753ae1c1395a14da9879dede6eaecd1e3a327090270={{ registry_public_url }}/cp/wiotp/ibm-iot-webui:6.2.1 -file:///cp/wiotp/ibm-iot-workspace@sha256:c8ae641bbe8b46352dce21495a600a407d7bdaa88dda23c1ca87b89d5368293e={{ registry_public_url }}/cp/wiotp/ibm-iot-workspace:2.5.1 -file:///cp/wiotp/kafkamodel-actions@sha256:da423d4fbb7d8dbada9b1ddb6d7d55b63f6c95e97e58e61ff69aaa1dc6d02b5b={{ registry_public_url }}/cp/wiotp/kafkamodel-actions:1.4.6 -file:///cp/wiotp/kafkamodel-auth@sha256:ad1e3090cbec3dc68dbed33baf4466bad053c14a3439db6766a497aced8e572e={{ registry_public_url }}/cp/wiotp/kafkamodel-auth:1.7.6 -file:///cp/wiotp/kafkamodel-fpl@sha256:29ac84c25d04af5828a2af92420904ebbeb594e67ee093c80f4801bdcbf2b0d8={{ registry_public_url }}/cp/wiotp/kafkamodel-fpl:1.1.6 -file:///cp/wiotp/kafkamodel-mfgx@sha256:f950837e92d28ac2904b2ce39d30817e2ae4348a003d0b83898b87c412813a2c={{ registry_public_url }}/cp/wiotp/kafkamodel-mfgx:1.3.6 -file:///cp/wiotp/kafkamodel-orgmgmt@sha256:d8f7be2c19042439dea0ecc487c2d8a6e94860af0642c52af0a651c9f48bc1d5={{ registry_public_url }}/cp/wiotp/kafkamodel-orgmgmt:2.2.6 -file:///cp/wiotp/kafkamodel-registry@sha256:de865fff88e4d25811287de01d3e998d7f246340f97004e7535fed11a8ee6eb8={{ registry_public_url }}/cp/wiotp/kafkamodel-registry:1.4.6 -file:///cp/wiotp/kafkamodel-state@sha256:0a8c7645ccf1ccecff108d1fbc993161e099e72becad085dcb0e7b837cfcabfb={{ registry_public_url }}/cp/wiotp/kafkamodel-state:1.4.6 -file:///cp/wiotp/masuseragent@sha256:06a65a089e71b9cb07bf64da7bdd75faa265970de868309e2da460cdc5aed1d3={{ registry_public_url }}/cp/wiotp/masuseragent:1.4.8 -file:///cp/wiotp/messagesight-configurator@sha256:e90f01c84e4a84fefafdcf02731b8f79c08d23d3501c3cdccb1f532514c346bf={{ registry_public_url }}/cp/wiotp/messagesight-configurator:5.14.35 -file:///cp/wiotp/messagesight@sha256:24597be9d139ba12c4573fff74a41f1323d77be9443469d8633fbd4bde8e6744={{ registry_public_url }}/cp/wiotp/messagesight:5.14.35 -file:///cp/wiotp/monagent-iot@sha256:da292da546da4acdc27f9865b0040384dcb79ca16eb9455ea907ffdcd5c37b0c={{ registry_public_url }}/cp/wiotp/monagent-iot:1.1.0 -file:///cp/wiotp/monagent-msserver@sha256:9067b5c576333dcd272883cb20ccd45950cdd5f30e2d566c8af7501e2079b1be={{ registry_public_url }}/cp/wiotp/monagent-msserver:4.3.70 -file:///cp/wiotp/monagent-org@sha256:0ac2638d4d4c590fa7a3566aa6b8a902de54e8ecbd93c61c851a0cbe461cedf3={{ registry_public_url }}/cp/wiotp/monagent-org:2.1.83 -file:///cp/wiotp/mongomodel-actions@sha256:b30714e01aedcf2654f718308a29591157937290301cb2c157ea1536975bdd79={{ registry_public_url }}/cp/wiotp/mongomodel-actions:1.2.76 -file:///cp/wiotp/mongomodel-dm@sha256:e9eb535d9b156c1e6f7c6fc3a3ddf6de346fadc0e30aef6a7f32354cba4bc9df={{ registry_public_url }}/cp/wiotp/mongomodel-dm:1.3.80 -file:///cp/wiotp/mongomodel-dsc@sha256:f58651c00dd8c5f1f87ca323c521fe02dbd1787449a79f860e4a635c5c3cf667={{ registry_public_url }}/cp/wiotp/mongomodel-dsc:1.2.78 -file:///cp/wiotp/mongomodel-guardian@sha256:db8fbf2d5a1cc7b453dc262954e9de782d0d074bacc67b8bdf90c18154813349={{ registry_public_url }}/cp/wiotp/mongomodel-guardian:1.3.75 -file:///cp/wiotp/mongomodel-orgmgmt@sha256:c277dfa1c54e33aab55b195745bd74c967bb7febdf92f20d6f612eb36eb1372a={{ registry_public_url }}/cp/wiotp/mongomodel-orgmgmt:2.0.75 -file:///cp/wiotp/mongomodel-provision@sha256:76f16025df17d94ea32047624123b967186c95d74514e344f4dd892efd7f84fc={{ registry_public_url }}/cp/wiotp/mongomodel-provision:2.5.65 -file:///cp/wiotp/mongomodel-registry@sha256:bb49e23af6c150de2bcfdced8c617c7354814694c8137caacfabbb7bf78d7032={{ registry_public_url }}/cp/wiotp/mongomodel-registry:2.0.71 -file:///cp/wiotp/mongomodel-state@sha256:c3df1e9b8d2e895b92088a733745e683d1dd16d6b77b4a768c3df21cfcb30c9f={{ registry_public_url }}/cp/wiotp/mongomodel-state:1.2.72 -file:///cp/wiotp/mongomodel-webui@sha256:9cab9ba25341778564f5b98567a429b9e12f6cc4da55d92c935f7561c2a0ab28={{ registry_public_url }}/cp/wiotp/mongomodel-webui:2.3.79 -file:///cp/wiotp/mqtt-connector@sha256:b51922a2283c4581b910878e7d542b4ab2de6e5d670305657eedd7f0a15239b4={{ registry_public_url }}/cp/wiotp/mqtt-connector:2.0.73 -file:///cp/wiotp/msproxy-monitor@sha256:b53dfac38f002814c2a33c1c13778b55954a095a2132308543624dd61d27b05f={{ registry_public_url }}/cp/wiotp/msproxy-monitor:8.5.42 -file:///cp/wiotp/msproxy@sha256:417a3d8f94a19ededfe6b53d4ecd3d8622abd7bbbf990d3ac3ae303b92dbc1f8={{ registry_public_url }}/cp/wiotp/msproxy:8.5.42 -file:///cp/wiotp/org-store@sha256:72796a6a6c92299f68b3d5581b7325db9d3600cadd22e67133c43eb275295873={{ registry_public_url }}/cp/wiotp/org-store:4.0.46 -file:///cp/wiotp/orgdeprovmgr@sha256:78bc78197b119c0f92fa299d9a9473d4dc221d1b1c851f304c394e960a0c6baa={{ registry_public_url }}/cp/wiotp/orgdeprovmgr:2.2.66 -file:///cp/wiotp/orgpoolmgr@sha256:f4424d91a381116815b5a2448c7a606bb2a0ff6c45f74ab92bf5d31f7153bb0c={{ registry_public_url }}/cp/wiotp/orgpoolmgr:3.1.78 -file:///cp/wiotp/orgprovmgr@sha256:f71ede6293c82e2c85e89e0d331f3674541effcabfc4aeb851ae9f6551ec4611={{ registry_public_url }}/cp/wiotp/orgprovmgr:2.1.64 -file:///cp/wiotp/pipelinerouter@sha256:1c0047681cf5a8eb6c867c3581d613a64a03c326e23c7a9b292cd55573ca2171={{ registry_public_url }}/cp/wiotp/pipelinerouter:1.1.0 -file:///cp/wiotp/provagent-guardian@sha256:37b9d3a251c2597c0eb4e5a744b9fb871a0436691649d434e42ad8877a7b2d05={{ registry_public_url }}/cp/wiotp/provagent-guardian:2.1.65 -file:///cp/wiotp/s2s-store@sha256:4bbf05141df77240f76ca90b183b3200822b60d3d13df05e5be39c232a8b1c6a={{ registry_public_url }}/cp/wiotp/s2s-store:6.1.31 -file:///cp/wiotp/sqlmodel-auth@sha256:6ec3d1dc0ab7b04be6b3d948606e030f2ceaff91271650b37fee7b87b60643f1={{ registry_public_url }}/cp/wiotp/sqlmodel-auth:1.1.17 -file:///cp/wiotp/sqlmodel-fpl@sha256:c65076e777fa55fb94625cc228cea0e45b2e7397e1ed4aa1e9d82ea0d823e215={{ registry_public_url }}/cp/wiotp/sqlmodel-fpl:1.0.15 -file:///cp/wiotp/sqlmodel-state@sha256:9a1551a240c5b925f272e0c43cfa949b099d12136d98dfb9c213608e0446a27b={{ registry_public_url }}/cp/wiotp/sqlmodel-state:1.0.62 -file:///cp/wiotp/state-updater@sha256:0fdffec079afe9277c6ce656b270cc1ef1b2aaca80e0b425ddc4c1c596d72aab={{ registry_public_url }}/cp/wiotp/state-updater:2.0.35 -file:///cp/wiotp/statsd@sha256:5cb1c6005fa136d1c1cb0c0b04ae4cfb5a6588d98d74bf046784725d1a50aa3d={{ registry_public_url }}/cp/wiotp/statsd:1.3.70 -file:///cp/wiotp/ui-dashboard@sha256:d0ccf2e8175c83a77aa8136ceebe46188800eb82db6a745ef455e2812b068916={{ registry_public_url }}/cp/wiotp/ui-dashboard:7.1.67 -file:///cpopen/ibm-mas-iot-operator-bundle@sha256:e576f5731f052b99d95bba2f1491c37349259de1696ae46124ff16bdc253bb83={{ registry_public_url }}/cpopen/ibm-mas-iot-operator-bundle:8.6.0 -file:///cpopen/ibm-mas-iot-operator-catalog@sha256:09b6560684c9f651a9c837b080498e46162a334fb02ff5bb7a5b8faf75734e70={{ registry_public_url }}/cpopen/ibm-mas-iot-operator-catalog:latest -file:///cpopen/ibm-mas-iot@sha256:870d956908cec2bbc7d6fd5ffa9863fe02d643c183910e7e6fd211d391027564={{ registry_public_url }}/cpopen/ibm-mas-iot:8.6.0 \ No newline at end of file +file:///cp/mas/ltpakeys-generator@sha256:0fabd61f1cc6313e57b74ed393fff1eb7445001c7421efde93c0a755ec70133d={{ registry_public_url_with_path }}/cp/mas/ltpakeys-generator:2.3.35 +file:///cp/wiotp/action-mgr@sha256:989c76823d3ef446ef5e367d0fdfa44a9cf73f5658051553b43f04a14a1093f6={{ registry_public_url_with_path }}/cp/wiotp/action-mgr:2.0.53 +file:///cp/wiotp/api-actions@sha256:4dba1876b2aeaffb3fb1d24091c62ecebc0954cbdf624e55fb9ed8549f4bf02f={{ registry_public_url_with_path }}/cp/wiotp/api-actions:2.1.47 +file:///cp/wiotp/api-authentication@sha256:158351ddf133c2bb982df9afe947c2fd842dc9c89e7d17d3f2f598a1d5bb7a5f={{ registry_public_url_with_path }}/cp/wiotp/api-authentication:3.3.49 +file:///cp/wiotp/api-authorization@sha256:04762e32d7b1d1705c24c56bf567f3e1cbd6a72104dd20fe4758c96a17d9aa9b={{ registry_public_url_with_path }}/cp/wiotp/api-authorization:5.3.46 +file:///cp/wiotp/api-connectors@sha256:62942ed423bcb793ba872b133b810c3211c626dcce820bc8232601bfc0a3c7b0={{ registry_public_url_with_path }}/cp/wiotp/api-connectors:4.1.3 +file:///cp/wiotp/api-dashboard@sha256:dd98151e83eada6c97fd0c51a4cfcf561c7af383fe653170f777036ab7278863={{ registry_public_url_with_path }}/cp/wiotp/api-dashboard:3.0.82 +file:///cp/wiotp/api-devicemgmt@sha256:ed3bb00f6e823052faeb643cbb2a6e02a3772981a9146aae64f830a81755823f={{ registry_public_url_with_path }}/cp/wiotp/api-devicemgmt:3.2.37 +file:///cp/wiotp/api-devops@sha256:fae4ba84a2b7af5b036906bd6d430e4b086e672bb1d2fca3073cb9001007b174={{ registry_public_url_with_path }}/cp/wiotp/api-devops:3.7.49 +file:///cp/wiotp/api-importconnectors@sha256:73b0c2ce525ed53ed4426487c25f0f1579fd51a09104b67eb15f489724d3c9c2={{ registry_public_url_with_path }}/cp/wiotp/api-importconnectors:2.0.30 +file:///cp/wiotp/api-mbgadmin@sha256:f60f533237cffcb760fb29c93e2faebddefa97be78a03d7997f4d177ece14fd2={{ registry_public_url_with_path }}/cp/wiotp/api-mbgadmin:2.4.74 +file:///cp/wiotp/api-messagesight@sha256:7f37bfd840122de1f3aba6d84adf426a0cf26a89b367ad056ee06d79f9ddffe6={{ registry_public_url_with_path }}/cp/wiotp/api-messagesight:3.1.41 +file:///cp/wiotp/api-org@sha256:307f7023dc004eb6c5fd579e17f1bc47cbcf9fc454c4b819e56d6d8463553544={{ registry_public_url_with_path }}/cp/wiotp/api-org:8.2.29 +file:///cp/wiotp/api-organizations@sha256:e4cb0a5f4acb347ff27696a82b96e46ff3044d92904716597bbc37f393ffbbe3={{ registry_public_url_with_path }}/cp/wiotp/api-organizations:10.0.39 +file:///cp/wiotp/api-pipeline@sha256:4e926b536b7713aa8a2ee166aaeebaac5d9e74a78fbf03abeeac484387054917={{ registry_public_url_with_path }}/cp/wiotp/api-pipeline:1.1.0 +file:///cp/wiotp/api-resourcecontroller@sha256:5b8b4ef64ff65e727e2c7a7134785a673ba2cdb134655800d84753107c90a8ce={{ registry_public_url_with_path }}/cp/wiotp/api-resourcecontroller:3.0.72 +file:///cp/wiotp/api-riskmgmt-secguardian@sha256:ef80d0b8fbdd629b3d47ffd9fc00b0cd63e69e5b51133a9816e09117646463bd={{ registry_public_url_with_path }}/cp/wiotp/api-riskmgmt-secguardian:5.1.47 +file:///cp/wiotp/api-s2s@sha256:476966d25e6086e36b4212cebfe8a8e6146ad80021a64eeb3dad0ea55a1850cd={{ registry_public_url_with_path }}/cp/wiotp/api-s2s:10.1.29 +file:///cp/wiotp/api-schemas@sha256:2885df674610b01e98daff4a676c03b178fbf49e5696fd872148a3f4d16602a5={{ registry_public_url_with_path }}/cp/wiotp/api-schemas:4.0.31 +file:///cp/wiotp/api-state@sha256:10b9f206b48235dfc5187496fada9077fb7d6e0082d760b4f1299d5afe4f9447={{ registry_public_url_with_path }}/cp/wiotp/api-state:5.0.32 +file:///cp/wiotp/api-status@sha256:4164d2d2b5cf0f6314b4fdfa11e780003f4632d9a2545f7fb041990fb45bcad2={{ registry_public_url_with_path }}/cp/wiotp/api-status:3.4.8 +file:///cp/wiotp/auth-store@sha256:1725ed33844394a808d6b154e4eff2016d0a21dac73fb397aa2c772dc533cc3f={{ registry_public_url_with_path }}/cp/wiotp/auth-store:2.4.51 +file:///cp/wiotp/config-store@sha256:c14d7489d9ca23441076e57758c673422e7abb00785570ed8159c4caece9cd35={{ registry_public_url_with_path }}/cp/wiotp/config-store:4.0.46 +file:///cp/wiotp/connectors-store@sha256:b6f4a3078bef6ec614db4d8114592e7768b46e294ce215ec34dee4bd62f34a9b={{ registry_public_url_with_path }}/cp/wiotp/connectors-store:8.0.34 +file:///cp/wiotp/data-import-connector@sha256:fa0ce13bb49df74d3fb4aeaa3211f6dcdb9d311e1cb3f3b979bc33dbc04b1e61={{ registry_public_url_with_path }}/cp/wiotp/data-import-connector:2.0.66 +file:///cp/wiotp/datapower-config@sha256:547f74ac50f1724c037ca8d2a15f34263cf21309d3eb3d6e37e85c1a750d5126={{ registry_public_url_with_path }}/cp/wiotp/datapower-config:10.3.1 +file:///cp/wiotp/datapower@sha256:d4122be5bb21f1333032a446849124ae1d769195576ad0cc4d5ea1d76d7c460c={{ registry_public_url_with_path }}/cp/wiotp/datapower:7.2.1 +file:///cp/wiotp/deprovagent-actions@sha256:a9a7cb86f430e9f8a1b52bb312d8a96b8f424d128007d7dab5ab42aa6a3306ca={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-actions:1.1.65 +file:///cp/wiotp/deprovagent-auth@sha256:d621f386046b9bd942db9aabf8b1b2da0225ff7f4de716ad5207f87654edb4cd={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-auth:2.3.61 +file:///cp/wiotp/deprovagent-dm@sha256:22ba1a9fb466c609a08db49adba23715c7874e0caf119683b74a7efbec87fc6f={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-dm:2.1.61 +file:///cp/wiotp/deprovagent-fpl@sha256:59963eef65e070034c8642772b804f63d670fe9f14a06d91c60e7c3d220e3073={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-fpl:1.0.14 +file:///cp/wiotp/deprovagent-guardian@sha256:e1cea24868cbea17c5415b2fe445cdc80a3ee824c3e100e0f9239b5d66814bff={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-guardian:3.1.61 +file:///cp/wiotp/deprovagent-orgmgmt@sha256:28927e185b80ad8ec971f4e1f8746c0a4434590f91bc481f2ce3ed8fddec9c5d={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-orgmgmt:2.1.63 +file:///cp/wiotp/deprovagent-provision@sha256:64caf0bf3b94f875b0c92a2a9f9567a4f6f6cdb68f869d84f91700d5043edef9={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-provision:4.4.57 +file:///cp/wiotp/deprovagent-registry@sha256:a57a0e64c81a771cf4ef6960489ca1ac7980783c2001b435e74de0129b782690={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-registry:2.2.57 +file:///cp/wiotp/deprovagent-state@sha256:37055b0230d4372e47b4981dd8c858adcf387254c81d7be5fa58a6b51e5100e8={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-state:1.2.62 +file:///cp/wiotp/deprovagent-webui@sha256:2de7621297e2e8306cb5f5e1475007058194083a33e97d0b9042e81602632323={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-webui:1.2.64 +file:///cp/wiotp/device-store@sha256:86879b7fccf9f984406633f7e4e8edceed03e3ff51d2e0705de149b9e06515da={{ registry_public_url_with_path }}/cp/wiotp/device-store:2.2.39 +file:///cp/wiotp/devicemgmt-server@sha256:15252f496515254daa8747f3e39026f2a03eefed99062afcd2b5cbfbbda236a3={{ registry_public_url_with_path }}/cp/wiotp/devicemgmt-server:3.2.35 +file:///cp/wiotp/entity-connector@sha256:1dc81de46abbad063e6db41badf74758fad0f0ab7236833329e4b1b9f437d3c7={{ registry_public_url_with_path }}/cp/wiotp/entity-connector:1.1.85 +file:///cp/wiotp/entity-manager@sha256:7a72775b4c2058387b5298d6ddeabc0add47ccad0b41e0879b4bf6c081f3baa5={{ registry_public_url_with_path }}/cp/wiotp/entity-manager:2.1.93 +file:///cp/wiotp/functionsexecutor@sha256:4b5cb0daaff9ff8366a715927a265ee6ca87e4ea49b55e8b5df19a7ad7c70871={{ registry_public_url_with_path }}/cp/wiotp/functionsexecutor:1.1.0 +file:///cp/wiotp/graphite-exporter@sha256:afb2d4af1cd0bb8e52de3e1da92b4e853aa490b1e2580f0a434ba8ceeba12fbd={{ registry_public_url_with_path }}/cp/wiotp/graphite-exporter:2.1.79 +file:///cp/wiotp/historian-configuration-manager@sha256:374d51451948d28367486f9582c2facb0232e53561608d41b95fa6e61daeaeab={{ registry_public_url_with_path }}/cp/wiotp/historian-configuration-manager:7.1.2 +file:///cp/wiotp/historian-connector@sha256:2d57ac856c1dc34096bc294d80e3184c0f0d121598ae58a66aa82e59875bf1fb={{ registry_public_url_with_path }}/cp/wiotp/historian-connector:8.1.5 +file:///cp/wiotp/ibm-iot-actions@sha256:d150aaa38ed8f3dc32685b248b4422c1848a2d87a2b168a92b50558cdfeb03e2={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-actions:2.3.0 +file:///cp/wiotp/ibm-iot-auth@sha256:1ee7931a46ae4c0a257e9587305d2a427c2a1cf140aebf5866a69a4f58fdc789={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-auth:6.3.1 +file:///cp/wiotp/ibm-iot-datapower@sha256:9c7874c1088091fc1d88a3882785ad80b9d54b639a988d7be32b8b16b5120131={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-datapower:7.2.1 +file:///cp/wiotp/ibm-iot-devops@sha256:253b6274c2d87bb0ba12df129000dd34e8081a37b04bd653ab66ef6287576086={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-devops:10.2.1 +file:///cp/wiotp/ibm-iot-dm@sha256:46a83b53e8ec51d1db6c6c86f877e72f31af4ba06b002bf9fb1165bd77658678={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-dm:2.2.1 +file:///cp/wiotp/ibm-iot-dsc@sha256:72e8c6b0ce0f772a60c415b095494a0f744de638288bcf1a3aebd2b28e049bdc={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-dsc:6.2.1 +file:///cp/wiotp/ibm-iot-fpl@sha256:b21d37ff6414e0ac423c1bf95e38b747b345eebe5152901fa09ad6954788a02e={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-fpl:1.1.0 +file:///cp/wiotp/ibm-iot-guardian@sha256:3af0f8a7ce6011e27988d2ec779d567e729bd6bdc584112d78cfea1f546d8a69={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-guardian:4.2.1 +file:///cp/wiotp/ibm-iot-mbgx@sha256:5adf9cefbfdee1cc3a30598e6c55794ada9d577a975161891303eb1e2826dd67={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-mbgx:3.3.1 +file:///cp/wiotp/ibm-iot-mfgx@sha256:f1c5a8bb66c2fd879763c8981dd5a9ab0d2de34426eb59310816049f3cd5f92f={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-mfgx:6.3.1 +file:///cp/wiotp/ibm-iot-monitor@sha256:5241962c295cfe71a6d4f4c90306fdc765a7486f81ec4674caf64c3d476da194={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-monitor:2.3.0 +file:///cp/wiotp/ibm-iot-orgmgmt@sha256:4f575f552e11599d35c941c3d53d968201ea9019eec7e601244bb891cc2b5cdd={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-orgmgmt:6.3.0 +file:///cp/wiotp/ibm-iot-provision@sha256:06f28fa79b0bd9184c1b0c5c43f0e07e1cdd70d06d503b20e7dc1d43b8ce71a7={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-provision:10.2.1 +file:///cp/wiotp/ibm-iot-registry@sha256:730fc97a9e72eebac635aea847e567c5e1f89708c585221e1453be062c97c0ac={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-registry:8.2.1 +file:///cp/wiotp/ibm-iot-state@sha256:fe9ded36f0bcaeeacb9d270760baaffb7a7081fd54717cc30b7473f5154226ce={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-state:5.2.1 +file:///cp/wiotp/ibm-iot-webui@sha256:b1c3e07ab5bd302c24f93753ae1c1395a14da9879dede6eaecd1e3a327090270={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-webui:6.2.1 +file:///cp/wiotp/ibm-iot-workspace@sha256:c8ae641bbe8b46352dce21495a600a407d7bdaa88dda23c1ca87b89d5368293e={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-workspace:2.5.1 +file:///cp/wiotp/kafkamodel-actions@sha256:da423d4fbb7d8dbada9b1ddb6d7d55b63f6c95e97e58e61ff69aaa1dc6d02b5b={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-actions:1.4.6 +file:///cp/wiotp/kafkamodel-auth@sha256:ad1e3090cbec3dc68dbed33baf4466bad053c14a3439db6766a497aced8e572e={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-auth:1.7.6 +file:///cp/wiotp/kafkamodel-fpl@sha256:29ac84c25d04af5828a2af92420904ebbeb594e67ee093c80f4801bdcbf2b0d8={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-fpl:1.1.6 +file:///cp/wiotp/kafkamodel-mfgx@sha256:f950837e92d28ac2904b2ce39d30817e2ae4348a003d0b83898b87c412813a2c={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-mfgx:1.3.6 +file:///cp/wiotp/kafkamodel-orgmgmt@sha256:d8f7be2c19042439dea0ecc487c2d8a6e94860af0642c52af0a651c9f48bc1d5={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-orgmgmt:2.2.6 +file:///cp/wiotp/kafkamodel-registry@sha256:de865fff88e4d25811287de01d3e998d7f246340f97004e7535fed11a8ee6eb8={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-registry:1.4.6 +file:///cp/wiotp/kafkamodel-state@sha256:0a8c7645ccf1ccecff108d1fbc993161e099e72becad085dcb0e7b837cfcabfb={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-state:1.4.6 +file:///cp/wiotp/masuseragent@sha256:06a65a089e71b9cb07bf64da7bdd75faa265970de868309e2da460cdc5aed1d3={{ registry_public_url_with_path }}/cp/wiotp/masuseragent:1.4.8 +file:///cp/wiotp/messagesight-configurator@sha256:e90f01c84e4a84fefafdcf02731b8f79c08d23d3501c3cdccb1f532514c346bf={{ registry_public_url_with_path }}/cp/wiotp/messagesight-configurator:5.14.35 +file:///cp/wiotp/messagesight@sha256:24597be9d139ba12c4573fff74a41f1323d77be9443469d8633fbd4bde8e6744={{ registry_public_url_with_path }}/cp/wiotp/messagesight:5.14.35 +file:///cp/wiotp/monagent-iot@sha256:da292da546da4acdc27f9865b0040384dcb79ca16eb9455ea907ffdcd5c37b0c={{ registry_public_url_with_path }}/cp/wiotp/monagent-iot:1.1.0 +file:///cp/wiotp/monagent-msserver@sha256:9067b5c576333dcd272883cb20ccd45950cdd5f30e2d566c8af7501e2079b1be={{ registry_public_url_with_path }}/cp/wiotp/monagent-msserver:4.3.70 +file:///cp/wiotp/monagent-org@sha256:0ac2638d4d4c590fa7a3566aa6b8a902de54e8ecbd93c61c851a0cbe461cedf3={{ registry_public_url_with_path }}/cp/wiotp/monagent-org:2.1.83 +file:///cp/wiotp/mongomodel-actions@sha256:b30714e01aedcf2654f718308a29591157937290301cb2c157ea1536975bdd79={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-actions:1.2.76 +file:///cp/wiotp/mongomodel-dm@sha256:e9eb535d9b156c1e6f7c6fc3a3ddf6de346fadc0e30aef6a7f32354cba4bc9df={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-dm:1.3.80 +file:///cp/wiotp/mongomodel-dsc@sha256:f58651c00dd8c5f1f87ca323c521fe02dbd1787449a79f860e4a635c5c3cf667={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-dsc:1.2.78 +file:///cp/wiotp/mongomodel-guardian@sha256:db8fbf2d5a1cc7b453dc262954e9de782d0d074bacc67b8bdf90c18154813349={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-guardian:1.3.75 +file:///cp/wiotp/mongomodel-orgmgmt@sha256:c277dfa1c54e33aab55b195745bd74c967bb7febdf92f20d6f612eb36eb1372a={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-orgmgmt:2.0.75 +file:///cp/wiotp/mongomodel-provision@sha256:76f16025df17d94ea32047624123b967186c95d74514e344f4dd892efd7f84fc={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-provision:2.5.65 +file:///cp/wiotp/mongomodel-registry@sha256:bb49e23af6c150de2bcfdced8c617c7354814694c8137caacfabbb7bf78d7032={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-registry:2.0.71 +file:///cp/wiotp/mongomodel-state@sha256:c3df1e9b8d2e895b92088a733745e683d1dd16d6b77b4a768c3df21cfcb30c9f={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-state:1.2.72 +file:///cp/wiotp/mongomodel-webui@sha256:9cab9ba25341778564f5b98567a429b9e12f6cc4da55d92c935f7561c2a0ab28={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-webui:2.3.79 +file:///cp/wiotp/mqtt-connector@sha256:b51922a2283c4581b910878e7d542b4ab2de6e5d670305657eedd7f0a15239b4={{ registry_public_url_with_path }}/cp/wiotp/mqtt-connector:2.0.73 +file:///cp/wiotp/msproxy-monitor@sha256:b53dfac38f002814c2a33c1c13778b55954a095a2132308543624dd61d27b05f={{ registry_public_url_with_path }}/cp/wiotp/msproxy-monitor:8.5.42 +file:///cp/wiotp/msproxy@sha256:417a3d8f94a19ededfe6b53d4ecd3d8622abd7bbbf990d3ac3ae303b92dbc1f8={{ registry_public_url_with_path }}/cp/wiotp/msproxy:8.5.42 +file:///cp/wiotp/org-store@sha256:72796a6a6c92299f68b3d5581b7325db9d3600cadd22e67133c43eb275295873={{ registry_public_url_with_path }}/cp/wiotp/org-store:4.0.46 +file:///cp/wiotp/orgdeprovmgr@sha256:78bc78197b119c0f92fa299d9a9473d4dc221d1b1c851f304c394e960a0c6baa={{ registry_public_url_with_path }}/cp/wiotp/orgdeprovmgr:2.2.66 +file:///cp/wiotp/orgpoolmgr@sha256:f4424d91a381116815b5a2448c7a606bb2a0ff6c45f74ab92bf5d31f7153bb0c={{ registry_public_url_with_path }}/cp/wiotp/orgpoolmgr:3.1.78 +file:///cp/wiotp/orgprovmgr@sha256:f71ede6293c82e2c85e89e0d331f3674541effcabfc4aeb851ae9f6551ec4611={{ registry_public_url_with_path }}/cp/wiotp/orgprovmgr:2.1.64 +file:///cp/wiotp/pipelinerouter@sha256:1c0047681cf5a8eb6c867c3581d613a64a03c326e23c7a9b292cd55573ca2171={{ registry_public_url_with_path }}/cp/wiotp/pipelinerouter:1.1.0 +file:///cp/wiotp/provagent-guardian@sha256:37b9d3a251c2597c0eb4e5a744b9fb871a0436691649d434e42ad8877a7b2d05={{ registry_public_url_with_path }}/cp/wiotp/provagent-guardian:2.1.65 +file:///cp/wiotp/s2s-store@sha256:4bbf05141df77240f76ca90b183b3200822b60d3d13df05e5be39c232a8b1c6a={{ registry_public_url_with_path }}/cp/wiotp/s2s-store:6.1.31 +file:///cp/wiotp/sqlmodel-auth@sha256:6ec3d1dc0ab7b04be6b3d948606e030f2ceaff91271650b37fee7b87b60643f1={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-auth:1.1.17 +file:///cp/wiotp/sqlmodel-fpl@sha256:c65076e777fa55fb94625cc228cea0e45b2e7397e1ed4aa1e9d82ea0d823e215={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-fpl:1.0.15 +file:///cp/wiotp/sqlmodel-state@sha256:9a1551a240c5b925f272e0c43cfa949b099d12136d98dfb9c213608e0446a27b={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-state:1.0.62 +file:///cp/wiotp/state-updater@sha256:0fdffec079afe9277c6ce656b270cc1ef1b2aaca80e0b425ddc4c1c596d72aab={{ registry_public_url_with_path }}/cp/wiotp/state-updater:2.0.35 +file:///cp/wiotp/statsd@sha256:5cb1c6005fa136d1c1cb0c0b04ae4cfb5a6588d98d74bf046784725d1a50aa3d={{ registry_public_url_with_path }}/cp/wiotp/statsd:1.3.70 +file:///cp/wiotp/ui-dashboard@sha256:d0ccf2e8175c83a77aa8136ceebe46188800eb82db6a745ef455e2812b068916={{ registry_public_url_with_path }}/cp/wiotp/ui-dashboard:7.1.67 +file:///cpopen/ibm-mas-iot-operator-bundle@sha256:e576f5731f052b99d95bba2f1491c37349259de1696ae46124ff16bdc253bb83={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot-operator-bundle:8.6.0 +file:///cpopen/ibm-mas-iot-operator-catalog@sha256:09b6560684c9f651a9c837b080498e46162a334fb02ff5bb7a5b8faf75734e70={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot-operator-catalog:latest +file:///cpopen/ibm-mas-iot@sha256:870d956908cec2bbc7d6fd5ffa9863fe02d643c183910e7e6fd211d391027564={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot:8.6.0 \ No newline at end of file diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/iot867fix/direct.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/iot867fix/direct.txt.j2 index a682d7f3e9..203dd98436 100644 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/iot867fix/direct.txt.j2 +++ b/ibm/mas_devops/roles/mirror_case_prepare/templates/iot867fix/direct.txt.j2 @@ -1,102 +1,102 @@ -cp.icr.io/cp/mas/ltpakeys-generator@sha256:230d68dcb26a5890efafa591243b6450a42ef695514be6cb60a9eface269499b={{ registry_public_url }}/cp/mas/ltpakeys-generator:2.6.64 -cp.icr.io/cp/wiotp/action-mgr@sha256:b7a8cabcf40b8e72863647b37024b5d21a8ec85f6b7143f3ce73a3553e31948b={{ registry_public_url }}/cp/wiotp/action-mgr:2.1.19 -cp.icr.io/cp/wiotp/api-actions@sha256:19c859464b67067d3fee95742838495d7315299c174f4d61a032f4d06aa71eed={{ registry_public_url }}/cp/wiotp/api-actions:2.1.69 -cp.icr.io/cp/wiotp/api-authentication@sha256:f7e136742d5056d169979779969f53a46a83521b1b1c02f54f5466aaa657ff84={{ registry_public_url }}/cp/wiotp/api-authentication:3.3.72 -cp.icr.io/cp/wiotp/api-authorization@sha256:ab78d68d7b3a988ea74f51432db103e9e7afd61659c0840b50c92739bcff0130={{ registry_public_url }}/cp/wiotp/api-authorization:5.3.69 -cp.icr.io/cp/wiotp/api-connectors@sha256:d881ed9439d5d9a9f6ed65a3023099fe6ba380bc6b67172ea229848b7045b317={{ registry_public_url }}/cp/wiotp/api-connectors:4.1.31 -cp.icr.io/cp/wiotp/api-dashboard@sha256:0438deb2b715c21f7e34323e98664321a83d57481048ce9ae97f43fb36516eed={{ registry_public_url }}/cp/wiotp/api-dashboard:3.0.114 -cp.icr.io/cp/wiotp/api-devicemgmt@sha256:603f981262b914c7a69bfdf4f63664adb8951889e951e359187f83361dffca8c={{ registry_public_url }}/cp/wiotp/api-devicemgmt:3.2.62 -cp.icr.io/cp/wiotp/api-devops@sha256:7e850ed53d16e6071b19398f4ca98dbd71444c2e27a8cdd0a6e4fee974153f5f={{ registry_public_url }}/cp/wiotp/api-devops:3.7.76 -cp.icr.io/cp/wiotp/api-importconnectors@sha256:7f34e81197fc1a4ac8b2beb8fc7ac0ca3e9bca6c63b874b31040f7088f4b97e2={{ registry_public_url }}/cp/wiotp/api-importconnectors:2.0.62 -cp.icr.io/cp/wiotp/api-mbgadmin@sha256:3b137a48623d91d0ffcee3985fdee7793caa130fd3eb7386e7ca35203bb16c77={{ registry_public_url }}/cp/wiotp/api-mbgadmin:2.4.107 -cp.icr.io/cp/wiotp/api-messagesight@sha256:26784a0b6a665d2908acb14f9f4d6a632901260c92a54c2eda3fc20b4bdc2f6b={{ registry_public_url }}/cp/wiotp/api-messagesight:3.1.65 -cp.icr.io/cp/wiotp/api-org@sha256:383aafbbaa37a16c24e6124165c0cabbdaf4ac34e048e26bbf942594ddbcf61a={{ registry_public_url }}/cp/wiotp/api-org:8.2.53 -cp.icr.io/cp/wiotp/api-organizations@sha256:942ede80f46a3f42e708732321d24976bc419601f849439ca1ae1f882bb08d80={{ registry_public_url }}/cp/wiotp/api-organizations:10.0.65 -cp.icr.io/cp/wiotp/api-pipeline@sha256:998497b0d1c5852700651ff6108b4f974516996237a50c9392f11828e616a98b={{ registry_public_url }}/cp/wiotp/api-pipeline:1.1.15 -cp.icr.io/cp/wiotp/api-resourcecontroller@sha256:d01f5ac32696f7b975231097e3a1215869c29b7452b1d39d1643af642f22bdd6={{ registry_public_url }}/cp/wiotp/api-resourcecontroller:3.0.103 -cp.icr.io/cp/wiotp/api-riskmgmt-secguardian@sha256:72f34118ae172d1e2f18c77e12b35ce04c846cd4d684cf8914d5b3db6b91b55a={{ registry_public_url }}/cp/wiotp/api-riskmgmt-secguardian:5.1.73 -cp.icr.io/cp/wiotp/api-s2s@sha256:4275b8d8aab9e519aecd8478a8fe3af9417ed70f2a0a43a49609d75cbb375e82={{ registry_public_url }}/cp/wiotp/api-s2s:10.1.54 -cp.icr.io/cp/wiotp/api-schemas@sha256:bc09bfd8d5acbaf08fa36a1507315c5be0e782c1e97781420a23d5e8c0436ff3={{ registry_public_url }}/cp/wiotp/api-schemas:4.0.61 -cp.icr.io/cp/wiotp/api-state@sha256:e74bd94d9d9e21f6182483b3d98935c31e382f42d488af5e5eb270c6802219f2={{ registry_public_url }}/cp/wiotp/api-state:5.1.21 -cp.icr.io/cp/wiotp/api-status@sha256:e1bf748cede2208cf905067bd0a0f99da56985bf76fab8669e672f0e61473808={{ registry_public_url }}/cp/wiotp/api-status:3.4.37 -cp.icr.io/cp/wiotp/auth-store@sha256:914b6f285ae9af82ff42024ebee330962d5521f441307478a22393ce9e2af2fa={{ registry_public_url }}/cp/wiotp/auth-store:2.5.12 -cp.icr.io/cp/wiotp/config-store@sha256:aa728cd2c3d86a791136ddd9f9ba48c372ce59154b70978f018f7e6a24025494={{ registry_public_url }}/cp/wiotp/config-store:4.0.73 -cp.icr.io/cp/wiotp/connectors-store@sha256:f27990e2095ffa3c0de7a5babb896e85152ece831d19b4ee077337a9b6a43324={{ registry_public_url }}/cp/wiotp/connectors-store:8.0.60 -cp.icr.io/cp/wiotp/data-import-connector@sha256:487879c5ca803c8dea56bace82e722f422ef661affccf17e678fb0dac776acfe={{ registry_public_url }}/cp/wiotp/data-import-connector:2.1.25 -cp.icr.io/cp/wiotp/datapower-config@sha256:c92418d8c3442fe0832cd257b5a1be2670f88c378df33b2f7d2a613b4e446e06={{ registry_public_url }}/cp/wiotp/datapower-config:10.4.2 -cp.icr.io/cp/wiotp/datapower@sha256:3e3407c06e399c7d30d2298b9ea1ceed7a7f3a2a06d97ea952e379d454b4f252={{ registry_public_url }}/cp/wiotp/datapower:7.4.4 -cp.icr.io/cp/wiotp/deprovagent-actions@sha256:08fdd43daeb7f66146e7cecb8bbfed69fb499cb358492b397fc3f9560f56d341={{ registry_public_url }}/cp/wiotp/deprovagent-actions:1.1.89 -cp.icr.io/cp/wiotp/deprovagent-auth@sha256:d51acf8747d0ec2537fc92a9d4a01ea84583e887debc4de50922811ef9176861={{ registry_public_url }}/cp/wiotp/deprovagent-auth:2.3.85 -cp.icr.io/cp/wiotp/deprovagent-dm@sha256:4dab0cab69eeaabc7900b656c61587bf46302328d1ba57da1d0ca5e84a3b9e1d={{ registry_public_url }}/cp/wiotp/deprovagent-dm:2.1.85 -cp.icr.io/cp/wiotp/deprovagent-fpl@sha256:160ef36c62a6fd4ac31c58a2b214084c51b85b9a5bbbe1ae918d65094a8f4856={{ registry_public_url }}/cp/wiotp/deprovagent-fpl:1.0.38 -cp.icr.io/cp/wiotp/deprovagent-guardian@sha256:63b42d1243938bac7e040ab8bd142b2ab4a5216388b8db789292d1ee0bf45879={{ registry_public_url }}/cp/wiotp/deprovagent-guardian:3.1.86 -cp.icr.io/cp/wiotp/deprovagent-orgmgmt@sha256:e584c51fcff47b1d0d2fdaea4a01c0d33d64c39959062f745d3f65b9b0343e2f={{ registry_public_url }}/cp/wiotp/deprovagent-orgmgmt:2.1.87 -cp.icr.io/cp/wiotp/deprovagent-provision@sha256:1e6d835aac567d560ab4044e0fd65f45bd6faa85dfd53d08498a36548744019f={{ registry_public_url }}/cp/wiotp/deprovagent-provision:4.4.81 -cp.icr.io/cp/wiotp/deprovagent-registry@sha256:a73dfe91c9186f72d679cb58d51630e915cca252cc3bac4cad96e667b0741c4f={{ registry_public_url }}/cp/wiotp/deprovagent-registry:2.2.81 -cp.icr.io/cp/wiotp/deprovagent-state@sha256:c0106b48a2c803b6ec8d713527893d63e54a1ad465dd55341c443eb5a3356195={{ registry_public_url }}/cp/wiotp/deprovagent-state:1.2.86 -cp.icr.io/cp/wiotp/deprovagent-webui@sha256:cecfe1ae2db564233796651b129a5274fc6a50d0b2c8349ae6fee15e9670e0e9={{ registry_public_url }}/cp/wiotp/deprovagent-webui:1.2.88 -cp.icr.io/cp/wiotp/device-store@sha256:a82f3435b5bb30f2d3945e182471033d4b56e536dc06826e5837aff49bd7aa43={{ registry_public_url }}/cp/wiotp/device-store:2.2.66 -cp.icr.io/cp/wiotp/devicemgmt-server@sha256:4375a6ca7678907aff6eb94d9e16a239d8449bbd1f315fa2969abd957aa206b5={{ registry_public_url }}/cp/wiotp/devicemgmt-server:3.2.61 -cp.icr.io/cp/wiotp/entity-connector@sha256:155ab5d7be26d1c9c8cd2e7275b0eb43c03abdbd3b9aa6be6156d3c87a055e31={{ registry_public_url }}/cp/wiotp/entity-connector:1.1.115 -cp.icr.io/cp/wiotp/entity-manager@sha256:70bd1fdc6216c81be4e91e927b70c5b669141a89baee8701a0ea0ad9122da684={{ registry_public_url }}/cp/wiotp/entity-manager:2.1.122 -cp.icr.io/cp/wiotp/functionsexecutor@sha256:3bedf02cebbde0869ea94f46bc04b16c89bfc2d350f99abe911c30feb7118830={{ registry_public_url }}/cp/wiotp/functionsexecutor:1.1.28 -cp.icr.io/cp/wiotp/graphite-exporter@sha256:beb713921357d06cb12deb3715eac2b8d75b41bb6fe5e24cc7b4dc46061bc00c={{ registry_public_url }}/cp/wiotp/graphite-exporter:2.1.111 -cp.icr.io/cp/wiotp/historian-configuration-manager@sha256:4469140c1583e62145f21b040fe7c8f228f8eb5dd441587b3706627746ef4fda={{ registry_public_url }}/cp/wiotp/historian-configuration-manager:7.1.26 -cp.icr.io/cp/wiotp/historian-connector@sha256:a548da2a8d7a342c13b21862f19b07b9b511c366dfd424a2ebf2a0f9a556d90a={{ registry_public_url }}/cp/wiotp/historian-connector:8.2.22 -cp.icr.io/cp/wiotp/ibm-iot-actions@sha256:39706dac57a92048e1721d9f47c84834c51ae625bae9914d92ddadca0205d0ae={{ registry_public_url }}/cp/wiotp/ibm-iot-actions:2.5.5 -cp.icr.io/cp/wiotp/ibm-iot-auth@sha256:86121b4d1936030307c820751dac44678c9c50de78ccddfc4a5915f195e7a91e={{ registry_public_url }}/cp/wiotp/ibm-iot-auth:6.4.6 -cp.icr.io/cp/wiotp/ibm-iot-datapower@sha256:960d3448ab649388fff3de772f86c31dc8417751358bdf8c16e05df2368fd7af={{ registry_public_url }}/cp/wiotp/ibm-iot-datapower:7.4.4 -cp.icr.io/cp/wiotp/ibm-iot-devops@sha256:d364349df5f903ae91a318152d2a97ce47db2c1871472c9a1105c4ed7e3a9b32={{ registry_public_url }}/cp/wiotp/ibm-iot-devops:10.3.8 -cp.icr.io/cp/wiotp/ibm-iot-dm@sha256:25e6773484cc0c94fcf9d17108c9555155004834511392b917f0c3ad44ca5e1f={{ registry_public_url }}/cp/wiotp/ibm-iot-dm:2.3.6 -cp.icr.io/cp/wiotp/ibm-iot-dsc@sha256:335b0f0a5b0142184c7d179bcd436cfdd203918066d7f7951d0fc28d0c4222fb={{ registry_public_url }}/cp/wiotp/ibm-iot-dsc:6.3.6 -cp.icr.io/cp/wiotp/ibm-iot-fpl@sha256:6bad878ca0af8dbc39d6394d38b2018430fe07d9954f502c00113fc131e03907={{ registry_public_url }}/cp/wiotp/ibm-iot-fpl:1.1.10 -cp.icr.io/cp/wiotp/ibm-iot-guardian@sha256:5fa64940e086b8d35da2bef5d5ea7edf724a217522e72a7454d345347c277b34={{ registry_public_url }}/cp/wiotp/ibm-iot-guardian:4.3.6 -cp.icr.io/cp/wiotp/ibm-iot-mbgx@sha256:a39c03250a1db315cd806476f920dba083674c0ab14aeedb9cdd6232570c047c={{ registry_public_url }}/cp/wiotp/ibm-iot-mbgx:3.3.10 -cp.icr.io/cp/wiotp/ibm-iot-mfgx@sha256:5c7268ae46a22140b2eecea1d18fc1c5f474833480b478d2a0f3b8d8e7d54e34={{ registry_public_url }}/cp/wiotp/ibm-iot-mfgx:6.3.11 -cp.icr.io/cp/wiotp/ibm-iot-monitor@sha256:cd56208a25fb6a73f0d5728d5b5b3a063e8a2f66ea3805f0a91763e9e6c86a48={{ registry_public_url }}/cp/wiotp/ibm-iot-monitor:2.5.6 -cp.icr.io/cp/wiotp/ibm-iot-orgmgmt@sha256:2233efd33e2eaeb065abec2cdd5dd28503a6a96186417cb28cc293ad5b0ff5e3={{ registry_public_url }}/cp/wiotp/ibm-iot-orgmgmt:6.8.2 -cp.icr.io/cp/wiotp/ibm-iot-provision@sha256:9f43c84d039711aa6c1370cc9da70970bc9a1d40b6d48bf98cdcabdb5f3d34ad={{ registry_public_url }}/cp/wiotp/ibm-iot-provision:10.3.6 -cp.icr.io/cp/wiotp/ibm-iot-registry@sha256:f596037ea35e22046628393132f7995c364436198367354f728572ceed7ed931={{ registry_public_url }}/cp/wiotp/ibm-iot-registry:8.3.5 -cp.icr.io/cp/wiotp/ibm-iot-state@sha256:55a2b9965d53cb76c4b667a437be8de1a977ef9d47d63a650a397506856268cd={{ registry_public_url }}/cp/wiotp/ibm-iot-state:5.3.8 -cp.icr.io/cp/wiotp/ibm-iot-webui@sha256:e69526c637df84daf56198673931c1c56614b873f35316b92bad5fe98b174890={{ registry_public_url }}/cp/wiotp/ibm-iot-webui:6.4.5 -cp.icr.io/cp/wiotp/ibm-iot-workspace@sha256:9510e6e2a07991c5a5bb7394aaecb8948a4bec3898dc7a17e192b136d842a1d7={{ registry_public_url }}/cp/wiotp/ibm-iot-workspace:2.6.6 -cp.icr.io/cp/wiotp/kafkamodel-actions@sha256:c87c7cd46ad137cae4d1e79171b4be2d8d618182e3283bba5f45941042f4a654={{ registry_public_url }}/cp/wiotp/kafkamodel-actions:1.4.28 -cp.icr.io/cp/wiotp/kafkamodel-auth@sha256:431cde1ee9c9cca61b868a9bb525f3b43301087c9ed6d726374e82a5742ef7fa={{ registry_public_url }}/cp/wiotp/kafkamodel-auth:1.7.28 -cp.icr.io/cp/wiotp/kafkamodel-mfgx@sha256:b8e2af076c21878511a9b8c242d4ba76320db3ac1ff3c12f692585c812e4587d={{ registry_public_url }}/cp/wiotp/kafkamodel-mfgx:1.3.30 -cp.icr.io/cp/wiotp/kafkamodel-orgmgmt@sha256:a13b094015d0b7a557e2e4bde60ad4c933c7fb19b02940d2634dfdfc156f2111={{ registry_public_url }}/cp/wiotp/kafkamodel-orgmgmt:2.2.28 -cp.icr.io/cp/wiotp/kafkamodel-registry@sha256:2973afe6531d47aa8cc5ab96a984855994d48999f91390394c7705d8697e90a1={{ registry_public_url }}/cp/wiotp/kafkamodel-registry:1.4.28 -cp.icr.io/cp/wiotp/kafkamodel-state@sha256:6d4e9447b0a2fca8bd80a2b06fc695b2349ecc1b68739857dcd9e3513d2546b7={{ registry_public_url }}/cp/wiotp/kafkamodel-state:1.4.28 -cp.icr.io/cp/wiotp/masuseragent@sha256:82650ce1f193d4a7e54cfeab8f26a612cfaf4e0599dc49f5dcd87f02a1deb644={{ registry_public_url }}/cp/wiotp/masuseragent:1.4.39 -cp.icr.io/cp/wiotp/messagesight-configurator@sha256:020abc6cd44da995b67c9e03d4085ff61830ba4bc1fb4c0ba9406ce1e77c9c25={{ registry_public_url }}/cp/wiotp/messagesight-configurator:5.14.53 -cp.icr.io/cp/wiotp/messagesight@sha256:01e55d9f71c5213a652338b99ce3a186c780af6ac453d20248886ab8fcb06e71={{ registry_public_url }}/cp/wiotp/messagesight:5.14.53 -cp.icr.io/cp/wiotp/monagent-iot@sha256:10dcb43ad3b1601c99233f0862355959765fabc7287b1b25aeb6ee80d06a64c9={{ registry_public_url }}/cp/wiotp/monagent-iot:2.0.4 -cp.icr.io/cp/wiotp/monagent-msserver@sha256:00a865afd7cde57f23ebbc311193112d5a2b7630ab0aa451b01a7699ee0ae73b={{ registry_public_url }}/cp/wiotp/monagent-msserver:4.3.102 -cp.icr.io/cp/wiotp/monagent-org@sha256:b07f5cb43f288f67e2fdf311815542989377408295474854c06443883875a6e8={{ registry_public_url }}/cp/wiotp/monagent-org:2.1.112 -cp.icr.io/cp/wiotp/mongomodel-actions@sha256:66c7769d5ee4b512e7ba2df994b9f0839ba9faf3ed5a11a24e47851487e1a35e={{ registry_public_url }}/cp/wiotp/mongomodel-actions:1.2.101 -cp.icr.io/cp/wiotp/mongomodel-dm@sha256:03b98920ec9c89ddfcb77bcd6906739235d3e59f78560a058735057a0847b6f9={{ registry_public_url }}/cp/wiotp/mongomodel-dm:1.3.105 -cp.icr.io/cp/wiotp/mongomodel-dsc@sha256:e29b85f767b44b1b9e4957ce1c3350d639be46776ec2f9e99570a00db974f528={{ registry_public_url }}/cp/wiotp/mongomodel-dsc:1.2.104 -cp.icr.io/cp/wiotp/mongomodel-guardian@sha256:30abe64ec3b86a585eb2596cc07aac1dc3ac0be63107841a67ddb2a568948454={{ registry_public_url }}/cp/wiotp/mongomodel-guardian:1.3.101 -cp.icr.io/cp/wiotp/mongomodel-orgmgmt@sha256:24fab89bd6dc3ad91dc65af3d2cb65d7286f6e526eeddbaa481a4c9747eecf4e={{ registry_public_url }}/cp/wiotp/mongomodel-orgmgmt:2.0.99 -cp.icr.io/cp/wiotp/mongomodel-provision@sha256:88955bfc4285a084b80eb56cb6144d25deea71d0504df04ce20ffa522274d8c3={{ registry_public_url }}/cp/wiotp/mongomodel-provision:2.5.90 -cp.icr.io/cp/wiotp/mongomodel-registry@sha256:03fcc9c55758a1ef4ad159c5af57e31d3540e7e49a957844af2dcf9d195604ad={{ registry_public_url }}/cp/wiotp/mongomodel-registry:2.0.95 -cp.icr.io/cp/wiotp/mongomodel-state@sha256:9bdeaef9131dc2192945db63a368aee4155a53b9e9a941c975b157f2c93e6137={{ registry_public_url }}/cp/wiotp/mongomodel-state:1.2.97 -cp.icr.io/cp/wiotp/mongomodel-webui@sha256:4f0f6f396bbfbefe4572941f1028d4e3961e4497b3b8cedf3c871008d6d30369={{ registry_public_url }}/cp/wiotp/mongomodel-webui:2.3.104 -cp.icr.io/cp/wiotp/mqtt-connector@sha256:5e3ccf1909c09999731c2aa9b65439f9b0cfa8bdab59e1c2e67e6ebb1eb22404={{ registry_public_url }}/cp/wiotp/mqtt-connector:2.0.103 -cp.icr.io/cp/wiotp/msproxy-monitor@sha256:9e2fe23f914bbf904caa92fd8782e170f6fba87635a39c848b3b38c9a6c9673b={{ registry_public_url }}/cp/wiotp/msproxy-monitor:8.6.31 -cp.icr.io/cp/wiotp/msproxy@sha256:8836dd25392c9e1ac74039046c79be36492aac574bc16aaa1c8b2e498c073a5e={{ registry_public_url }}/cp/wiotp/msproxy:8.6.31 -cp.icr.io/cp/wiotp/org-store@sha256:d3e58211d9cb8d89f0a5ade4bfd45f1c9590e303febe914f1a9594fae2c05676={{ registry_public_url }}/cp/wiotp/org-store:4.0.72 -cp.icr.io/cp/wiotp/orgdeprovmgr@sha256:349824703278e4c3a179a1cc86dfaf0c33e655e0cd26f93b26fc1e6f3b86f2ea={{ registry_public_url }}/cp/wiotp/orgdeprovmgr:2.2.91 -cp.icr.io/cp/wiotp/orgpoolmgr@sha256:52c948f7697ebda09fc530ada7faea6b4d7c6c0525729ba5f21e7593831fec82={{ registry_public_url }}/cp/wiotp/orgpoolmgr:3.1.108 -cp.icr.io/cp/wiotp/orgprovmgr@sha256:37bbd47ece8a22ad7010d0327d11732753bfc207060d30f5775ed249357d31ae={{ registry_public_url }}/cp/wiotp/orgprovmgr:2.1.90 -cp.icr.io/cp/wiotp/pipelinerouter@sha256:e340853661440d6406775df1ff8bf7a00d744e3ee82a40139410f4634a40ffeb={{ registry_public_url }}/cp/wiotp/pipelinerouter:1.1.25 -cp.icr.io/cp/wiotp/provagent-guardian@sha256:f755105203727598a3c24fa50b4245293b32f532ad28861c4b2c2fafd7974b20={{ registry_public_url }}/cp/wiotp/provagent-guardian:2.1.88 -cp.icr.io/cp/wiotp/s2s-store@sha256:dd3770076413737fd826a2d1db721e69a8ad1553db338774038fe1d82f048dfd={{ registry_public_url }}/cp/wiotp/s2s-store:6.1.54 -cp.icr.io/cp/wiotp/sqlmodel-auth@sha256:8d3194e0315f1cd8cdeaecc7980ac525a27afee39283aef56f5cd82e5e2d05f2={{ registry_public_url }}/cp/wiotp/sqlmodel-auth:1.1.40 -cp.icr.io/cp/wiotp/sqlmodel-fpl@sha256:045ea5513b39251b55706638039fbe2af4feef23368d5dd72d48584a2cde5d81={{ registry_public_url }}/cp/wiotp/sqlmodel-fpl:1.0.33 -cp.icr.io/cp/wiotp/sqlmodel-state@sha256:09631b932110a0e17d702a143ea57ad82dee6642825138b2cc89ab8ea670f84d={{ registry_public_url }}/cp/wiotp/sqlmodel-state:1.0.85 -cp.icr.io/cp/wiotp/state-updater@sha256:503c371e30b15858b6c70e2e4c766555c367ed6b18e225a6bd391efa7d723e73={{ registry_public_url }}/cp/wiotp/state-updater:2.2.19 -cp.icr.io/cp/wiotp/statsd@sha256:a03313c0064137668c333d5bf9ab50749947c35981366b931f83855b723df8c3={{ registry_public_url }}/cp/wiotp/statsd:1.3.99 -cp.icr.io/cp/wiotp/ui-dashboard@sha256:2721067fc2cfc8bf9e041ff731d18bdd616f1ffc2b4aad6dcd8615aa0de19c4c={{ registry_public_url }}/cp/wiotp/ui-dashboard:7.4.8 -icr.io/cpopen/ibm-mas-iot-operator-bundle@sha256:524ec52f1142db1641ae30b6c2b91b80d6b4ab45534f308c93319a9103b9bcf5={{ registry_public_url }}/cpopen/ibm-mas-iot-operator-bundle:8.6.7 -icr.io/cpopen/ibm-mas-iot-operator-catalog@sha256:09b6560684c9f651a9c837b080498e46162a334fb02ff5bb7a5b8faf75734e70={{ registry_public_url }}/cpopen/ibm-mas-iot-operator-catalog:latest -icr.io/cpopen/ibm-mas-iot@sha256:eea161ace585c004d0442439f0acac7e3af1aea78478bb8ee5ded41049e27839={{ registry_public_url }}/cpopen/ibm-mas-iot:8.6.7 \ No newline at end of file +cp.icr.io/cp/mas/ltpakeys-generator@sha256:230d68dcb26a5890efafa591243b6450a42ef695514be6cb60a9eface269499b={{ registry_public_url_with_path }}/cp/mas/ltpakeys-generator:2.6.64 +cp.icr.io/cp/wiotp/action-mgr@sha256:b7a8cabcf40b8e72863647b37024b5d21a8ec85f6b7143f3ce73a3553e31948b={{ registry_public_url_with_path }}/cp/wiotp/action-mgr:2.1.19 +cp.icr.io/cp/wiotp/api-actions@sha256:19c859464b67067d3fee95742838495d7315299c174f4d61a032f4d06aa71eed={{ registry_public_url_with_path }}/cp/wiotp/api-actions:2.1.69 +cp.icr.io/cp/wiotp/api-authentication@sha256:f7e136742d5056d169979779969f53a46a83521b1b1c02f54f5466aaa657ff84={{ registry_public_url_with_path }}/cp/wiotp/api-authentication:3.3.72 +cp.icr.io/cp/wiotp/api-authorization@sha256:ab78d68d7b3a988ea74f51432db103e9e7afd61659c0840b50c92739bcff0130={{ registry_public_url_with_path }}/cp/wiotp/api-authorization:5.3.69 +cp.icr.io/cp/wiotp/api-connectors@sha256:d881ed9439d5d9a9f6ed65a3023099fe6ba380bc6b67172ea229848b7045b317={{ registry_public_url_with_path }}/cp/wiotp/api-connectors:4.1.31 +cp.icr.io/cp/wiotp/api-dashboard@sha256:0438deb2b715c21f7e34323e98664321a83d57481048ce9ae97f43fb36516eed={{ registry_public_url_with_path }}/cp/wiotp/api-dashboard:3.0.114 +cp.icr.io/cp/wiotp/api-devicemgmt@sha256:603f981262b914c7a69bfdf4f63664adb8951889e951e359187f83361dffca8c={{ registry_public_url_with_path }}/cp/wiotp/api-devicemgmt:3.2.62 +cp.icr.io/cp/wiotp/api-devops@sha256:7e850ed53d16e6071b19398f4ca98dbd71444c2e27a8cdd0a6e4fee974153f5f={{ registry_public_url_with_path }}/cp/wiotp/api-devops:3.7.76 +cp.icr.io/cp/wiotp/api-importconnectors@sha256:7f34e81197fc1a4ac8b2beb8fc7ac0ca3e9bca6c63b874b31040f7088f4b97e2={{ registry_public_url_with_path }}/cp/wiotp/api-importconnectors:2.0.62 +cp.icr.io/cp/wiotp/api-mbgadmin@sha256:3b137a48623d91d0ffcee3985fdee7793caa130fd3eb7386e7ca35203bb16c77={{ registry_public_url_with_path }}/cp/wiotp/api-mbgadmin:2.4.107 +cp.icr.io/cp/wiotp/api-messagesight@sha256:26784a0b6a665d2908acb14f9f4d6a632901260c92a54c2eda3fc20b4bdc2f6b={{ registry_public_url_with_path }}/cp/wiotp/api-messagesight:3.1.65 +cp.icr.io/cp/wiotp/api-org@sha256:383aafbbaa37a16c24e6124165c0cabbdaf4ac34e048e26bbf942594ddbcf61a={{ registry_public_url_with_path }}/cp/wiotp/api-org:8.2.53 +cp.icr.io/cp/wiotp/api-organizations@sha256:942ede80f46a3f42e708732321d24976bc419601f849439ca1ae1f882bb08d80={{ registry_public_url_with_path }}/cp/wiotp/api-organizations:10.0.65 +cp.icr.io/cp/wiotp/api-pipeline@sha256:998497b0d1c5852700651ff6108b4f974516996237a50c9392f11828e616a98b={{ registry_public_url_with_path }}/cp/wiotp/api-pipeline:1.1.15 +cp.icr.io/cp/wiotp/api-resourcecontroller@sha256:d01f5ac32696f7b975231097e3a1215869c29b7452b1d39d1643af642f22bdd6={{ registry_public_url_with_path }}/cp/wiotp/api-resourcecontroller:3.0.103 +cp.icr.io/cp/wiotp/api-riskmgmt-secguardian@sha256:72f34118ae172d1e2f18c77e12b35ce04c846cd4d684cf8914d5b3db6b91b55a={{ registry_public_url_with_path }}/cp/wiotp/api-riskmgmt-secguardian:5.1.73 +cp.icr.io/cp/wiotp/api-s2s@sha256:4275b8d8aab9e519aecd8478a8fe3af9417ed70f2a0a43a49609d75cbb375e82={{ registry_public_url_with_path }}/cp/wiotp/api-s2s:10.1.54 +cp.icr.io/cp/wiotp/api-schemas@sha256:bc09bfd8d5acbaf08fa36a1507315c5be0e782c1e97781420a23d5e8c0436ff3={{ registry_public_url_with_path }}/cp/wiotp/api-schemas:4.0.61 +cp.icr.io/cp/wiotp/api-state@sha256:e74bd94d9d9e21f6182483b3d98935c31e382f42d488af5e5eb270c6802219f2={{ registry_public_url_with_path }}/cp/wiotp/api-state:5.1.21 +cp.icr.io/cp/wiotp/api-status@sha256:e1bf748cede2208cf905067bd0a0f99da56985bf76fab8669e672f0e61473808={{ registry_public_url_with_path }}/cp/wiotp/api-status:3.4.37 +cp.icr.io/cp/wiotp/auth-store@sha256:914b6f285ae9af82ff42024ebee330962d5521f441307478a22393ce9e2af2fa={{ registry_public_url_with_path }}/cp/wiotp/auth-store:2.5.12 +cp.icr.io/cp/wiotp/config-store@sha256:aa728cd2c3d86a791136ddd9f9ba48c372ce59154b70978f018f7e6a24025494={{ registry_public_url_with_path }}/cp/wiotp/config-store:4.0.73 +cp.icr.io/cp/wiotp/connectors-store@sha256:f27990e2095ffa3c0de7a5babb896e85152ece831d19b4ee077337a9b6a43324={{ registry_public_url_with_path }}/cp/wiotp/connectors-store:8.0.60 +cp.icr.io/cp/wiotp/data-import-connector@sha256:487879c5ca803c8dea56bace82e722f422ef661affccf17e678fb0dac776acfe={{ registry_public_url_with_path }}/cp/wiotp/data-import-connector:2.1.25 +cp.icr.io/cp/wiotp/datapower-config@sha256:c92418d8c3442fe0832cd257b5a1be2670f88c378df33b2f7d2a613b4e446e06={{ registry_public_url_with_path }}/cp/wiotp/datapower-config:10.4.2 +cp.icr.io/cp/wiotp/datapower@sha256:3e3407c06e399c7d30d2298b9ea1ceed7a7f3a2a06d97ea952e379d454b4f252={{ registry_public_url_with_path }}/cp/wiotp/datapower:7.4.4 +cp.icr.io/cp/wiotp/deprovagent-actions@sha256:08fdd43daeb7f66146e7cecb8bbfed69fb499cb358492b397fc3f9560f56d341={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-actions:1.1.89 +cp.icr.io/cp/wiotp/deprovagent-auth@sha256:d51acf8747d0ec2537fc92a9d4a01ea84583e887debc4de50922811ef9176861={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-auth:2.3.85 +cp.icr.io/cp/wiotp/deprovagent-dm@sha256:4dab0cab69eeaabc7900b656c61587bf46302328d1ba57da1d0ca5e84a3b9e1d={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-dm:2.1.85 +cp.icr.io/cp/wiotp/deprovagent-fpl@sha256:160ef36c62a6fd4ac31c58a2b214084c51b85b9a5bbbe1ae918d65094a8f4856={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-fpl:1.0.38 +cp.icr.io/cp/wiotp/deprovagent-guardian@sha256:63b42d1243938bac7e040ab8bd142b2ab4a5216388b8db789292d1ee0bf45879={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-guardian:3.1.86 +cp.icr.io/cp/wiotp/deprovagent-orgmgmt@sha256:e584c51fcff47b1d0d2fdaea4a01c0d33d64c39959062f745d3f65b9b0343e2f={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-orgmgmt:2.1.87 +cp.icr.io/cp/wiotp/deprovagent-provision@sha256:1e6d835aac567d560ab4044e0fd65f45bd6faa85dfd53d08498a36548744019f={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-provision:4.4.81 +cp.icr.io/cp/wiotp/deprovagent-registry@sha256:a73dfe91c9186f72d679cb58d51630e915cca252cc3bac4cad96e667b0741c4f={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-registry:2.2.81 +cp.icr.io/cp/wiotp/deprovagent-state@sha256:c0106b48a2c803b6ec8d713527893d63e54a1ad465dd55341c443eb5a3356195={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-state:1.2.86 +cp.icr.io/cp/wiotp/deprovagent-webui@sha256:cecfe1ae2db564233796651b129a5274fc6a50d0b2c8349ae6fee15e9670e0e9={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-webui:1.2.88 +cp.icr.io/cp/wiotp/device-store@sha256:a82f3435b5bb30f2d3945e182471033d4b56e536dc06826e5837aff49bd7aa43={{ registry_public_url_with_path }}/cp/wiotp/device-store:2.2.66 +cp.icr.io/cp/wiotp/devicemgmt-server@sha256:4375a6ca7678907aff6eb94d9e16a239d8449bbd1f315fa2969abd957aa206b5={{ registry_public_url_with_path }}/cp/wiotp/devicemgmt-server:3.2.61 +cp.icr.io/cp/wiotp/entity-connector@sha256:155ab5d7be26d1c9c8cd2e7275b0eb43c03abdbd3b9aa6be6156d3c87a055e31={{ registry_public_url_with_path }}/cp/wiotp/entity-connector:1.1.115 +cp.icr.io/cp/wiotp/entity-manager@sha256:70bd1fdc6216c81be4e91e927b70c5b669141a89baee8701a0ea0ad9122da684={{ registry_public_url_with_path }}/cp/wiotp/entity-manager:2.1.122 +cp.icr.io/cp/wiotp/functionsexecutor@sha256:3bedf02cebbde0869ea94f46bc04b16c89bfc2d350f99abe911c30feb7118830={{ registry_public_url_with_path }}/cp/wiotp/functionsexecutor:1.1.28 +cp.icr.io/cp/wiotp/graphite-exporter@sha256:beb713921357d06cb12deb3715eac2b8d75b41bb6fe5e24cc7b4dc46061bc00c={{ registry_public_url_with_path }}/cp/wiotp/graphite-exporter:2.1.111 +cp.icr.io/cp/wiotp/historian-configuration-manager@sha256:4469140c1583e62145f21b040fe7c8f228f8eb5dd441587b3706627746ef4fda={{ registry_public_url_with_path }}/cp/wiotp/historian-configuration-manager:7.1.26 +cp.icr.io/cp/wiotp/historian-connector@sha256:a548da2a8d7a342c13b21862f19b07b9b511c366dfd424a2ebf2a0f9a556d90a={{ registry_public_url_with_path }}/cp/wiotp/historian-connector:8.2.22 +cp.icr.io/cp/wiotp/ibm-iot-actions@sha256:39706dac57a92048e1721d9f47c84834c51ae625bae9914d92ddadca0205d0ae={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-actions:2.5.5 +cp.icr.io/cp/wiotp/ibm-iot-auth@sha256:86121b4d1936030307c820751dac44678c9c50de78ccddfc4a5915f195e7a91e={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-auth:6.4.6 +cp.icr.io/cp/wiotp/ibm-iot-datapower@sha256:960d3448ab649388fff3de772f86c31dc8417751358bdf8c16e05df2368fd7af={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-datapower:7.4.4 +cp.icr.io/cp/wiotp/ibm-iot-devops@sha256:d364349df5f903ae91a318152d2a97ce47db2c1871472c9a1105c4ed7e3a9b32={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-devops:10.3.8 +cp.icr.io/cp/wiotp/ibm-iot-dm@sha256:25e6773484cc0c94fcf9d17108c9555155004834511392b917f0c3ad44ca5e1f={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-dm:2.3.6 +cp.icr.io/cp/wiotp/ibm-iot-dsc@sha256:335b0f0a5b0142184c7d179bcd436cfdd203918066d7f7951d0fc28d0c4222fb={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-dsc:6.3.6 +cp.icr.io/cp/wiotp/ibm-iot-fpl@sha256:6bad878ca0af8dbc39d6394d38b2018430fe07d9954f502c00113fc131e03907={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-fpl:1.1.10 +cp.icr.io/cp/wiotp/ibm-iot-guardian@sha256:5fa64940e086b8d35da2bef5d5ea7edf724a217522e72a7454d345347c277b34={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-guardian:4.3.6 +cp.icr.io/cp/wiotp/ibm-iot-mbgx@sha256:a39c03250a1db315cd806476f920dba083674c0ab14aeedb9cdd6232570c047c={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-mbgx:3.3.10 +cp.icr.io/cp/wiotp/ibm-iot-mfgx@sha256:5c7268ae46a22140b2eecea1d18fc1c5f474833480b478d2a0f3b8d8e7d54e34={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-mfgx:6.3.11 +cp.icr.io/cp/wiotp/ibm-iot-monitor@sha256:cd56208a25fb6a73f0d5728d5b5b3a063e8a2f66ea3805f0a91763e9e6c86a48={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-monitor:2.5.6 +cp.icr.io/cp/wiotp/ibm-iot-orgmgmt@sha256:2233efd33e2eaeb065abec2cdd5dd28503a6a96186417cb28cc293ad5b0ff5e3={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-orgmgmt:6.8.2 +cp.icr.io/cp/wiotp/ibm-iot-provision@sha256:9f43c84d039711aa6c1370cc9da70970bc9a1d40b6d48bf98cdcabdb5f3d34ad={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-provision:10.3.6 +cp.icr.io/cp/wiotp/ibm-iot-registry@sha256:f596037ea35e22046628393132f7995c364436198367354f728572ceed7ed931={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-registry:8.3.5 +cp.icr.io/cp/wiotp/ibm-iot-state@sha256:55a2b9965d53cb76c4b667a437be8de1a977ef9d47d63a650a397506856268cd={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-state:5.3.8 +cp.icr.io/cp/wiotp/ibm-iot-webui@sha256:e69526c637df84daf56198673931c1c56614b873f35316b92bad5fe98b174890={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-webui:6.4.5 +cp.icr.io/cp/wiotp/ibm-iot-workspace@sha256:9510e6e2a07991c5a5bb7394aaecb8948a4bec3898dc7a17e192b136d842a1d7={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-workspace:2.6.6 +cp.icr.io/cp/wiotp/kafkamodel-actions@sha256:c87c7cd46ad137cae4d1e79171b4be2d8d618182e3283bba5f45941042f4a654={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-actions:1.4.28 +cp.icr.io/cp/wiotp/kafkamodel-auth@sha256:431cde1ee9c9cca61b868a9bb525f3b43301087c9ed6d726374e82a5742ef7fa={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-auth:1.7.28 +cp.icr.io/cp/wiotp/kafkamodel-mfgx@sha256:b8e2af076c21878511a9b8c242d4ba76320db3ac1ff3c12f692585c812e4587d={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-mfgx:1.3.30 +cp.icr.io/cp/wiotp/kafkamodel-orgmgmt@sha256:a13b094015d0b7a557e2e4bde60ad4c933c7fb19b02940d2634dfdfc156f2111={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-orgmgmt:2.2.28 +cp.icr.io/cp/wiotp/kafkamodel-registry@sha256:2973afe6531d47aa8cc5ab96a984855994d48999f91390394c7705d8697e90a1={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-registry:1.4.28 +cp.icr.io/cp/wiotp/kafkamodel-state@sha256:6d4e9447b0a2fca8bd80a2b06fc695b2349ecc1b68739857dcd9e3513d2546b7={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-state:1.4.28 +cp.icr.io/cp/wiotp/masuseragent@sha256:82650ce1f193d4a7e54cfeab8f26a612cfaf4e0599dc49f5dcd87f02a1deb644={{ registry_public_url_with_path }}/cp/wiotp/masuseragent:1.4.39 +cp.icr.io/cp/wiotp/messagesight-configurator@sha256:020abc6cd44da995b67c9e03d4085ff61830ba4bc1fb4c0ba9406ce1e77c9c25={{ registry_public_url_with_path }}/cp/wiotp/messagesight-configurator:5.14.53 +cp.icr.io/cp/wiotp/messagesight@sha256:01e55d9f71c5213a652338b99ce3a186c780af6ac453d20248886ab8fcb06e71={{ registry_public_url_with_path }}/cp/wiotp/messagesight:5.14.53 +cp.icr.io/cp/wiotp/monagent-iot@sha256:10dcb43ad3b1601c99233f0862355959765fabc7287b1b25aeb6ee80d06a64c9={{ registry_public_url_with_path }}/cp/wiotp/monagent-iot:2.0.4 +cp.icr.io/cp/wiotp/monagent-msserver@sha256:00a865afd7cde57f23ebbc311193112d5a2b7630ab0aa451b01a7699ee0ae73b={{ registry_public_url_with_path }}/cp/wiotp/monagent-msserver:4.3.102 +cp.icr.io/cp/wiotp/monagent-org@sha256:b07f5cb43f288f67e2fdf311815542989377408295474854c06443883875a6e8={{ registry_public_url_with_path }}/cp/wiotp/monagent-org:2.1.112 +cp.icr.io/cp/wiotp/mongomodel-actions@sha256:66c7769d5ee4b512e7ba2df994b9f0839ba9faf3ed5a11a24e47851487e1a35e={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-actions:1.2.101 +cp.icr.io/cp/wiotp/mongomodel-dm@sha256:03b98920ec9c89ddfcb77bcd6906739235d3e59f78560a058735057a0847b6f9={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-dm:1.3.105 +cp.icr.io/cp/wiotp/mongomodel-dsc@sha256:e29b85f767b44b1b9e4957ce1c3350d639be46776ec2f9e99570a00db974f528={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-dsc:1.2.104 +cp.icr.io/cp/wiotp/mongomodel-guardian@sha256:30abe64ec3b86a585eb2596cc07aac1dc3ac0be63107841a67ddb2a568948454={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-guardian:1.3.101 +cp.icr.io/cp/wiotp/mongomodel-orgmgmt@sha256:24fab89bd6dc3ad91dc65af3d2cb65d7286f6e526eeddbaa481a4c9747eecf4e={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-orgmgmt:2.0.99 +cp.icr.io/cp/wiotp/mongomodel-provision@sha256:88955bfc4285a084b80eb56cb6144d25deea71d0504df04ce20ffa522274d8c3={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-provision:2.5.90 +cp.icr.io/cp/wiotp/mongomodel-registry@sha256:03fcc9c55758a1ef4ad159c5af57e31d3540e7e49a957844af2dcf9d195604ad={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-registry:2.0.95 +cp.icr.io/cp/wiotp/mongomodel-state@sha256:9bdeaef9131dc2192945db63a368aee4155a53b9e9a941c975b157f2c93e6137={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-state:1.2.97 +cp.icr.io/cp/wiotp/mongomodel-webui@sha256:4f0f6f396bbfbefe4572941f1028d4e3961e4497b3b8cedf3c871008d6d30369={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-webui:2.3.104 +cp.icr.io/cp/wiotp/mqtt-connector@sha256:5e3ccf1909c09999731c2aa9b65439f9b0cfa8bdab59e1c2e67e6ebb1eb22404={{ registry_public_url_with_path }}/cp/wiotp/mqtt-connector:2.0.103 +cp.icr.io/cp/wiotp/msproxy-monitor@sha256:9e2fe23f914bbf904caa92fd8782e170f6fba87635a39c848b3b38c9a6c9673b={{ registry_public_url_with_path }}/cp/wiotp/msproxy-monitor:8.6.31 +cp.icr.io/cp/wiotp/msproxy@sha256:8836dd25392c9e1ac74039046c79be36492aac574bc16aaa1c8b2e498c073a5e={{ registry_public_url_with_path }}/cp/wiotp/msproxy:8.6.31 +cp.icr.io/cp/wiotp/org-store@sha256:d3e58211d9cb8d89f0a5ade4bfd45f1c9590e303febe914f1a9594fae2c05676={{ registry_public_url_with_path }}/cp/wiotp/org-store:4.0.72 +cp.icr.io/cp/wiotp/orgdeprovmgr@sha256:349824703278e4c3a179a1cc86dfaf0c33e655e0cd26f93b26fc1e6f3b86f2ea={{ registry_public_url_with_path }}/cp/wiotp/orgdeprovmgr:2.2.91 +cp.icr.io/cp/wiotp/orgpoolmgr@sha256:52c948f7697ebda09fc530ada7faea6b4d7c6c0525729ba5f21e7593831fec82={{ registry_public_url_with_path }}/cp/wiotp/orgpoolmgr:3.1.108 +cp.icr.io/cp/wiotp/orgprovmgr@sha256:37bbd47ece8a22ad7010d0327d11732753bfc207060d30f5775ed249357d31ae={{ registry_public_url_with_path }}/cp/wiotp/orgprovmgr:2.1.90 +cp.icr.io/cp/wiotp/pipelinerouter@sha256:e340853661440d6406775df1ff8bf7a00d744e3ee82a40139410f4634a40ffeb={{ registry_public_url_with_path }}/cp/wiotp/pipelinerouter:1.1.25 +cp.icr.io/cp/wiotp/provagent-guardian@sha256:f755105203727598a3c24fa50b4245293b32f532ad28861c4b2c2fafd7974b20={{ registry_public_url_with_path }}/cp/wiotp/provagent-guardian:2.1.88 +cp.icr.io/cp/wiotp/s2s-store@sha256:dd3770076413737fd826a2d1db721e69a8ad1553db338774038fe1d82f048dfd={{ registry_public_url_with_path }}/cp/wiotp/s2s-store:6.1.54 +cp.icr.io/cp/wiotp/sqlmodel-auth@sha256:8d3194e0315f1cd8cdeaecc7980ac525a27afee39283aef56f5cd82e5e2d05f2={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-auth:1.1.40 +cp.icr.io/cp/wiotp/sqlmodel-fpl@sha256:045ea5513b39251b55706638039fbe2af4feef23368d5dd72d48584a2cde5d81={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-fpl:1.0.33 +cp.icr.io/cp/wiotp/sqlmodel-state@sha256:09631b932110a0e17d702a143ea57ad82dee6642825138b2cc89ab8ea670f84d={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-state:1.0.85 +cp.icr.io/cp/wiotp/state-updater@sha256:503c371e30b15858b6c70e2e4c766555c367ed6b18e225a6bd391efa7d723e73={{ registry_public_url_with_path }}/cp/wiotp/state-updater:2.2.19 +cp.icr.io/cp/wiotp/statsd@sha256:a03313c0064137668c333d5bf9ab50749947c35981366b931f83855b723df8c3={{ registry_public_url_with_path }}/cp/wiotp/statsd:1.3.99 +cp.icr.io/cp/wiotp/ui-dashboard@sha256:2721067fc2cfc8bf9e041ff731d18bdd616f1ffc2b4aad6dcd8615aa0de19c4c={{ registry_public_url_with_path }}/cp/wiotp/ui-dashboard:7.4.8 +icr.io/cpopen/ibm-mas-iot-operator-bundle@sha256:524ec52f1142db1641ae30b6c2b91b80d6b4ab45534f308c93319a9103b9bcf5={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot-operator-bundle:8.6.7 +icr.io/cpopen/ibm-mas-iot-operator-catalog@sha256:09b6560684c9f651a9c837b080498e46162a334fb02ff5bb7a5b8faf75734e70={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot-operator-catalog:latest +icr.io/cpopen/ibm-mas-iot@sha256:eea161ace585c004d0442439f0acac7e3af1aea78478bb8ee5ded41049e27839={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot:8.6.7 \ No newline at end of file diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/iot867fix/from-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/iot867fix/from-filesystem.txt.j2 index d1026cc6b0..ddddb46e58 100644 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/iot867fix/from-filesystem.txt.j2 +++ b/ibm/mas_devops/roles/mirror_case_prepare/templates/iot867fix/from-filesystem.txt.j2 @@ -1,102 +1,102 @@ -file:///cp/mas/ltpakeys-generator@sha256:230d68dcb26a5890efafa591243b6450a42ef695514be6cb60a9eface269499b={{ registry_public_url }}/cp/mas/ltpakeys-generator:2.6.64 -file:///cp/wiotp/action-mgr@sha256:b7a8cabcf40b8e72863647b37024b5d21a8ec85f6b7143f3ce73a3553e31948b={{ registry_public_url }}/cp/wiotp/action-mgr:2.1.19 -file:///cp/wiotp/api-actions@sha256:19c859464b67067d3fee95742838495d7315299c174f4d61a032f4d06aa71eed={{ registry_public_url }}/cp/wiotp/api-actions:2.1.69 -file:///cp/wiotp/api-authentication@sha256:f7e136742d5056d169979779969f53a46a83521b1b1c02f54f5466aaa657ff84={{ registry_public_url }}/cp/wiotp/api-authentication:3.3.72 -file:///cp/wiotp/api-authorization@sha256:ab78d68d7b3a988ea74f51432db103e9e7afd61659c0840b50c92739bcff0130={{ registry_public_url }}/cp/wiotp/api-authorization:5.3.69 -file:///cp/wiotp/api-connectors@sha256:d881ed9439d5d9a9f6ed65a3023099fe6ba380bc6b67172ea229848b7045b317={{ registry_public_url }}/cp/wiotp/api-connectors:4.1.31 -file:///cp/wiotp/api-dashboard@sha256:0438deb2b715c21f7e34323e98664321a83d57481048ce9ae97f43fb36516eed={{ registry_public_url }}/cp/wiotp/api-dashboard:3.0.114 -file:///cp/wiotp/api-devicemgmt@sha256:603f981262b914c7a69bfdf4f63664adb8951889e951e359187f83361dffca8c={{ registry_public_url }}/cp/wiotp/api-devicemgmt:3.2.62 -file:///cp/wiotp/api-devops@sha256:7e850ed53d16e6071b19398f4ca98dbd71444c2e27a8cdd0a6e4fee974153f5f={{ registry_public_url }}/cp/wiotp/api-devops:3.7.76 -file:///cp/wiotp/api-importconnectors@sha256:7f34e81197fc1a4ac8b2beb8fc7ac0ca3e9bca6c63b874b31040f7088f4b97e2={{ registry_public_url }}/cp/wiotp/api-importconnectors:2.0.62 -file:///cp/wiotp/api-mbgadmin@sha256:3b137a48623d91d0ffcee3985fdee7793caa130fd3eb7386e7ca35203bb16c77={{ registry_public_url }}/cp/wiotp/api-mbgadmin:2.4.107 -file:///cp/wiotp/api-messagesight@sha256:26784a0b6a665d2908acb14f9f4d6a632901260c92a54c2eda3fc20b4bdc2f6b={{ registry_public_url }}/cp/wiotp/api-messagesight:3.1.65 -file:///cp/wiotp/api-org@sha256:383aafbbaa37a16c24e6124165c0cabbdaf4ac34e048e26bbf942594ddbcf61a={{ registry_public_url }}/cp/wiotp/api-org:8.2.53 -file:///cp/wiotp/api-organizations@sha256:942ede80f46a3f42e708732321d24976bc419601f849439ca1ae1f882bb08d80={{ registry_public_url }}/cp/wiotp/api-organizations:10.0.65 -file:///cp/wiotp/api-pipeline@sha256:998497b0d1c5852700651ff6108b4f974516996237a50c9392f11828e616a98b={{ registry_public_url }}/cp/wiotp/api-pipeline:1.1.15 -file:///cp/wiotp/api-resourcecontroller@sha256:d01f5ac32696f7b975231097e3a1215869c29b7452b1d39d1643af642f22bdd6={{ registry_public_url }}/cp/wiotp/api-resourcecontroller:3.0.103 -file:///cp/wiotp/api-riskmgmt-secguardian@sha256:72f34118ae172d1e2f18c77e12b35ce04c846cd4d684cf8914d5b3db6b91b55a={{ registry_public_url }}/cp/wiotp/api-riskmgmt-secguardian:5.1.73 -file:///cp/wiotp/api-s2s@sha256:4275b8d8aab9e519aecd8478a8fe3af9417ed70f2a0a43a49609d75cbb375e82={{ registry_public_url }}/cp/wiotp/api-s2s:10.1.54 -file:///cp/wiotp/api-schemas@sha256:bc09bfd8d5acbaf08fa36a1507315c5be0e782c1e97781420a23d5e8c0436ff3={{ registry_public_url }}/cp/wiotp/api-schemas:4.0.61 -file:///cp/wiotp/api-state@sha256:e74bd94d9d9e21f6182483b3d98935c31e382f42d488af5e5eb270c6802219f2={{ registry_public_url }}/cp/wiotp/api-state:5.1.21 -file:///cp/wiotp/api-status@sha256:e1bf748cede2208cf905067bd0a0f99da56985bf76fab8669e672f0e61473808={{ registry_public_url }}/cp/wiotp/api-status:3.4.37 -file:///cp/wiotp/auth-store@sha256:914b6f285ae9af82ff42024ebee330962d5521f441307478a22393ce9e2af2fa={{ registry_public_url }}/cp/wiotp/auth-store:2.5.12 -file:///cp/wiotp/config-store@sha256:aa728cd2c3d86a791136ddd9f9ba48c372ce59154b70978f018f7e6a24025494={{ registry_public_url }}/cp/wiotp/config-store:4.0.73 -file:///cp/wiotp/connectors-store@sha256:f27990e2095ffa3c0de7a5babb896e85152ece831d19b4ee077337a9b6a43324={{ registry_public_url }}/cp/wiotp/connectors-store:8.0.60 -file:///cp/wiotp/data-import-connector@sha256:487879c5ca803c8dea56bace82e722f422ef661affccf17e678fb0dac776acfe={{ registry_public_url }}/cp/wiotp/data-import-connector:2.1.25 -file:///cp/wiotp/datapower-config@sha256:c92418d8c3442fe0832cd257b5a1be2670f88c378df33b2f7d2a613b4e446e06={{ registry_public_url }}/cp/wiotp/datapower-config:10.4.2 -file:///cp/wiotp/datapower@sha256:3e3407c06e399c7d30d2298b9ea1ceed7a7f3a2a06d97ea952e379d454b4f252={{ registry_public_url }}/cp/wiotp/datapower:7.4.4 -file:///cp/wiotp/deprovagent-actions@sha256:08fdd43daeb7f66146e7cecb8bbfed69fb499cb358492b397fc3f9560f56d341={{ registry_public_url }}/cp/wiotp/deprovagent-actions:1.1.89 -file:///cp/wiotp/deprovagent-auth@sha256:d51acf8747d0ec2537fc92a9d4a01ea84583e887debc4de50922811ef9176861={{ registry_public_url }}/cp/wiotp/deprovagent-auth:2.3.85 -file:///cp/wiotp/deprovagent-dm@sha256:4dab0cab69eeaabc7900b656c61587bf46302328d1ba57da1d0ca5e84a3b9e1d={{ registry_public_url }}/cp/wiotp/deprovagent-dm:2.1.85 -file:///cp/wiotp/deprovagent-fpl@sha256:160ef36c62a6fd4ac31c58a2b214084c51b85b9a5bbbe1ae918d65094a8f4856={{ registry_public_url }}/cp/wiotp/deprovagent-fpl:1.0.38 -file:///cp/wiotp/deprovagent-guardian@sha256:63b42d1243938bac7e040ab8bd142b2ab4a5216388b8db789292d1ee0bf45879={{ registry_public_url }}/cp/wiotp/deprovagent-guardian:3.1.86 -file:///cp/wiotp/deprovagent-orgmgmt@sha256:e584c51fcff47b1d0d2fdaea4a01c0d33d64c39959062f745d3f65b9b0343e2f={{ registry_public_url }}/cp/wiotp/deprovagent-orgmgmt:2.1.87 -file:///cp/wiotp/deprovagent-provision@sha256:1e6d835aac567d560ab4044e0fd65f45bd6faa85dfd53d08498a36548744019f={{ registry_public_url }}/cp/wiotp/deprovagent-provision:4.4.81 -file:///cp/wiotp/deprovagent-registry@sha256:a73dfe91c9186f72d679cb58d51630e915cca252cc3bac4cad96e667b0741c4f={{ registry_public_url }}/cp/wiotp/deprovagent-registry:2.2.81 -file:///cp/wiotp/deprovagent-state@sha256:c0106b48a2c803b6ec8d713527893d63e54a1ad465dd55341c443eb5a3356195={{ registry_public_url }}/cp/wiotp/deprovagent-state:1.2.86 -file:///cp/wiotp/deprovagent-webui@sha256:cecfe1ae2db564233796651b129a5274fc6a50d0b2c8349ae6fee15e9670e0e9={{ registry_public_url }}/cp/wiotp/deprovagent-webui:1.2.88 -file:///cp/wiotp/device-store@sha256:a82f3435b5bb30f2d3945e182471033d4b56e536dc06826e5837aff49bd7aa43={{ registry_public_url }}/cp/wiotp/device-store:2.2.66 -file:///cp/wiotp/devicemgmt-server@sha256:4375a6ca7678907aff6eb94d9e16a239d8449bbd1f315fa2969abd957aa206b5={{ registry_public_url }}/cp/wiotp/devicemgmt-server:3.2.61 -file:///cp/wiotp/entity-connector@sha256:155ab5d7be26d1c9c8cd2e7275b0eb43c03abdbd3b9aa6be6156d3c87a055e31={{ registry_public_url }}/cp/wiotp/entity-connector:1.1.115 -file:///cp/wiotp/entity-manager@sha256:70bd1fdc6216c81be4e91e927b70c5b669141a89baee8701a0ea0ad9122da684={{ registry_public_url }}/cp/wiotp/entity-manager:2.1.122 -file:///cp/wiotp/functionsexecutor@sha256:3bedf02cebbde0869ea94f46bc04b16c89bfc2d350f99abe911c30feb7118830={{ registry_public_url }}/cp/wiotp/functionsexecutor:1.1.28 -file:///cp/wiotp/graphite-exporter@sha256:beb713921357d06cb12deb3715eac2b8d75b41bb6fe5e24cc7b4dc46061bc00c={{ registry_public_url }}/cp/wiotp/graphite-exporter:2.1.111 -file:///cp/wiotp/historian-configuration-manager@sha256:4469140c1583e62145f21b040fe7c8f228f8eb5dd441587b3706627746ef4fda={{ registry_public_url }}/cp/wiotp/historian-configuration-manager:7.1.26 -file:///cp/wiotp/historian-connector@sha256:a548da2a8d7a342c13b21862f19b07b9b511c366dfd424a2ebf2a0f9a556d90a={{ registry_public_url }}/cp/wiotp/historian-connector:8.2.22 -file:///cp/wiotp/ibm-iot-actions@sha256:39706dac57a92048e1721d9f47c84834c51ae625bae9914d92ddadca0205d0ae={{ registry_public_url }}/cp/wiotp/ibm-iot-actions:2.5.5 -file:///cp/wiotp/ibm-iot-auth@sha256:86121b4d1936030307c820751dac44678c9c50de78ccddfc4a5915f195e7a91e={{ registry_public_url }}/cp/wiotp/ibm-iot-auth:6.4.6 -file:///cp/wiotp/ibm-iot-datapower@sha256:960d3448ab649388fff3de772f86c31dc8417751358bdf8c16e05df2368fd7af={{ registry_public_url }}/cp/wiotp/ibm-iot-datapower:7.4.4 -file:///cp/wiotp/ibm-iot-devops@sha256:d364349df5f903ae91a318152d2a97ce47db2c1871472c9a1105c4ed7e3a9b32={{ registry_public_url }}/cp/wiotp/ibm-iot-devops:10.3.8 -file:///cp/wiotp/ibm-iot-dm@sha256:25e6773484cc0c94fcf9d17108c9555155004834511392b917f0c3ad44ca5e1f={{ registry_public_url }}/cp/wiotp/ibm-iot-dm:2.3.6 -file:///cp/wiotp/ibm-iot-dsc@sha256:335b0f0a5b0142184c7d179bcd436cfdd203918066d7f7951d0fc28d0c4222fb={{ registry_public_url }}/cp/wiotp/ibm-iot-dsc:6.3.6 -file:///cp/wiotp/ibm-iot-fpl@sha256:6bad878ca0af8dbc39d6394d38b2018430fe07d9954f502c00113fc131e03907={{ registry_public_url }}/cp/wiotp/ibm-iot-fpl:1.1.10 -file:///cp/wiotp/ibm-iot-guardian@sha256:5fa64940e086b8d35da2bef5d5ea7edf724a217522e72a7454d345347c277b34={{ registry_public_url }}/cp/wiotp/ibm-iot-guardian:4.3.6 -file:///cp/wiotp/ibm-iot-mbgx@sha256:a39c03250a1db315cd806476f920dba083674c0ab14aeedb9cdd6232570c047c={{ registry_public_url }}/cp/wiotp/ibm-iot-mbgx:3.3.10 -file:///cp/wiotp/ibm-iot-mfgx@sha256:5c7268ae46a22140b2eecea1d18fc1c5f474833480b478d2a0f3b8d8e7d54e34={{ registry_public_url }}/cp/wiotp/ibm-iot-mfgx:6.3.11 -file:///cp/wiotp/ibm-iot-monitor@sha256:cd56208a25fb6a73f0d5728d5b5b3a063e8a2f66ea3805f0a91763e9e6c86a48={{ registry_public_url }}/cp/wiotp/ibm-iot-monitor:2.5.6 -file:///cp/wiotp/ibm-iot-orgmgmt@sha256:2233efd33e2eaeb065abec2cdd5dd28503a6a96186417cb28cc293ad5b0ff5e3={{ registry_public_url }}/cp/wiotp/ibm-iot-orgmgmt:6.8.2 -file:///cp/wiotp/ibm-iot-provision@sha256:9f43c84d039711aa6c1370cc9da70970bc9a1d40b6d48bf98cdcabdb5f3d34ad={{ registry_public_url }}/cp/wiotp/ibm-iot-provision:10.3.6 -file:///cp/wiotp/ibm-iot-registry@sha256:f596037ea35e22046628393132f7995c364436198367354f728572ceed7ed931={{ registry_public_url }}/cp/wiotp/ibm-iot-registry:8.3.5 -file:///cp/wiotp/ibm-iot-state@sha256:55a2b9965d53cb76c4b667a437be8de1a977ef9d47d63a650a397506856268cd={{ registry_public_url }}/cp/wiotp/ibm-iot-state:5.3.8 -file:///cp/wiotp/ibm-iot-webui@sha256:e69526c637df84daf56198673931c1c56614b873f35316b92bad5fe98b174890={{ registry_public_url }}/cp/wiotp/ibm-iot-webui:6.4.5 -file:///cp/wiotp/ibm-iot-workspace@sha256:9510e6e2a07991c5a5bb7394aaecb8948a4bec3898dc7a17e192b136d842a1d7={{ registry_public_url }}/cp/wiotp/ibm-iot-workspace:2.6.6 -file:///cp/wiotp/kafkamodel-actions@sha256:c87c7cd46ad137cae4d1e79171b4be2d8d618182e3283bba5f45941042f4a654={{ registry_public_url }}/cp/wiotp/kafkamodel-actions:1.4.28 -file:///cp/wiotp/kafkamodel-auth@sha256:431cde1ee9c9cca61b868a9bb525f3b43301087c9ed6d726374e82a5742ef7fa={{ registry_public_url }}/cp/wiotp/kafkamodel-auth:1.7.28 -file:///cp/wiotp/kafkamodel-mfgx@sha256:b8e2af076c21878511a9b8c242d4ba76320db3ac1ff3c12f692585c812e4587d={{ registry_public_url }}/cp/wiotp/kafkamodel-mfgx:1.3.30 -file:///cp/wiotp/kafkamodel-orgmgmt@sha256:a13b094015d0b7a557e2e4bde60ad4c933c7fb19b02940d2634dfdfc156f2111={{ registry_public_url }}/cp/wiotp/kafkamodel-orgmgmt:2.2.28 -file:///cp/wiotp/kafkamodel-registry@sha256:2973afe6531d47aa8cc5ab96a984855994d48999f91390394c7705d8697e90a1={{ registry_public_url }}/cp/wiotp/kafkamodel-registry:1.4.28 -file:///cp/wiotp/kafkamodel-state@sha256:6d4e9447b0a2fca8bd80a2b06fc695b2349ecc1b68739857dcd9e3513d2546b7={{ registry_public_url }}/cp/wiotp/kafkamodel-state:1.4.28 -file:///cp/wiotp/masuseragent@sha256:82650ce1f193d4a7e54cfeab8f26a612cfaf4e0599dc49f5dcd87f02a1deb644={{ registry_public_url }}/cp/wiotp/masuseragent:1.4.39 -file:///cp/wiotp/messagesight-configurator@sha256:020abc6cd44da995b67c9e03d4085ff61830ba4bc1fb4c0ba9406ce1e77c9c25={{ registry_public_url }}/cp/wiotp/messagesight-configurator:5.14.53 -file:///cp/wiotp/messagesight@sha256:01e55d9f71c5213a652338b99ce3a186c780af6ac453d20248886ab8fcb06e71={{ registry_public_url }}/cp/wiotp/messagesight:5.14.53 -file:///cp/wiotp/monagent-iot@sha256:10dcb43ad3b1601c99233f0862355959765fabc7287b1b25aeb6ee80d06a64c9={{ registry_public_url }}/cp/wiotp/monagent-iot:2.0.4 -file:///cp/wiotp/monagent-msserver@sha256:00a865afd7cde57f23ebbc311193112d5a2b7630ab0aa451b01a7699ee0ae73b={{ registry_public_url }}/cp/wiotp/monagent-msserver:4.3.102 -file:///cp/wiotp/monagent-org@sha256:b07f5cb43f288f67e2fdf311815542989377408295474854c06443883875a6e8={{ registry_public_url }}/cp/wiotp/monagent-org:2.1.112 -file:///cp/wiotp/mongomodel-actions@sha256:66c7769d5ee4b512e7ba2df994b9f0839ba9faf3ed5a11a24e47851487e1a35e={{ registry_public_url }}/cp/wiotp/mongomodel-actions:1.2.101 -file:///cp/wiotp/mongomodel-dm@sha256:03b98920ec9c89ddfcb77bcd6906739235d3e59f78560a058735057a0847b6f9={{ registry_public_url }}/cp/wiotp/mongomodel-dm:1.3.105 -file:///cp/wiotp/mongomodel-dsc@sha256:e29b85f767b44b1b9e4957ce1c3350d639be46776ec2f9e99570a00db974f528={{ registry_public_url }}/cp/wiotp/mongomodel-dsc:1.2.104 -file:///cp/wiotp/mongomodel-guardian@sha256:30abe64ec3b86a585eb2596cc07aac1dc3ac0be63107841a67ddb2a568948454={{ registry_public_url }}/cp/wiotp/mongomodel-guardian:1.3.101 -file:///cp/wiotp/mongomodel-orgmgmt@sha256:24fab89bd6dc3ad91dc65af3d2cb65d7286f6e526eeddbaa481a4c9747eecf4e={{ registry_public_url }}/cp/wiotp/mongomodel-orgmgmt:2.0.99 -file:///cp/wiotp/mongomodel-provision@sha256:88955bfc4285a084b80eb56cb6144d25deea71d0504df04ce20ffa522274d8c3={{ registry_public_url }}/cp/wiotp/mongomodel-provision:2.5.90 -file:///cp/wiotp/mongomodel-registry@sha256:03fcc9c55758a1ef4ad159c5af57e31d3540e7e49a957844af2dcf9d195604ad={{ registry_public_url }}/cp/wiotp/mongomodel-registry:2.0.95 -file:///cp/wiotp/mongomodel-state@sha256:9bdeaef9131dc2192945db63a368aee4155a53b9e9a941c975b157f2c93e6137={{ registry_public_url }}/cp/wiotp/mongomodel-state:1.2.97 -file:///cp/wiotp/mongomodel-webui@sha256:4f0f6f396bbfbefe4572941f1028d4e3961e4497b3b8cedf3c871008d6d30369={{ registry_public_url }}/cp/wiotp/mongomodel-webui:2.3.104 -file:///cp/wiotp/mqtt-connector@sha256:5e3ccf1909c09999731c2aa9b65439f9b0cfa8bdab59e1c2e67e6ebb1eb22404={{ registry_public_url }}/cp/wiotp/mqtt-connector:2.0.103 -file:///cp/wiotp/msproxy-monitor@sha256:9e2fe23f914bbf904caa92fd8782e170f6fba87635a39c848b3b38c9a6c9673b={{ registry_public_url }}/cp/wiotp/msproxy-monitor:8.6.31 -file:///cp/wiotp/msproxy@sha256:8836dd25392c9e1ac74039046c79be36492aac574bc16aaa1c8b2e498c073a5e={{ registry_public_url }}/cp/wiotp/msproxy:8.6.31 -file:///cp/wiotp/org-store@sha256:d3e58211d9cb8d89f0a5ade4bfd45f1c9590e303febe914f1a9594fae2c05676={{ registry_public_url }}/cp/wiotp/org-store:4.0.72 -file:///cp/wiotp/orgdeprovmgr@sha256:349824703278e4c3a179a1cc86dfaf0c33e655e0cd26f93b26fc1e6f3b86f2ea={{ registry_public_url }}/cp/wiotp/orgdeprovmgr:2.2.91 -file:///cp/wiotp/orgpoolmgr@sha256:52c948f7697ebda09fc530ada7faea6b4d7c6c0525729ba5f21e7593831fec82={{ registry_public_url }}/cp/wiotp/orgpoolmgr:3.1.108 -file:///cp/wiotp/orgprovmgr@sha256:37bbd47ece8a22ad7010d0327d11732753bfc207060d30f5775ed249357d31ae={{ registry_public_url }}/cp/wiotp/orgprovmgr:2.1.90 -file:///cp/wiotp/pipelinerouter@sha256:e340853661440d6406775df1ff8bf7a00d744e3ee82a40139410f4634a40ffeb={{ registry_public_url }}/cp/wiotp/pipelinerouter:1.1.25 -file:///cp/wiotp/provagent-guardian@sha256:f755105203727598a3c24fa50b4245293b32f532ad28861c4b2c2fafd7974b20={{ registry_public_url }}/cp/wiotp/provagent-guardian:2.1.88 -file:///cp/wiotp/s2s-store@sha256:dd3770076413737fd826a2d1db721e69a8ad1553db338774038fe1d82f048dfd={{ registry_public_url }}/cp/wiotp/s2s-store:6.1.54 -file:///cp/wiotp/sqlmodel-auth@sha256:8d3194e0315f1cd8cdeaecc7980ac525a27afee39283aef56f5cd82e5e2d05f2={{ registry_public_url }}/cp/wiotp/sqlmodel-auth:1.1.40 -file:///cp/wiotp/sqlmodel-fpl@sha256:045ea5513b39251b55706638039fbe2af4feef23368d5dd72d48584a2cde5d81={{ registry_public_url }}/cp/wiotp/sqlmodel-fpl:1.0.33 -file:///cp/wiotp/sqlmodel-state@sha256:09631b932110a0e17d702a143ea57ad82dee6642825138b2cc89ab8ea670f84d={{ registry_public_url }}/cp/wiotp/sqlmodel-state:1.0.85 -file:///cp/wiotp/state-updater@sha256:503c371e30b15858b6c70e2e4c766555c367ed6b18e225a6bd391efa7d723e73={{ registry_public_url }}/cp/wiotp/state-updater:2.2.19 -file:///cp/wiotp/statsd@sha256:a03313c0064137668c333d5bf9ab50749947c35981366b931f83855b723df8c3={{ registry_public_url }}/cp/wiotp/statsd:1.3.99 -file:///cp/wiotp/ui-dashboard@sha256:2721067fc2cfc8bf9e041ff731d18bdd616f1ffc2b4aad6dcd8615aa0de19c4c={{ registry_public_url }}/cp/wiotp/ui-dashboard:7.4.8 -file:///cpopen/ibm-mas-iot-operator-bundle@sha256:524ec52f1142db1641ae30b6c2b91b80d6b4ab45534f308c93319a9103b9bcf5={{ registry_public_url }}/cpopen/ibm-mas-iot-operator-bundle:8.6.7 -file:///cpopen/ibm-mas-iot-operator-catalog@sha256:09b6560684c9f651a9c837b080498e46162a334fb02ff5bb7a5b8faf75734e70={{ registry_public_url }}/cpopen/ibm-mas-iot-operator-catalog:latest -file:///cpopen/ibm-mas-iot@sha256:eea161ace585c004d0442439f0acac7e3af1aea78478bb8ee5ded41049e27839={{ registry_public_url }}/cpopen/ibm-mas-iot:8.6.7 \ No newline at end of file +file:///cp/mas/ltpakeys-generator@sha256:230d68dcb26a5890efafa591243b6450a42ef695514be6cb60a9eface269499b={{ registry_public_url_with_path }}/cp/mas/ltpakeys-generator:2.6.64 +file:///cp/wiotp/action-mgr@sha256:b7a8cabcf40b8e72863647b37024b5d21a8ec85f6b7143f3ce73a3553e31948b={{ registry_public_url_with_path }}/cp/wiotp/action-mgr:2.1.19 +file:///cp/wiotp/api-actions@sha256:19c859464b67067d3fee95742838495d7315299c174f4d61a032f4d06aa71eed={{ registry_public_url_with_path }}/cp/wiotp/api-actions:2.1.69 +file:///cp/wiotp/api-authentication@sha256:f7e136742d5056d169979779969f53a46a83521b1b1c02f54f5466aaa657ff84={{ registry_public_url_with_path }}/cp/wiotp/api-authentication:3.3.72 +file:///cp/wiotp/api-authorization@sha256:ab78d68d7b3a988ea74f51432db103e9e7afd61659c0840b50c92739bcff0130={{ registry_public_url_with_path }}/cp/wiotp/api-authorization:5.3.69 +file:///cp/wiotp/api-connectors@sha256:d881ed9439d5d9a9f6ed65a3023099fe6ba380bc6b67172ea229848b7045b317={{ registry_public_url_with_path }}/cp/wiotp/api-connectors:4.1.31 +file:///cp/wiotp/api-dashboard@sha256:0438deb2b715c21f7e34323e98664321a83d57481048ce9ae97f43fb36516eed={{ registry_public_url_with_path }}/cp/wiotp/api-dashboard:3.0.114 +file:///cp/wiotp/api-devicemgmt@sha256:603f981262b914c7a69bfdf4f63664adb8951889e951e359187f83361dffca8c={{ registry_public_url_with_path }}/cp/wiotp/api-devicemgmt:3.2.62 +file:///cp/wiotp/api-devops@sha256:7e850ed53d16e6071b19398f4ca98dbd71444c2e27a8cdd0a6e4fee974153f5f={{ registry_public_url_with_path }}/cp/wiotp/api-devops:3.7.76 +file:///cp/wiotp/api-importconnectors@sha256:7f34e81197fc1a4ac8b2beb8fc7ac0ca3e9bca6c63b874b31040f7088f4b97e2={{ registry_public_url_with_path }}/cp/wiotp/api-importconnectors:2.0.62 +file:///cp/wiotp/api-mbgadmin@sha256:3b137a48623d91d0ffcee3985fdee7793caa130fd3eb7386e7ca35203bb16c77={{ registry_public_url_with_path }}/cp/wiotp/api-mbgadmin:2.4.107 +file:///cp/wiotp/api-messagesight@sha256:26784a0b6a665d2908acb14f9f4d6a632901260c92a54c2eda3fc20b4bdc2f6b={{ registry_public_url_with_path }}/cp/wiotp/api-messagesight:3.1.65 +file:///cp/wiotp/api-org@sha256:383aafbbaa37a16c24e6124165c0cabbdaf4ac34e048e26bbf942594ddbcf61a={{ registry_public_url_with_path }}/cp/wiotp/api-org:8.2.53 +file:///cp/wiotp/api-organizations@sha256:942ede80f46a3f42e708732321d24976bc419601f849439ca1ae1f882bb08d80={{ registry_public_url_with_path }}/cp/wiotp/api-organizations:10.0.65 +file:///cp/wiotp/api-pipeline@sha256:998497b0d1c5852700651ff6108b4f974516996237a50c9392f11828e616a98b={{ registry_public_url_with_path }}/cp/wiotp/api-pipeline:1.1.15 +file:///cp/wiotp/api-resourcecontroller@sha256:d01f5ac32696f7b975231097e3a1215869c29b7452b1d39d1643af642f22bdd6={{ registry_public_url_with_path }}/cp/wiotp/api-resourcecontroller:3.0.103 +file:///cp/wiotp/api-riskmgmt-secguardian@sha256:72f34118ae172d1e2f18c77e12b35ce04c846cd4d684cf8914d5b3db6b91b55a={{ registry_public_url_with_path }}/cp/wiotp/api-riskmgmt-secguardian:5.1.73 +file:///cp/wiotp/api-s2s@sha256:4275b8d8aab9e519aecd8478a8fe3af9417ed70f2a0a43a49609d75cbb375e82={{ registry_public_url_with_path }}/cp/wiotp/api-s2s:10.1.54 +file:///cp/wiotp/api-schemas@sha256:bc09bfd8d5acbaf08fa36a1507315c5be0e782c1e97781420a23d5e8c0436ff3={{ registry_public_url_with_path }}/cp/wiotp/api-schemas:4.0.61 +file:///cp/wiotp/api-state@sha256:e74bd94d9d9e21f6182483b3d98935c31e382f42d488af5e5eb270c6802219f2={{ registry_public_url_with_path }}/cp/wiotp/api-state:5.1.21 +file:///cp/wiotp/api-status@sha256:e1bf748cede2208cf905067bd0a0f99da56985bf76fab8669e672f0e61473808={{ registry_public_url_with_path }}/cp/wiotp/api-status:3.4.37 +file:///cp/wiotp/auth-store@sha256:914b6f285ae9af82ff42024ebee330962d5521f441307478a22393ce9e2af2fa={{ registry_public_url_with_path }}/cp/wiotp/auth-store:2.5.12 +file:///cp/wiotp/config-store@sha256:aa728cd2c3d86a791136ddd9f9ba48c372ce59154b70978f018f7e6a24025494={{ registry_public_url_with_path }}/cp/wiotp/config-store:4.0.73 +file:///cp/wiotp/connectors-store@sha256:f27990e2095ffa3c0de7a5babb896e85152ece831d19b4ee077337a9b6a43324={{ registry_public_url_with_path }}/cp/wiotp/connectors-store:8.0.60 +file:///cp/wiotp/data-import-connector@sha256:487879c5ca803c8dea56bace82e722f422ef661affccf17e678fb0dac776acfe={{ registry_public_url_with_path }}/cp/wiotp/data-import-connector:2.1.25 +file:///cp/wiotp/datapower-config@sha256:c92418d8c3442fe0832cd257b5a1be2670f88c378df33b2f7d2a613b4e446e06={{ registry_public_url_with_path }}/cp/wiotp/datapower-config:10.4.2 +file:///cp/wiotp/datapower@sha256:3e3407c06e399c7d30d2298b9ea1ceed7a7f3a2a06d97ea952e379d454b4f252={{ registry_public_url_with_path }}/cp/wiotp/datapower:7.4.4 +file:///cp/wiotp/deprovagent-actions@sha256:08fdd43daeb7f66146e7cecb8bbfed69fb499cb358492b397fc3f9560f56d341={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-actions:1.1.89 +file:///cp/wiotp/deprovagent-auth@sha256:d51acf8747d0ec2537fc92a9d4a01ea84583e887debc4de50922811ef9176861={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-auth:2.3.85 +file:///cp/wiotp/deprovagent-dm@sha256:4dab0cab69eeaabc7900b656c61587bf46302328d1ba57da1d0ca5e84a3b9e1d={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-dm:2.1.85 +file:///cp/wiotp/deprovagent-fpl@sha256:160ef36c62a6fd4ac31c58a2b214084c51b85b9a5bbbe1ae918d65094a8f4856={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-fpl:1.0.38 +file:///cp/wiotp/deprovagent-guardian@sha256:63b42d1243938bac7e040ab8bd142b2ab4a5216388b8db789292d1ee0bf45879={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-guardian:3.1.86 +file:///cp/wiotp/deprovagent-orgmgmt@sha256:e584c51fcff47b1d0d2fdaea4a01c0d33d64c39959062f745d3f65b9b0343e2f={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-orgmgmt:2.1.87 +file:///cp/wiotp/deprovagent-provision@sha256:1e6d835aac567d560ab4044e0fd65f45bd6faa85dfd53d08498a36548744019f={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-provision:4.4.81 +file:///cp/wiotp/deprovagent-registry@sha256:a73dfe91c9186f72d679cb58d51630e915cca252cc3bac4cad96e667b0741c4f={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-registry:2.2.81 +file:///cp/wiotp/deprovagent-state@sha256:c0106b48a2c803b6ec8d713527893d63e54a1ad465dd55341c443eb5a3356195={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-state:1.2.86 +file:///cp/wiotp/deprovagent-webui@sha256:cecfe1ae2db564233796651b129a5274fc6a50d0b2c8349ae6fee15e9670e0e9={{ registry_public_url_with_path }}/cp/wiotp/deprovagent-webui:1.2.88 +file:///cp/wiotp/device-store@sha256:a82f3435b5bb30f2d3945e182471033d4b56e536dc06826e5837aff49bd7aa43={{ registry_public_url_with_path }}/cp/wiotp/device-store:2.2.66 +file:///cp/wiotp/devicemgmt-server@sha256:4375a6ca7678907aff6eb94d9e16a239d8449bbd1f315fa2969abd957aa206b5={{ registry_public_url_with_path }}/cp/wiotp/devicemgmt-server:3.2.61 +file:///cp/wiotp/entity-connector@sha256:155ab5d7be26d1c9c8cd2e7275b0eb43c03abdbd3b9aa6be6156d3c87a055e31={{ registry_public_url_with_path }}/cp/wiotp/entity-connector:1.1.115 +file:///cp/wiotp/entity-manager@sha256:70bd1fdc6216c81be4e91e927b70c5b669141a89baee8701a0ea0ad9122da684={{ registry_public_url_with_path }}/cp/wiotp/entity-manager:2.1.122 +file:///cp/wiotp/functionsexecutor@sha256:3bedf02cebbde0869ea94f46bc04b16c89bfc2d350f99abe911c30feb7118830={{ registry_public_url_with_path }}/cp/wiotp/functionsexecutor:1.1.28 +file:///cp/wiotp/graphite-exporter@sha256:beb713921357d06cb12deb3715eac2b8d75b41bb6fe5e24cc7b4dc46061bc00c={{ registry_public_url_with_path }}/cp/wiotp/graphite-exporter:2.1.111 +file:///cp/wiotp/historian-configuration-manager@sha256:4469140c1583e62145f21b040fe7c8f228f8eb5dd441587b3706627746ef4fda={{ registry_public_url_with_path }}/cp/wiotp/historian-configuration-manager:7.1.26 +file:///cp/wiotp/historian-connector@sha256:a548da2a8d7a342c13b21862f19b07b9b511c366dfd424a2ebf2a0f9a556d90a={{ registry_public_url_with_path }}/cp/wiotp/historian-connector:8.2.22 +file:///cp/wiotp/ibm-iot-actions@sha256:39706dac57a92048e1721d9f47c84834c51ae625bae9914d92ddadca0205d0ae={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-actions:2.5.5 +file:///cp/wiotp/ibm-iot-auth@sha256:86121b4d1936030307c820751dac44678c9c50de78ccddfc4a5915f195e7a91e={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-auth:6.4.6 +file:///cp/wiotp/ibm-iot-datapower@sha256:960d3448ab649388fff3de772f86c31dc8417751358bdf8c16e05df2368fd7af={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-datapower:7.4.4 +file:///cp/wiotp/ibm-iot-devops@sha256:d364349df5f903ae91a318152d2a97ce47db2c1871472c9a1105c4ed7e3a9b32={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-devops:10.3.8 +file:///cp/wiotp/ibm-iot-dm@sha256:25e6773484cc0c94fcf9d17108c9555155004834511392b917f0c3ad44ca5e1f={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-dm:2.3.6 +file:///cp/wiotp/ibm-iot-dsc@sha256:335b0f0a5b0142184c7d179bcd436cfdd203918066d7f7951d0fc28d0c4222fb={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-dsc:6.3.6 +file:///cp/wiotp/ibm-iot-fpl@sha256:6bad878ca0af8dbc39d6394d38b2018430fe07d9954f502c00113fc131e03907={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-fpl:1.1.10 +file:///cp/wiotp/ibm-iot-guardian@sha256:5fa64940e086b8d35da2bef5d5ea7edf724a217522e72a7454d345347c277b34={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-guardian:4.3.6 +file:///cp/wiotp/ibm-iot-mbgx@sha256:a39c03250a1db315cd806476f920dba083674c0ab14aeedb9cdd6232570c047c={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-mbgx:3.3.10 +file:///cp/wiotp/ibm-iot-mfgx@sha256:5c7268ae46a22140b2eecea1d18fc1c5f474833480b478d2a0f3b8d8e7d54e34={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-mfgx:6.3.11 +file:///cp/wiotp/ibm-iot-monitor@sha256:cd56208a25fb6a73f0d5728d5b5b3a063e8a2f66ea3805f0a91763e9e6c86a48={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-monitor:2.5.6 +file:///cp/wiotp/ibm-iot-orgmgmt@sha256:2233efd33e2eaeb065abec2cdd5dd28503a6a96186417cb28cc293ad5b0ff5e3={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-orgmgmt:6.8.2 +file:///cp/wiotp/ibm-iot-provision@sha256:9f43c84d039711aa6c1370cc9da70970bc9a1d40b6d48bf98cdcabdb5f3d34ad={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-provision:10.3.6 +file:///cp/wiotp/ibm-iot-registry@sha256:f596037ea35e22046628393132f7995c364436198367354f728572ceed7ed931={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-registry:8.3.5 +file:///cp/wiotp/ibm-iot-state@sha256:55a2b9965d53cb76c4b667a437be8de1a977ef9d47d63a650a397506856268cd={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-state:5.3.8 +file:///cp/wiotp/ibm-iot-webui@sha256:e69526c637df84daf56198673931c1c56614b873f35316b92bad5fe98b174890={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-webui:6.4.5 +file:///cp/wiotp/ibm-iot-workspace@sha256:9510e6e2a07991c5a5bb7394aaecb8948a4bec3898dc7a17e192b136d842a1d7={{ registry_public_url_with_path }}/cp/wiotp/ibm-iot-workspace:2.6.6 +file:///cp/wiotp/kafkamodel-actions@sha256:c87c7cd46ad137cae4d1e79171b4be2d8d618182e3283bba5f45941042f4a654={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-actions:1.4.28 +file:///cp/wiotp/kafkamodel-auth@sha256:431cde1ee9c9cca61b868a9bb525f3b43301087c9ed6d726374e82a5742ef7fa={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-auth:1.7.28 +file:///cp/wiotp/kafkamodel-mfgx@sha256:b8e2af076c21878511a9b8c242d4ba76320db3ac1ff3c12f692585c812e4587d={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-mfgx:1.3.30 +file:///cp/wiotp/kafkamodel-orgmgmt@sha256:a13b094015d0b7a557e2e4bde60ad4c933c7fb19b02940d2634dfdfc156f2111={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-orgmgmt:2.2.28 +file:///cp/wiotp/kafkamodel-registry@sha256:2973afe6531d47aa8cc5ab96a984855994d48999f91390394c7705d8697e90a1={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-registry:1.4.28 +file:///cp/wiotp/kafkamodel-state@sha256:6d4e9447b0a2fca8bd80a2b06fc695b2349ecc1b68739857dcd9e3513d2546b7={{ registry_public_url_with_path }}/cp/wiotp/kafkamodel-state:1.4.28 +file:///cp/wiotp/masuseragent@sha256:82650ce1f193d4a7e54cfeab8f26a612cfaf4e0599dc49f5dcd87f02a1deb644={{ registry_public_url_with_path }}/cp/wiotp/masuseragent:1.4.39 +file:///cp/wiotp/messagesight-configurator@sha256:020abc6cd44da995b67c9e03d4085ff61830ba4bc1fb4c0ba9406ce1e77c9c25={{ registry_public_url_with_path }}/cp/wiotp/messagesight-configurator:5.14.53 +file:///cp/wiotp/messagesight@sha256:01e55d9f71c5213a652338b99ce3a186c780af6ac453d20248886ab8fcb06e71={{ registry_public_url_with_path }}/cp/wiotp/messagesight:5.14.53 +file:///cp/wiotp/monagent-iot@sha256:10dcb43ad3b1601c99233f0862355959765fabc7287b1b25aeb6ee80d06a64c9={{ registry_public_url_with_path }}/cp/wiotp/monagent-iot:2.0.4 +file:///cp/wiotp/monagent-msserver@sha256:00a865afd7cde57f23ebbc311193112d5a2b7630ab0aa451b01a7699ee0ae73b={{ registry_public_url_with_path }}/cp/wiotp/monagent-msserver:4.3.102 +file:///cp/wiotp/monagent-org@sha256:b07f5cb43f288f67e2fdf311815542989377408295474854c06443883875a6e8={{ registry_public_url_with_path }}/cp/wiotp/monagent-org:2.1.112 +file:///cp/wiotp/mongomodel-actions@sha256:66c7769d5ee4b512e7ba2df994b9f0839ba9faf3ed5a11a24e47851487e1a35e={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-actions:1.2.101 +file:///cp/wiotp/mongomodel-dm@sha256:03b98920ec9c89ddfcb77bcd6906739235d3e59f78560a058735057a0847b6f9={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-dm:1.3.105 +file:///cp/wiotp/mongomodel-dsc@sha256:e29b85f767b44b1b9e4957ce1c3350d639be46776ec2f9e99570a00db974f528={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-dsc:1.2.104 +file:///cp/wiotp/mongomodel-guardian@sha256:30abe64ec3b86a585eb2596cc07aac1dc3ac0be63107841a67ddb2a568948454={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-guardian:1.3.101 +file:///cp/wiotp/mongomodel-orgmgmt@sha256:24fab89bd6dc3ad91dc65af3d2cb65d7286f6e526eeddbaa481a4c9747eecf4e={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-orgmgmt:2.0.99 +file:///cp/wiotp/mongomodel-provision@sha256:88955bfc4285a084b80eb56cb6144d25deea71d0504df04ce20ffa522274d8c3={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-provision:2.5.90 +file:///cp/wiotp/mongomodel-registry@sha256:03fcc9c55758a1ef4ad159c5af57e31d3540e7e49a957844af2dcf9d195604ad={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-registry:2.0.95 +file:///cp/wiotp/mongomodel-state@sha256:9bdeaef9131dc2192945db63a368aee4155a53b9e9a941c975b157f2c93e6137={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-state:1.2.97 +file:///cp/wiotp/mongomodel-webui@sha256:4f0f6f396bbfbefe4572941f1028d4e3961e4497b3b8cedf3c871008d6d30369={{ registry_public_url_with_path }}/cp/wiotp/mongomodel-webui:2.3.104 +file:///cp/wiotp/mqtt-connector@sha256:5e3ccf1909c09999731c2aa9b65439f9b0cfa8bdab59e1c2e67e6ebb1eb22404={{ registry_public_url_with_path }}/cp/wiotp/mqtt-connector:2.0.103 +file:///cp/wiotp/msproxy-monitor@sha256:9e2fe23f914bbf904caa92fd8782e170f6fba87635a39c848b3b38c9a6c9673b={{ registry_public_url_with_path }}/cp/wiotp/msproxy-monitor:8.6.31 +file:///cp/wiotp/msproxy@sha256:8836dd25392c9e1ac74039046c79be36492aac574bc16aaa1c8b2e498c073a5e={{ registry_public_url_with_path }}/cp/wiotp/msproxy:8.6.31 +file:///cp/wiotp/org-store@sha256:d3e58211d9cb8d89f0a5ade4bfd45f1c9590e303febe914f1a9594fae2c05676={{ registry_public_url_with_path }}/cp/wiotp/org-store:4.0.72 +file:///cp/wiotp/orgdeprovmgr@sha256:349824703278e4c3a179a1cc86dfaf0c33e655e0cd26f93b26fc1e6f3b86f2ea={{ registry_public_url_with_path }}/cp/wiotp/orgdeprovmgr:2.2.91 +file:///cp/wiotp/orgpoolmgr@sha256:52c948f7697ebda09fc530ada7faea6b4d7c6c0525729ba5f21e7593831fec82={{ registry_public_url_with_path }}/cp/wiotp/orgpoolmgr:3.1.108 +file:///cp/wiotp/orgprovmgr@sha256:37bbd47ece8a22ad7010d0327d11732753bfc207060d30f5775ed249357d31ae={{ registry_public_url_with_path }}/cp/wiotp/orgprovmgr:2.1.90 +file:///cp/wiotp/pipelinerouter@sha256:e340853661440d6406775df1ff8bf7a00d744e3ee82a40139410f4634a40ffeb={{ registry_public_url_with_path }}/cp/wiotp/pipelinerouter:1.1.25 +file:///cp/wiotp/provagent-guardian@sha256:f755105203727598a3c24fa50b4245293b32f532ad28861c4b2c2fafd7974b20={{ registry_public_url_with_path }}/cp/wiotp/provagent-guardian:2.1.88 +file:///cp/wiotp/s2s-store@sha256:dd3770076413737fd826a2d1db721e69a8ad1553db338774038fe1d82f048dfd={{ registry_public_url_with_path }}/cp/wiotp/s2s-store:6.1.54 +file:///cp/wiotp/sqlmodel-auth@sha256:8d3194e0315f1cd8cdeaecc7980ac525a27afee39283aef56f5cd82e5e2d05f2={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-auth:1.1.40 +file:///cp/wiotp/sqlmodel-fpl@sha256:045ea5513b39251b55706638039fbe2af4feef23368d5dd72d48584a2cde5d81={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-fpl:1.0.33 +file:///cp/wiotp/sqlmodel-state@sha256:09631b932110a0e17d702a143ea57ad82dee6642825138b2cc89ab8ea670f84d={{ registry_public_url_with_path }}/cp/wiotp/sqlmodel-state:1.0.85 +file:///cp/wiotp/state-updater@sha256:503c371e30b15858b6c70e2e4c766555c367ed6b18e225a6bd391efa7d723e73={{ registry_public_url_with_path }}/cp/wiotp/state-updater:2.2.19 +file:///cp/wiotp/statsd@sha256:a03313c0064137668c333d5bf9ab50749947c35981366b931f83855b723df8c3={{ registry_public_url_with_path }}/cp/wiotp/statsd:1.3.99 +file:///cp/wiotp/ui-dashboard@sha256:2721067fc2cfc8bf9e041ff731d18bdd616f1ffc2b4aad6dcd8615aa0de19c4c={{ registry_public_url_with_path }}/cp/wiotp/ui-dashboard:7.4.8 +file:///cpopen/ibm-mas-iot-operator-bundle@sha256:524ec52f1142db1641ae30b6c2b91b80d6b4ab45534f308c93319a9103b9bcf5={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot-operator-bundle:8.6.7 +file:///cpopen/ibm-mas-iot-operator-catalog@sha256:09b6560684c9f651a9c837b080498e46162a334fb02ff5bb7a5b8faf75734e70={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot-operator-catalog:latest +file:///cpopen/ibm-mas-iot@sha256:eea161ace585c004d0442439f0acac7e3af1aea78478bb8ee5ded41049e27839={{ registry_public_url_with_path }}/cpopen/ibm-mas-iot:8.6.7 \ No newline at end of file diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/sls350fix/direct.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/sls350fix/direct.txt.j2 index ab9cff2def..b37a1fd326 100644 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/sls350fix/direct.txt.j2 +++ b/ibm/mas_devops/roles/mirror_case_prepare/templates/sls350fix/direct.txt.j2 @@ -1,3 +1,3 @@ -cp.icr.io/cp/sls/api-licensing@sha256:680474c128cd7d95d6bad5c348d059afe56454923b935f955d75669259c55a6c={{ registry_public_url }}/cp/sls/api-licensing:5.8.1 -cp.icr.io/cpopen/ibm-sls-operator-bundle@sha256:8ca4f5a4165a99aa6a82d72d9b6917bacf5959fb0851ab4c2447cba630e57f1e={{ registry_public_url }}/cpopen/ibm-sls-operator-bundle:3.5.0 -cp.icr.io/cpopen/ibm-sls@sha256:e66b36217c1cf34d6f83931fb9f55f57aa2032573db58ea767516879438e552f={{ registry_public_url }}/cpopen/ibm-sls:3.5.0 +cp.icr.io/cp/sls/api-licensing@sha256:680474c128cd7d95d6bad5c348d059afe56454923b935f955d75669259c55a6c={{ registry_public_url_with_path }}/cp/sls/api-licensing:5.8.1 +cp.icr.io/cpopen/ibm-sls-operator-bundle@sha256:8ca4f5a4165a99aa6a82d72d9b6917bacf5959fb0851ab4c2447cba630e57f1e={{ registry_public_url_with_path }}/cpopen/ibm-sls-operator-bundle:3.5.0 +cp.icr.io/cpopen/ibm-sls@sha256:e66b36217c1cf34d6f83931fb9f55f57aa2032573db58ea767516879438e552f={{ registry_public_url_with_path }}/cpopen/ibm-sls:3.5.0 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/sls350fix/from-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/sls350fix/from-filesystem.txt.j2 index aeb2a98dbb..27fd7671c4 100644 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/sls350fix/from-filesystem.txt.j2 +++ b/ibm/mas_devops/roles/mirror_case_prepare/templates/sls350fix/from-filesystem.txt.j2 @@ -1,3 +1,3 @@ -file:///cp/sls/api-licensing@sha256:680474c128cd7d95d6bad5c348d059afe56454923b935f955d75669259c55a6c={{ registry_public_url }}/cp/sls/api-licensing:5.8.1 -file://cpopen/ibm-sls-operator-bundle@sha256:8ca4f5a4165a99aa6a82d72d9b6917bacf5959fb0851ab4c2447cba630e57f1e={{ registry_public_url }}/cpopen/ibm-sls-operator-bundle:3.5.0 -file://cpopen/ibm-sls@sha256:e66b36217c1cf34d6f83931fb9f55f57aa2032573db58ea767516879438e552f={{ registry_public_url }}/cpopen/ibm-sls:3.5.0 +file:///cp/sls/api-licensing@sha256:680474c128cd7d95d6bad5c348d059afe56454923b935f955d75669259c55a6c={{ registry_public_url_with_path }}/cp/sls/api-licensing:5.8.1 +file://cpopen/ibm-sls-operator-bundle@sha256:8ca4f5a4165a99aa6a82d72d9b6917bacf5959fb0851ab4c2447cba630e57f1e={{ registry_public_url_with_path }}/cpopen/ibm-sls-operator-bundle:3.5.0 +file://cpopen/ibm-sls@sha256:e66b36217c1cf34d6f83931fb9f55f57aa2032573db58ea767516879438e552f={{ registry_public_url_with_path }}/cpopen/ibm-sls:3.5.0 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/direct.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/direct.txt.j2 deleted file mode 100644 index 5fd63fa6bb..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/direct.txt.j2 +++ /dev/null @@ -1,15 +0,0 @@ -icr.io/cpopen/ibm-uds/dataexchange-cli@sha256:bba1d98514a62eda85f7b7c9f23ec27f554583c054937aafa8ab2f881554974e={{ registry_public_url }}/cpopen/ibm-uds/dataexchange-cli:2.0.10 -icr.io/cpopen/ibm-uds/dataexchange-service@sha256:2ca2511f02f77f2210ba6e2180c6f13d91faaf8641dbadc9629843e13de39fc7={{ registry_public_url }}/cpopen/ibm-uds/dataexchange-service:2.0.10 -icr.io/cpopen/ibm-uds/event-api@sha256:c33a0059f26f6f57d7b1322c448a50af7345fa95151fb29a9bf20a7a72c6af74={{ registry_public_url }}/cpopen/ibm-uds/event-api:2.0.10 -icr.io/cpopen/ibm-uds/event-reader@sha256:27818819ad5dd0a187fa816b2c65c7673ae870af21fa55c781c9cf76b1d058fb={{ registry_public_url }}/cpopen/ibm-uds/event-reader:2.0.10 -icr.io/cpopen/ibm-uds/event-scheduler@sha256:eae226b96e904e4305670815b7ba2182c399b09e0b096e20bf75beab636f5ffe={{ registry_public_url }}/cpopen/ibm-uds/event-scheduler:2.0.10 -icr.io/cpopen/ibm-uds/store-api@sha256:d660de1e58024c1d3019a5041feb8cd26530fd258d36f4d4e834be80d19936e3={{ registry_public_url }}/cpopen/ibm-uds/store-api:2.0.10 -icr.io/cpopen/ibm-uds/uds-growth-stack-base@sha256:d13187798917f9d26cc530083e25cf1d16983073d182bf6d8e02b9097073a308={{ registry_public_url }}/cpopen/ibm-uds/uds-growth-stack-base:2.0.10 -icr.io/cpopen/ibm-user-data-services-catalog@sha256:f2fdd1c82563d183f69b02a28a3b1bf25052f942a6243c53e88a1c70d1351006={{ registry_public_url }}/cpopen/ibm-user-data-services-catalog:2.0.10 -icr.io/cpopen/ibm-user-data-services-operator-bundle@sha256:15b391fb854403e35c0858bbd8931311a86c67a977c0da2e07c3cf11fc5f489e={{ registry_public_url }}/cpopen/ibm-user-data-services-operator-bundle:2.0.10 -icr.io/cpopen/ibm-user-data-services-operator@sha256:e01defa911a3027c770d873e4c92aa5f496b4fa38e600653fd189323777918a2={{ registry_public_url }}/cpopen/ibm-user-data-services-operator:2.0.10 -registry.connect.redhat.com/crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b={{ registry_public_url }}/crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -registry.connect.redhat.com/crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a={{ registry_public_url }}/crunchydata/crunchy-postgres:ubi8-13.6-1 -registry.redhat.io/openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f={{ registry_public_url }}/openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -registry.redhat.io/rhel8/postgresql-12@sha256:fa920188f567e51d75aacd723f0964026e42ac060fed392036e8d4b3c7a8129f={{ registry_public_url }}/rhel8/postgresql-12:1-109 -registry.redhat.io/ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc={{ registry_public_url }}/ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/from-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/from-filesystem.txt.j2 deleted file mode 100644 index 79d2624a72..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/from-filesystem.txt.j2 +++ /dev/null @@ -1,15 +0,0 @@ -file:///cpopen/ibm-uds/dataexchange-cli@sha256:bba1d98514a62eda85f7b7c9f23ec27f554583c054937aafa8ab2f881554974e={{ registry_public_url }}/cpopen/ibm-uds/dataexchange-cli:2.0.10 -file:///cpopen/ibm-uds/dataexchange-service@sha256:2ca2511f02f77f2210ba6e2180c6f13d91faaf8641dbadc9629843e13de39fc7={{ registry_public_url }}/cpopen/ibm-uds/dataexchange-service:2.0.10 -file:///cpopen/ibm-uds/event-api@sha256:c33a0059f26f6f57d7b1322c448a50af7345fa95151fb29a9bf20a7a72c6af74={{ registry_public_url }}/cpopen/ibm-uds/event-api:2.0.10 -file:///cpopen/ibm-uds/event-reader@sha256:27818819ad5dd0a187fa816b2c65c7673ae870af21fa55c781c9cf76b1d058fb={{ registry_public_url }}/cpopen/ibm-uds/event-reader:2.0.10 -file:///cpopen/ibm-uds/event-scheduler@sha256:eae226b96e904e4305670815b7ba2182c399b09e0b096e20bf75beab636f5ffe={{ registry_public_url }}/cpopen/ibm-uds/event-scheduler:2.0.10 -file:///cpopen/ibm-uds/store-api@sha256:d660de1e58024c1d3019a5041feb8cd26530fd258d36f4d4e834be80d19936e3={{ registry_public_url }}/cpopen/ibm-uds/store-api:2.0.10 -file:///cpopen/ibm-uds/uds-growth-stack-base@sha256:d13187798917f9d26cc530083e25cf1d16983073d182bf6d8e02b9097073a308={{ registry_public_url }}/cpopen/ibm-uds/uds-growth-stack-base:2.0.10 -file:///cpopen/ibm-user-data-services-catalog@sha256:f2fdd1c82563d183f69b02a28a3b1bf25052f942a6243c53e88a1c70d1351006={{ registry_public_url }}/cpopen/ibm-user-data-services-catalog:2.0.10 -file:///cpopen/ibm-user-data-services-operator-bundle@sha256:15b391fb854403e35c0858bbd8931311a86c67a977c0da2e07c3cf11fc5f489e={{ registry_public_url }}/cpopen/ibm-user-data-services-operator-bundle:2.0.10 -file:///cpopen/ibm-user-data-services-operator@sha256:e01defa911a3027c770d873e4c92aa5f496b4fa38e600653fd189323777918a2={{ registry_public_url }}/cpopen/ibm-user-data-services-operator:2.0.10 -file:///crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b={{ registry_public_url }}/crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -file:///crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a={{ registry_public_url }}/crunchydata/crunchy-postgres:ubi8-13.6-1 -file:///openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f={{ registry_public_url }}/openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -file:///rhel8/postgresql-12@sha256:fa920188f567e51d75aacd723f0964026e42ac060fed392036e8d4b3c7a8129f={{ registry_public_url }}/rhel8/postgresql-12:1-109 -file:///ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc={{ registry_public_url }}/ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/to-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/to-filesystem.txt.j2 deleted file mode 100644 index 5c4a854bc9..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds2010fix/to-filesystem.txt.j2 +++ /dev/null @@ -1,15 +0,0 @@ -icr.io/cpopen/ibm-uds/dataexchange-cli@sha256:bba1d98514a62eda85f7b7c9f23ec27f554583c054937aafa8ab2f881554974e=file:///cpopen/ibm-uds/dataexchange-cli:2.0.10 -icr.io/cpopen/ibm-uds/dataexchange-service@sha256:2ca2511f02f77f2210ba6e2180c6f13d91faaf8641dbadc9629843e13de39fc7=file:///cpopen/ibm-uds/dataexchange-service:2.0.10 -icr.io/cpopen/ibm-uds/event-api@sha256:c33a0059f26f6f57d7b1322c448a50af7345fa95151fb29a9bf20a7a72c6af74=file:///cpopen/ibm-uds/event-api:2.0.10 -icr.io/cpopen/ibm-uds/event-reader@sha256:27818819ad5dd0a187fa816b2c65c7673ae870af21fa55c781c9cf76b1d058fb=file:///cpopen/ibm-uds/event-reader:2.0.10 -icr.io/cpopen/ibm-uds/event-scheduler@sha256:eae226b96e904e4305670815b7ba2182c399b09e0b096e20bf75beab636f5ffe=file:///cpopen/ibm-uds/event-scheduler:2.0.10 -icr.io/cpopen/ibm-uds/store-api@sha256:d660de1e58024c1d3019a5041feb8cd26530fd258d36f4d4e834be80d19936e3=file:///cpopen/ibm-uds/store-api:2.0.10 -icr.io/cpopen/ibm-uds/uds-growth-stack-base@sha256:d13187798917f9d26cc530083e25cf1d16983073d182bf6d8e02b9097073a308=file:///cpopen/ibm-uds/uds-growth-stack-base:2.0.10 -icr.io/cpopen/ibm-user-data-services-catalog@sha256:f2fdd1c82563d183f69b02a28a3b1bf25052f942a6243c53e88a1c70d1351006=file:///cpopen/ibm-user-data-services-catalog:2.0.10 -icr.io/cpopen/ibm-user-data-services-operator-bundle@sha256:15b391fb854403e35c0858bbd8931311a86c67a977c0da2e07c3cf11fc5f489e=file:///cpopen/ibm-user-data-services-operator-bundle:2.0.10 -icr.io/cpopen/ibm-user-data-services-operator@sha256:e01defa911a3027c770d873e4c92aa5f496b4fa38e600653fd189323777918a2=file:///cpopen/ibm-user-data-services-operator:2.0.10 -registry.connect.redhat.com/crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b=file:///crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -registry.connect.redhat.com/crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a=file:///crunchydata/crunchy-postgres:ubi8-13.6-1 -registry.redhat.io/openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f=file:///openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -registry.redhat.io/rhel8/postgresql-12@sha256:fa920188f567e51d75aacd723f0964026e42ac060fed392036e8d4b3c7a8129f=file:///rhel8/postgresql-12:1-109 -registry.redhat.io/ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc=file:///ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/direct.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/direct.txt.j2 deleted file mode 100644 index c248c09e26..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/direct.txt.j2 +++ /dev/null @@ -1,22 +0,0 @@ -icr.io/cpopen/ibm-user-data-services-catalog@sha256:22d3fcbf6aa45cc14e194f66d042468c401667b6a35d141a75f9248c97c51c5a={{ registry_public_url }}/cpopen/ibm-user-data-services-catalog:2.0.8 -icr.io/cpopen/ibm-user-data-services-operator-bundle@sha256:ea3be336807dc12e66f9a8c01bc3bbbab8e98e4a4608721f8aadb54c99c0cd24={{ registry_public_url }}/cpopen/ibm-user-data-services-operator-bundle:2.0.8 -icr.io/cpopen/ibm-user-data-services-operator@sha256:c8aebe926fd8de11ad84d5495f572e2c9db4e6fe81b0b383dbd0a63e49108067={{ registry_public_url }}/cpopen/ibm-user-data-services-operator:2.0.8 -icr.io/ibm-uds/dataexchange-service@sha256:046a59acaadff8e072307ca7842fcb49cc618f226ae4fa22f28f9d419796025d={{ registry_public_url }}/ibm-uds/dataexchange-service:2.0.8 -icr.io/ibm-uds/event-api@sha256:b56db8f4730765336cfec3dd3ce6a7e2dd216a08406363806530c3c2a250c8aa={{ registry_public_url }}/ibm-uds/event-api:2.0.8 -icr.io/ibm-uds/event-reader@sha256:e933de8d98480fcd31e60cd9da8c43a3e351cb9fff2269846aeed1a57b7702d6={{ registry_public_url }}/ibm-uds/event-reader:2.0.8 -icr.io/ibm-uds/event-scheduler@sha256:c5468edaa03f3f4a5d9bb17ffe40cc41af0539bc4fcb1e0f7b5a243c99b7d06a={{ registry_public_url }}/ibm-uds/event-scheduler:2.0.8 -icr.io/ibm-uds/store-api@sha256:8d0e58ef20cb6b7b2fc6ada2a75944614b986629ee0a4750e7a2501b3b3e30d4={{ registry_public_url }}/ibm-uds/store-api:2.0.8 -icr.io/ibm-uds/uds-growth-stack-base@sha256:2495ac4ae49118da6912aba7af05b56f5f1c650eb8157e6b120343463168a367={{ registry_public_url }}/ibm-uds/uds-growth-stack-base:2.0.8 -registry.connect.redhat.com/crunchydata/crunchy-pgadmin4@sha256:581cd941d9becbe7fb0c0b11167f2a35b536cd73f9b236c8449344eb2c77bac5={{ registry_public_url }}/crunchydata/crunchy-pgadmin4:ubi8-4.30-0 -registry.connect.redhat.com/crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b={{ registry_public_url }}/crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -registry.connect.redhat.com/crunchydata/crunchy-pgbouncer@sha256:7cced75b5a800104a469d09300100a8d8d777bf164054273c7cba5cfa2be1e26={{ registry_public_url }}/crunchydata/crunchy-pgbouncer:ubi8-1.16-2 -registry.connect.redhat.com/crunchydata/crunchy-postgres-exporter@sha256:dbcfeb1d49a4776ea4dc55dea9d7f3ad8ea95aebab8bca38a15942429adce49c={{ registry_public_url }}/crunchydata/crunchy-postgres-exporter:ubi8-5.1.0-0 -registry.connect.redhat.com/crunchydata/crunchy-postgres-gis@sha256:13bc18c31086037c159a7f4b661c12655193073824fc8a7dc2891e3d39d45fe9={{ registry_public_url }}/crunchydata/crunchy-postgres-gis:ubi8-13.6-3.0-1 -registry.connect.redhat.com/crunchydata/crunchy-postgres-gis@sha256:e996679a8821f0fa537a579125664854256a94c73ff7cc2cface3edb85737920={{ registry_public_url }}/crunchydata/crunchy-postgres-gis:ubi8-14.2-3.1-1 -registry.connect.redhat.com/crunchydata/crunchy-postgres-gis@sha256:f45e90e38f555cfce208bbace97a2abc473058839ffdd638c024432afafde8b2={{ registry_public_url }}/crunchydata/crunchy-postgres-gis:ubi8-13.6-3.1-1 -registry.connect.redhat.com/crunchydata/crunchy-postgres@sha256:5553680d6923f7517ab84181f073c92bb2b5a7acca8056f05ac61954b43b04d3={{ registry_public_url }}/crunchydata/crunchy-postgres:ubi8-14.2-1 -registry.connect.redhat.com/crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a={{ registry_public_url }}/crunchydata/crunchy-postgres:ubi8-13.6-1 -registry.connect.redhat.com/crunchydata/postgres-operator@sha256:440375bdcb9a41807682336d593ad9231ba5b12ebda83db2d0d6b510f88195f1={{ registry_public_url }}/crunchydata/postgres-operator:ubi8-5.1.0-0 -registry.redhat.io/openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f={{ registry_public_url }}/openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -registry.redhat.io/rhel8/postgresql-12@sha256:fa920188f567e51d75aacd723f0964026e42ac060fed392036e8d4b3c7a8129f={{ registry_public_url }}/rhel8/postgresql-12:1-109 -registry.redhat.io/ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc={{ registry_public_url }}/ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/from-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/from-filesystem.txt.j2 deleted file mode 100644 index e1976601f0..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/from-filesystem.txt.j2 +++ /dev/null @@ -1,22 +0,0 @@ -file:///cpopen/ibm-user-data-services-catalog@sha256:22d3fcbf6aa45cc14e194f66d042468c401667b6a35d141a75f9248c97c51c5a={{ registry_public_url }}/cpopen/ibm-user-data-services-catalog:2.0.8 -file:///cpopen/ibm-user-data-services-operator-bundle@sha256:ea3be336807dc12e66f9a8c01bc3bbbab8e98e4a4608721f8aadb54c99c0cd24={{ registry_public_url }}/cpopen/ibm-user-data-services-operator-bundle:2.0.8 -file:///cpopen/ibm-user-data-services-operator@sha256:c8aebe926fd8de11ad84d5495f572e2c9db4e6fe81b0b383dbd0a63e49108067={{ registry_public_url }}/cpopen/ibm-user-data-services-operator:2.0.8 -file:///crunchydata/crunchy-pgadmin4@sha256:581cd941d9becbe7fb0c0b11167f2a35b536cd73f9b236c8449344eb2c77bac5={{ registry_public_url }}/crunchydata/crunchy-pgadmin4:ubi8-4.30-0 -file:///crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b={{ registry_public_url }}/crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -file:///crunchydata/crunchy-pgbouncer@sha256:7cced75b5a800104a469d09300100a8d8d777bf164054273c7cba5cfa2be1e26={{ registry_public_url }}/crunchydata/crunchy-pgbouncer:ubi8-1.16-2 -file:///crunchydata/crunchy-postgres-exporter@sha256:dbcfeb1d49a4776ea4dc55dea9d7f3ad8ea95aebab8bca38a15942429adce49c={{ registry_public_url }}/crunchydata/crunchy-postgres-exporter:ubi8-5.1.0-0 -file:///crunchydata/crunchy-postgres-gis@sha256:13bc18c31086037c159a7f4b661c12655193073824fc8a7dc2891e3d39d45fe9={{ registry_public_url }}/crunchydata/crunchy-postgres-gis:ubi8-13.6-3.0-1 -file:///crunchydata/crunchy-postgres-gis@sha256:e996679a8821f0fa537a579125664854256a94c73ff7cc2cface3edb85737920={{ registry_public_url }}/crunchydata/crunchy-postgres-gis:ubi8-14.2-3.1-1 -file:///crunchydata/crunchy-postgres-gis@sha256:f45e90e38f555cfce208bbace97a2abc473058839ffdd638c024432afafde8b2={{ registry_public_url }}/crunchydata/crunchy-postgres-gis:ubi8-13.6-3.1-1 -file:///crunchydata/crunchy-postgres@sha256:5553680d6923f7517ab84181f073c92bb2b5a7acca8056f05ac61954b43b04d3={{ registry_public_url }}/crunchydata/crunchy-postgres:ubi8-14.2-1 -file:///crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a={{ registry_public_url }}/crunchydata/crunchy-postgres:ubi8-13.6-1 -file:///crunchydata/postgres-operator@sha256:440375bdcb9a41807682336d593ad9231ba5b12ebda83db2d0d6b510f88195f1={{ registry_public_url }}/crunchydata/postgres-operator:ubi8-5.1.0-0 -file:///ibm-uds/dataexchange-service@sha256:046a59acaadff8e072307ca7842fcb49cc618f226ae4fa22f28f9d419796025d={{ registry_public_url }}/ibm-uds/dataexchange-service:2.0.8 -file:///ibm-uds/event-api@sha256:b56db8f4730765336cfec3dd3ce6a7e2dd216a08406363806530c3c2a250c8aa={{ registry_public_url }}/ibm-uds/event-api:2.0.8 -file:///ibm-uds/event-reader@sha256:e933de8d98480fcd31e60cd9da8c43a3e351cb9fff2269846aeed1a57b7702d6={{ registry_public_url }}/ibm-uds/event-reader:2.0.8 -file:///ibm-uds/event-scheduler@sha256:c5468edaa03f3f4a5d9bb17ffe40cc41af0539bc4fcb1e0f7b5a243c99b7d06a={{ registry_public_url }}/ibm-uds/event-scheduler:2.0.8 -file:///ibm-uds/store-api@sha256:8d0e58ef20cb6b7b2fc6ada2a75944614b986629ee0a4750e7a2501b3b3e30d4={{ registry_public_url }}/ibm-uds/store-api:2.0.8 -file:///ibm-uds/uds-growth-stack-base@sha256:2495ac4ae49118da6912aba7af05b56f5f1c650eb8157e6b120343463168a367={{ registry_public_url }}/ibm-uds/uds-growth-stack-base:2.0.8 -file:///openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f={{ registry_public_url }}/openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -file:///rhel8/postgresql-12@sha256:fa920188f567e51d75aacd723f0964026e42ac060fed392036e8d4b3c7a8129f={{ registry_public_url }}/rhel8/postgresql-12:1-109 -file:///ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc={{ registry_public_url }}/ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/to-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/to-filesystem.txt.j2 deleted file mode 100644 index 9326d16b63..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds208fix/to-filesystem.txt.j2 +++ /dev/null @@ -1,22 +0,0 @@ -icr.io/cpopen/ibm-user-data-services-catalog@sha256:22d3fcbf6aa45cc14e194f66d042468c401667b6a35d141a75f9248c97c51c5a=file:///cpopen/ibm-user-data-services-catalog:2.0.8 -icr.io/cpopen/ibm-user-data-services-operator-bundle@sha256:ea3be336807dc12e66f9a8c01bc3bbbab8e98e4a4608721f8aadb54c99c0cd24=file:///cpopen/ibm-user-data-services-operator-bundle:2.0.8 -icr.io/cpopen/ibm-user-data-services-operator@sha256:c8aebe926fd8de11ad84d5495f572e2c9db4e6fe81b0b383dbd0a63e49108067=file:///cpopen/ibm-user-data-services-operator:2.0.8 -icr.io/ibm-uds/dataexchange-service@sha256:046a59acaadff8e072307ca7842fcb49cc618f226ae4fa22f28f9d419796025d=file:///ibm-uds/dataexchange-service:2.0.8 -icr.io/ibm-uds/event-api@sha256:b56db8f4730765336cfec3dd3ce6a7e2dd216a08406363806530c3c2a250c8aa=file:///ibm-uds/event-api:2.0.8 -icr.io/ibm-uds/event-reader@sha256:e933de8d98480fcd31e60cd9da8c43a3e351cb9fff2269846aeed1a57b7702d6=file:///ibm-uds/event-reader:2.0.8 -icr.io/ibm-uds/event-scheduler@sha256:c5468edaa03f3f4a5d9bb17ffe40cc41af0539bc4fcb1e0f7b5a243c99b7d06a=file:///ibm-uds/event-scheduler:2.0.8 -icr.io/ibm-uds/store-api@sha256:8d0e58ef20cb6b7b2fc6ada2a75944614b986629ee0a4750e7a2501b3b3e30d4=file:///ibm-uds/store-api:2.0.8 -icr.io/ibm-uds/uds-growth-stack-base@sha256:2495ac4ae49118da6912aba7af05b56f5f1c650eb8157e6b120343463168a367=file:///ibm-uds/uds-growth-stack-base:2.0.8 -registry.connect.redhat.com/crunchydata/crunchy-pgadmin4@sha256:581cd941d9becbe7fb0c0b11167f2a35b536cd73f9b236c8449344eb2c77bac5=file:///crunchydata/crunchy-pgadmin4:ubi8-4.30-0 -registry.connect.redhat.com/crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b=file:///crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -registry.connect.redhat.com/crunchydata/crunchy-pgbouncer@sha256:7cced75b5a800104a469d09300100a8d8d777bf164054273c7cba5cfa2be1e26=file:///crunchydata/crunchy-pgbouncer:ubi8-1.16-2 -registry.connect.redhat.com/crunchydata/crunchy-postgres-exporter@sha256:dbcfeb1d49a4776ea4dc55dea9d7f3ad8ea95aebab8bca38a15942429adce49c=file:///crunchydata/crunchy-postgres-exporter:ubi8-5.1.0-0 -registry.connect.redhat.com/crunchydata/crunchy-postgres-gis@sha256:13bc18c31086037c159a7f4b661c12655193073824fc8a7dc2891e3d39d45fe9=file:///crunchydata/crunchy-postgres-gis:ubi8-13.6-3.0-1 -registry.connect.redhat.com/crunchydata/crunchy-postgres-gis@sha256:e996679a8821f0fa537a579125664854256a94c73ff7cc2cface3edb85737920=file:///crunchydata/crunchy-postgres-gis:ubi8-14.2-3.1-1 -registry.connect.redhat.com/crunchydata/crunchy-postgres-gis@sha256:f45e90e38f555cfce208bbace97a2abc473058839ffdd638c024432afafde8b2=file:///crunchydata/crunchy-postgres-gis:ubi8-13.6-3.1-1 -registry.connect.redhat.com/crunchydata/crunchy-postgres@sha256:5553680d6923f7517ab84181f073c92bb2b5a7acca8056f05ac61954b43b04d3=file:///crunchydata/crunchy-postgres:ubi8-14.2-1 -registry.connect.redhat.com/crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a=file:///crunchydata/crunchy-postgres:ubi8-13.6-1 -registry.connect.redhat.com/crunchydata/postgres-operator@sha256:440375bdcb9a41807682336d593ad9231ba5b12ebda83db2d0d6b510f88195f1=file:///crunchydata/postgres-operator:ubi8-5.1.0-0 -registry.redhat.io/openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f=file:///openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -registry.redhat.io/rhel8/postgresql-12@sha256:fa920188f567e51d75aacd723f0964026e42ac060fed392036e8d4b3c7a8129f=file:///rhel8/postgresql-12:1-109 -registry.redhat.io/ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc=file:///ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/direct.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/direct.txt.j2 deleted file mode 100644 index f5f76d50fa..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/direct.txt.j2 +++ /dev/null @@ -1,15 +0,0 @@ -icr.io/cpopen/ibm-uds/dataexchange-cli@sha256:c147871fc5913a54855e348db1fcea1c031cb8a59e25e09f6ae32ef23357d26b={{ registry_public_url }}/cpopen/ibm-uds/dataexchange-cli:2.0.9 -icr.io/cpopen/ibm-uds/dataexchange-service@sha256:f7f113950c29da07c0ba74000be3483a80df07fc6334a22ce7abfbb005dbb23a={{ registry_public_url }}/cpopen/ibm-uds/dataexchange-service:2.0.9 -icr.io/cpopen/ibm-uds/event-api@sha256:13c139e2d5dd3e06b298efdc428c7f61d5302c4fdbb672697b6ce78ae1a4af77={{ registry_public_url }}/cpopen/ibm-uds/event-api:2.0.9 -icr.io/cpopen/ibm-uds/event-reader@sha256:d8eb989b7af4bd5b6efe336535e65e4b9ab18ea4a8025d186f987ba949edb7e7={{ registry_public_url }}/cpopen/ibm-uds/event-reader:2.0.9 -icr.io/cpopen/ibm-uds/event-scheduler@sha256:899b5c907503caba65c035c112f0b492966244722f67bb7f0718b1b9821c356f={{ registry_public_url }}/cpopen/ibm-uds/event-scheduler:2.0.9 -icr.io/cpopen/ibm-uds/store-api@sha256:0f4c4dbe5c922d5c46a9f0159c69ca6018b12fcc1e494e3e7c9bf72c15993e51={{ registry_public_url }}/cpopen/ibm-uds/store-api:2.0.9 -icr.io/cpopen/ibm-uds/uds-growth-stack-base@sha256:231c92403d4711027c86ec29cef0fa23452d6c0b46d4a5ac9d269b0743430a06={{ registry_public_url }}/cpopen/ibm-uds/uds-growth-stack-base:2.0.9 -icr.io/cpopen/ibm-user-data-services-catalog@sha256:e54dafa673c43b8af138c5555ba7b6de74ba7ab50a903f4d3d4c67a9b72c5c30={{ registry_public_url }}/cpopen/ibm-user-data-services-catalog:2.0.9 -icr.io/cpopen/ibm-user-data-services-operator-bundle@sha256:3b96dcdad554efea6c3b0a65019d54a1df9065674c769d0ba0812853a22e63d3={{ registry_public_url }}/cpopen/ibm-user-data-services-operator-bundle:2.0.9 -icr.io/cpopen/ibm-user-data-services-operator@sha256:ceb43a0e04d1ba8f87396af0a2012870af8b3f73e55c62e96335ec6fd79d0509={{ registry_public_url }}/cpopen/ibm-user-data-services-operator:2.0.9 -registry.connect.redhat.com/crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b={{ registry_public_url }}/crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -registry.connect.redhat.com/crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a={{ registry_public_url }}/crunchydata/crunchy-postgres:ubi8-13.6-1 -registry.redhat.io/openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f={{ registry_public_url }}/openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -registry.redhat.io/rhel8/postgresql-12@sha256:3d805540d777b09b4da6df99e7cddf9598d5ece4af9f6851721a9961df40f5a1={{ registry_public_url }}/rhel8/postgresql-12:1-130 -registry.redhat.io/ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc={{ registry_public_url }}/ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/from-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/from-filesystem.txt.j2 deleted file mode 100644 index 53f181716d..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/from-filesystem.txt.j2 +++ /dev/null @@ -1,15 +0,0 @@ -file:///cpopen/ibm-uds/dataexchange-cli@sha256:c147871fc5913a54855e348db1fcea1c031cb8a59e25e09f6ae32ef23357d26b={{ registry_public_url }}/cpopen/ibm-uds/dataexchange-cli:2.0.9 -file:///cpopen/ibm-uds/dataexchange-service@sha256:f7f113950c29da07c0ba74000be3483a80df07fc6334a22ce7abfbb005dbb23a={{ registry_public_url }}/cpopen/ibm-uds/dataexchange-service:2.0.9 -file:///cpopen/ibm-uds/event-api@sha256:13c139e2d5dd3e06b298efdc428c7f61d5302c4fdbb672697b6ce78ae1a4af77={{ registry_public_url }}/cpopen/ibm-uds/event-api:2.0.9 -file:///cpopen/ibm-uds/event-reader@sha256:d8eb989b7af4bd5b6efe336535e65e4b9ab18ea4a8025d186f987ba949edb7e7={{ registry_public_url }}/cpopen/ibm-uds/event-reader:2.0.9 -file:///cpopen/ibm-uds/event-scheduler@sha256:899b5c907503caba65c035c112f0b492966244722f67bb7f0718b1b9821c356f={{ registry_public_url }}/cpopen/ibm-uds/event-scheduler:2.0.9 -file:///cpopen/ibm-uds/store-api@sha256:0f4c4dbe5c922d5c46a9f0159c69ca6018b12fcc1e494e3e7c9bf72c15993e51={{ registry_public_url }}/cpopen/ibm-uds/store-api:2.0.9 -file:///cpopen/ibm-uds/uds-growth-stack-base@sha256:231c92403d4711027c86ec29cef0fa23452d6c0b46d4a5ac9d269b0743430a06={{ registry_public_url }}/cpopen/ibm-uds/uds-growth-stack-base:2.0.9 -file:///cpopen/ibm-user-data-services-catalog@sha256:e54dafa673c43b8af138c5555ba7b6de74ba7ab50a903f4d3d4c67a9b72c5c30={{ registry_public_url }}/cpopen/ibm-user-data-services-catalog:2.0.9 -file:///cpopen/ibm-user-data-services-operator-bundle@sha256:3b96dcdad554efea6c3b0a65019d54a1df9065674c769d0ba0812853a22e63d3={{ registry_public_url }}/cpopen/ibm-user-data-services-operator-bundle:2.0.9 -file:///cpopen/ibm-user-data-services-operator@sha256:ceb43a0e04d1ba8f87396af0a2012870af8b3f73e55c62e96335ec6fd79d0509={{ registry_public_url }}/cpopen/ibm-user-data-services-operator:2.0.9 -file:///crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b={{ registry_public_url }}/crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -file:///crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a={{ registry_public_url }}/crunchydata/crunchy-postgres:ubi8-13.6-1 -file:///openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f={{ registry_public_url }}/openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -file:///rhel8/postgresql-12@sha256:3d805540d777b09b4da6df99e7cddf9598d5ece4af9f6851721a9961df40f5a1={{ registry_public_url }}/rhel8/postgresql-12:1-130 -file:///ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc={{ registry_public_url }}/ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/to-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/to-filesystem.txt.j2 deleted file mode 100644 index b86839013b..0000000000 --- a/ibm/mas_devops/roles/mirror_case_prepare/templates/uds209fix/to-filesystem.txt.j2 +++ /dev/null @@ -1,15 +0,0 @@ -icr.io/cpopen/ibm-uds/dataexchange-cli@sha256:c147871fc5913a54855e348db1fcea1c031cb8a59e25e09f6ae32ef23357d26b=file:///cpopen/ibm-uds/dataexchange-cli:2.0.9 -icr.io/cpopen/ibm-uds/dataexchange-service@sha256:f7f113950c29da07c0ba74000be3483a80df07fc6334a22ce7abfbb005dbb23a=file:///cpopen/ibm-uds/dataexchange-service:2.0.9 -icr.io/cpopen/ibm-uds/event-api@sha256:13c139e2d5dd3e06b298efdc428c7f61d5302c4fdbb672697b6ce78ae1a4af77=file:///cpopen/ibm-uds/event-api:2.0.9 -icr.io/cpopen/ibm-uds/event-reader@sha256:d8eb989b7af4bd5b6efe336535e65e4b9ab18ea4a8025d186f987ba949edb7e7=file:///cpopen/ibm-uds/event-reader:2.0.9 -icr.io/cpopen/ibm-uds/event-scheduler@sha256:899b5c907503caba65c035c112f0b492966244722f67bb7f0718b1b9821c356f=file:///cpopen/ibm-uds/event-scheduler:2.0.9 -icr.io/cpopen/ibm-uds/store-api@sha256:0f4c4dbe5c922d5c46a9f0159c69ca6018b12fcc1e494e3e7c9bf72c15993e51=file:///cpopen/ibm-uds/store-api:2.0.9 -icr.io/cpopen/ibm-uds/uds-growth-stack-base@sha256:231c92403d4711027c86ec29cef0fa23452d6c0b46d4a5ac9d269b0743430a06=file:///cpopen/ibm-uds/uds-growth-stack-base:2.0.9 -icr.io/cpopen/ibm-user-data-services-catalog@sha256:e54dafa673c43b8af138c5555ba7b6de74ba7ab50a903f4d3d4c67a9b72c5c30=file:///cpopen/ibm-user-data-services-catalog:2.0.9 -icr.io/cpopen/ibm-user-data-services-operator-bundle@sha256:3b96dcdad554efea6c3b0a65019d54a1df9065674c769d0ba0812853a22e63d3=file:///cpopen/ibm-user-data-services-operator-bundle:2.0.9 -icr.io/cpopen/ibm-user-data-services-operator@sha256:ceb43a0e04d1ba8f87396af0a2012870af8b3f73e55c62e96335ec6fd79d0509=file:///cpopen/ibm-user-data-services-operator:2.0.9 -registry.connect.redhat.com/crunchydata/crunchy-pgbackrest@sha256:efe775d3208befb2b7f026ef5fee3b03b306a9ba773709ec5c4c3391880ee60b=file:///crunchydata/crunchy-pgbackrest:ubi8-2.38-0 -registry.connect.redhat.com/crunchydata/crunchy-postgres@sha256:6b570ee2922281eedc5c267c50ad30a895fbb4e8a132c3e2c3a38e29fe3d6f6a=file:///crunchydata/crunchy-postgres:ubi8-13.6-1 -registry.redhat.io/openshift4/ose-cli@sha256:ccc2d3d593bb4dab980b5483970eb441ede94bbe25972670b37d8890dce7f06f=file:///openshift4/ose-cli:v4.8.0-202205121606.p0.g41ff67e.assembly.stream -registry.redhat.io/rhel8/postgresql-12@sha256:3d805540d777b09b4da6df99e7cddf9598d5ece4af9f6851721a9961df40f5a1=file:///rhel8/postgresql-12:1-130 -registry.redhat.io/ubi8/nodejs-14@sha256:881e871f845b9395f5e21cfa45f0d1838dc9af60c4f18ece67bd56a9e44846cc=file:///ubi8/nodejs-14:1-75 diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/README.md b/ibm/mas_devops/roles/mirror_extras_prepare/README.md index 89b189f4c1..4f0eacaff6 100644 --- a/ibm/mas_devops/roles/mirror_extras_prepare/README.md +++ b/ibm/mas_devops/roles/mirror_extras_prepare/README.md @@ -10,7 +10,6 @@ Available Extras | catalog | N/A | Special extra package for mirroring the IBM Maximo Operator Catalog | | db2u | 1.0.0, 1.0.1 | Extra container images missing from the ibm-db2operator CASE bundle | | mongoce | 4.2.6, 4.2.23, 4.4.21 | Package containing all images required to use MongoCE Operator in the disconnected environment | -| uds | 1.0.0, 1.1.0, 1.2.0, 1.3.0 | Extra container images missing from the ibm-uds CASE bundle | | wd | 5.3.1 | Extra container images missing from the ibm-watson-discovery CASE bundle | | odf | 4.15 | Extra images needed for ODF 4.15 | diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/defaults/main.yml b/ibm/mas_devops/roles/mirror_extras_prepare/defaults/main.yml index 220026a48f..cf1af97fe2 100644 --- a/ibm/mas_devops/roles/mirror_extras_prepare/defaults/main.yml +++ b/ibm/mas_devops/roles/mirror_extras_prepare/defaults/main.yml @@ -3,15 +3,18 @@ # ----------------------------------------------------------------------------- registry_public_host: "{{ lookup('env', 'REGISTRY_PUBLIC_HOST') }}" registry_public_port: "{{ lookup('env', 'REGISTRY_PUBLIC_PORT') }}" -env_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') }}" -registry_prefix: "{% if env_prefix|length > 0 %}/{{ env_prefix }}{% endif %}" # If the prefix is not empty then add a slash -registry_public_url: "{{ registry_public_host }}:{{ registry_public_port }}{{ registry_prefix }}" +registry_public_url: "{{ registry_public_host }}{% if registry_public_port != '' %}:{{ registry_public_port }}{% endif %}" + +registry_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') | default('', true) }}" +registry_public_url_with_path: "{{ registry_public_url }}{% if registry_prefix | length > 0 %}/{{ registry_prefix }}{% endif %}" + # Extras config # ----------------------------------------------------------------------------- extras_name: "{{ lookup('env', 'EXTRAS_NAME') }}" extras_version: "{{ lookup('env', 'EXTRAS_VERSION') }}" + # Dev support # ----------------------------------------------------------------------------- artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/templates/direct.txt.j2 b/ibm/mas_devops/roles/mirror_extras_prepare/templates/direct.txt.j2 index 26565e6c2c..45c1e46c3a 100644 --- a/ibm/mas_devops/roles/mirror_extras_prepare/templates/direct.txt.j2 +++ b/ibm/mas_devops/roles/mirror_extras_prepare/templates/direct.txt.j2 @@ -1,3 +1,3 @@ {%- for image in extra_images %} -{{ image.registry }}/{{ image.name }}@{{ image.digest }}={{ registry_public_url }}/{{ image.name }}{{ ':' ~ image.tag if image.tag is defined else '' }} +{{ image.registry }}/{{ image.name }}@{{ image.digest }}={{ registry_public_url_with_path }}/{{ image.name }}{{ ':' ~ image.tag if image.tag is defined else '' }} {% endfor %} diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/templates/from-filesystem.txt.j2 b/ibm/mas_devops/roles/mirror_extras_prepare/templates/from-filesystem.txt.j2 index 285e096b8b..a425f4059a 100644 --- a/ibm/mas_devops/roles/mirror_extras_prepare/templates/from-filesystem.txt.j2 +++ b/ibm/mas_devops/roles/mirror_extras_prepare/templates/from-filesystem.txt.j2 @@ -1,3 +1,3 @@ {%- for image in extra_images %} -file:///{{ image.name }}@{{ image.digest }}={{ registry_public_url }}/{{ image.name }}{{ ':' ~ image.tag if image.tag is defined else '' }} +file:///{{ image.name }}@{{ image.digest }}={{ registry_public_url_with_path }}/{{ image.name }}{{ ':' ~ image.tag if image.tag is defined else '' }} {% endfor %} diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/spark_3.3.4.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/spark_3.3.4.yml new file mode 100644 index 0000000000..4b1ed0921c --- /dev/null +++ b/ibm/mas_devops/roles/mirror_extras_prepare/vars/spark_3.3.4.yml @@ -0,0 +1,16 @@ +--- +extra_images: + - name: cp/cpd/spark-hb-jkg + registry: cp.icr.io + tag: 3.3.4.v51-amd64 + digest: sha256:8eac99b7c400192c540d066f61db156560ecdb4a6049de0333533f2505cda1a8 + + - name: cp/cpd/spark-hb-jkg + registry: cp.icr.io + tag: 3.3.4.v51-s390x + digest: sha256:50ff80ba00aa5408a2fbfb1daea0726d21ae26082247fcf1770a7ff6b3ca8293 + + - name: cp/cpd/spark-hb-jkg + registry: cp.icr.io + tag: 3.3.4.v51-ppc64le + digest: sha256:b31b6865dd6d30bbc1593d085f0663cf89181fb2ad12c833b2530f0ab65165db diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/spark_3.4.2.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/spark_3.4.2.yml new file mode 100644 index 0000000000..300bb190b9 --- /dev/null +++ b/ibm/mas_devops/roles/mirror_extras_prepare/vars/spark_3.4.2.yml @@ -0,0 +1,16 @@ +--- +extra_images: + - name: cp/cpd/spark-hb-jkg + registry: cp.icr.io + tag: 3.4.2.v51-amd64 + digest: sha256:805b3d92a86967e33b10f2b433dce7f75473402b3869800ca46acf374c91977a + + - name: cp/cpd/spark-hb-jkg + registry: cp.icr.io + tag: 3.4.2.v51-s390x + digest: sha256:db1370a6582109343439b4910e6c4502834434c47fb6435972a095d2440ecd81 + + - name: cp/cpd/spark-hb-jkg + registry: cp.icr.io + tag: 3.4.2.v51-ppc64le + digest: sha256:92741f239b5d7024d160af0acd07cd4b75ffc31d4329bf2c962425f3ae8781dc diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.0.0.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.0.0.yml deleted file mode 100644 index 0664ec9a47..0000000000 --- a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.0.0.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -extra_images: - - name: cpopen/cpfs/ibm-events-kafka-3.1.0 - registry: icr.io - tag: 4.2.0 - digest: sha256:359be5faff98968f1d59fc5e07fc50735003730643e81ea0f25dbc1ba6eac164 - - - name: cpopen/cpfs/ibm-events-kafka-3.2.0 - registry: icr.io - tag: 4.2.0 - digest: sha256:f960eace2e54e368516f5b17c18150ae92d6ef281c888d4c7f1c03a919e31ad2 - - - name: cpopen/ibm-events-operator - registry: icr.io - tag: 4.2.0 - digest: sha256:f10d8161e4fae49b2c9663d3fc38087462ed519c1c96a6d7b9e2990ec38afe1b diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.1.0.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.1.0.yml deleted file mode 100644 index 105b7989f0..0000000000 --- a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.1.0.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -extra_images: - - name: cpopen/cpfs/ibm-events-kafka-3.2.0 - registry: icr.io - tag: 4.2.1 - digest: sha256:6eab4c8aecb7d5d6a391d8ed9b79405d9199ded12474a18f76e8eb16426ea2ef - - - name: cpopen/ibm-events-operator - registry: icr.io - tag: 4.2.1 - digest: sha256:1ac2659319076b49cc7f659110f739224eedde45ceb3e61ad2ea97a63cc45225 diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.2.0.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.2.0.yml deleted file mode 100644 index d4de120a56..0000000000 --- a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.2.0.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -extra_images: - - name: cpopen/cpfs/ibm-events-kafka-3.2.3 - registry: icr.io - tag: 4.4.0 - digest: sha256:1a40b470b6278d6d780eddd4d0196a8c10616455cd9025827677f30f87153f41 - - - name: cpopen/ibm-events-operator - registry: icr.io - tag: 4.4.0 - digest: sha256:fa35abd5c2a4e26f41682ac967329ed982ee915497971a1860df6f3663e305ca diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.3.0.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.3.0.yml deleted file mode 100644 index 25e518c336..0000000000 --- a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.3.0.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -extra_images: - - name: cpopen/cpfs/ibm-events-kafka-3.4.0 - registry: icr.io - tag: 4.6.0 - digest: sha256:c0eb8c824343f7f8d9aafcb79984535e984410e568c236f8a7832a1811817fcf - - - name: cpopen/ibm-events-operator - registry: icr.io - tag: 4.6.0 - digest: sha256:b18c5d744843440226c3a2f43271edb78dfce06549b19fd1fbc65608b5928614 diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.4.0.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.4.0.yml deleted file mode 100644 index 7ec4edb2d6..0000000000 --- a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.4.0.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -extra_images: - - name: cpopen/cpfs/ibm-events-kafka-3.4.0 - registry: icr.io - tag: 4.6.1 - digest: sha256:c21d3b14267c7b4746adb8bad1fc2d8b3883973039dd304b59b1a066b13f1772 - - - name: cpopen/ibm-events-operator - registry: icr.io - tag: 4.6.1 - digest: sha256:39b72f2df66325ac5317a8a414c4f3fed27343081cbcd07252abc389e9330d84 diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.5.0.yml b/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.5.0.yml deleted file mode 100644 index 624a77601a..0000000000 --- a/ibm/mas_devops/roles/mirror_extras_prepare/vars/uds_1.5.0.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -extra_images: - - name: cpopen/cpfs/ibm-events-kafka-3.5.1 - registry: icr.io - tag: 4.9.0 - digest: sha256:4049092de0221944ffcc8764b7d61ae53773f00c90938127341afb08e6f3704d - - - name: cpopen/ibm-events-operator - registry: icr.io - tag: 4.9.0 - digest: sha256:815003818cc6c5e9303580fed35de26720f3aa4336c5e7e0e676eb2475c0f2af diff --git a/ibm/mas_devops/roles/mirror_images/defaults/main.yml b/ibm/mas_devops/roles/mirror_images/defaults/main.yml index 72542b812c..7312b6b3ee 100644 --- a/ibm/mas_devops/roles/mirror_images/defaults/main.yml +++ b/ibm/mas_devops/roles/mirror_images/defaults/main.yml @@ -1,9 +1,10 @@ --- registry_public_host: "{{ lookup('env', 'REGISTRY_PUBLIC_HOST') }}" registry_public_port: "{{ lookup('env', 'REGISTRY_PUBLIC_PORT') }}" -env_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') }}" -registry_prefix: "{% if env_prefix|length > 0 %}/{{ env_prefix }}{% endif %}" # If the prefix is not empty then add a slash -registry_public_url: "{{ registry_public_host }}:{{ registry_public_port }}{{ registry_prefix }}" +registry_public_url: "{{ registry_public_host }}{% if registry_public_port != '' %}:{{ registry_public_port }}{% endif %}" + +registry_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') | default('', true) }}" +registry_public_url_with_path: "{{ registry_public_url }}{% if registry_prefix | length > 0 %}/{{ registry_prefix }}{% endif %}" # Development config # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mirror_ocp/README.md b/ibm/mas_devops/roles/mirror_ocp/README.md index 234ddb9c28..bb52a92321 100644 --- a/ibm/mas_devops/roles/mirror_ocp/README.md +++ b/ibm/mas_devops/roles/mirror_ocp/README.md @@ -11,11 +11,10 @@ Four actions are supported: Three **Catalogs** are mirrored, containing the following content: ### certified-operator-index -1. crunchy-postgres-operator (required by ibm.mas_devops.uds role) -2. gpu-operator-certified (required by ibm.mas_devops.nvidia_gpu role) -3. kubeturbo-certified (required by ibm.mas_devops.kubeturbo role) -4. ibm-metrics-operator (required by ibm.mas_devops.dro role) -5. ibm-data-reporter-operator (required by ibm.mas_devops.dro role) +1. gpu-operator-certified (required by ibm.mas_devops.nvidia_gpu role) +2. kubeturbo-certified (required by ibm.mas_devops.kubeturbo role) +3. ibm-metrics-operator (required by ibm.mas_devops.dro role) +4. ibm-data-reporter-operator (required by ibm.mas_devops.dro role) ### community-operator-index 1. grafana-operator (required by ibm.mas_devops.grafana role) @@ -82,21 +81,21 @@ Path to your Red Hat pull secret, available from: [https://console.redhat.com/op Role Variables - OpenShift Version ------------------------------------------------------------------------------- ### ocp_release -The Red Hat release you are mirroring content for, e.g. `4.12`. +The Red Hat release you are mirroring content for, e.g. `4.16`. - **Required** - Environment Variable: `OCP_RELEASE` - Default: None ### ocp_min_version -The minimum version of the Red Hat release to mirror platform content for, e.g. `4.12.0`. +The minimum version of the Red Hat release to mirror platform content for, e.g. `4.16.11`. - **Optional** - Environment Variable: `OCP_MIN_VERSION` - Default: None ### ocp_max_version -The maximimum version of the Red Hat release to mirror platform content for, e.g. `4.12.18`. +The maximimum version of the Red Hat release to mirror platform content for, e.g. `4.16.20`. - **Optional** - Environment Variable: `OCP_MAX_VERSION` @@ -119,10 +118,11 @@ The public port number for the target registry - Environment Variable: `REGISTRY_PUBLIC_PORT` - Default: None -### registry_prefix -The prefix used for the target registry. The images will not be mirrored to the registry at this time but will define the final destination in the form: {host}:{port}/{prefix}/{reponame} +### registry_prefix_redhat +The prefix used for the target registry. The images will not be mirrored to the registry at this time but will define the final destination in the form: `{host}[:{port}]/{prefix}/{reponame}` -- Environment Variable: `REGISTRY_PREFIX` +- Optional +- Environment Variable: `REGISTRY_PREFIX_REDHAT` - Default: None ### registry_username @@ -148,7 +148,7 @@ Example Playbook vars: registry_public_host: myregistry.mycompany.com registry_public_port: 5000 - registry_prefix: projectName + registry_prefix_redhat: "ocp416" registry_username: user1 registry_password: 8934jk77s862! # Not a real password, don't worry security folks @@ -157,7 +157,7 @@ Example Playbook mirror_redhat_platform: false mirror_redhat_operators: true - ocp_release: 4.15 + ocp_release: 4.16 redhat_pullsecret: ~/pull-secret.json roles: diff --git a/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml b/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml index f24b79749c..169c7f6e4a 100644 --- a/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml +++ b/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml @@ -12,9 +12,11 @@ mirror_redhat_operators: "{{ lookup('env', 'MIRROR_REDHAT_OPERATORS') | default( # ----------------------------------------------------------------------------- registry_public_host: "{{ lookup('env', 'REGISTRY_PUBLIC_HOST') }}" registry_public_port: "{{ lookup('env', 'REGISTRY_PUBLIC_PORT') }}" -env_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') }}" -registry_prefix: "{% if env_prefix|length > 0 %}/{{ env_prefix }}{% endif %}" # If the prefix is not empty then add a slash -registry_public_url: "{{ registry_public_host }}:{{ registry_public_port }}{{ registry_prefix }}" +registry_public_url: "{{ registry_public_host }}{% if registry_public_port != '' %}:{{ registry_public_port }}{% endif %}" + +registry_prefix_redhat: "{{ lookup('env', 'REGISTRY_PREFIX_REDHAT') | default('', true) }}" +registry_public_url_with_path_redhat: "{{ registry_public_url }}{% if registry_prefix_redhat | length > 0 %}/{{ registry_prefix_redhat }}{% endif %}" + registry_username: "{{ lookup('env', 'REGISTRY_USERNAME') }}" registry_password: "{{ lookup('env', 'REGISTRY_PASSWORD') }}" registry_auth: "{{ registry_username }}:{{ registry_password }}" diff --git a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml index 689fc8278b..53d30669da 100644 --- a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml +++ b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml @@ -5,7 +5,7 @@ assert: that: - ocp_release is defined and ocp_release != "" - - registry_public_url is defined and registry_public_url != "" + - registry_public_url_with_path_redhat is defined and registry_public_url_with_path_redhat != "" - mirror_working_dir is defined and mirror_working_dir != "" fail_msg: "One or more required properties are missing" @@ -21,9 +21,9 @@ - name: "Debug Information" debug: msg: - - "Command ...................... DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dest-skip-tls --config={{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml docker://{{ registry_public_url }}" + - "Command ...................... DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dest-skip-tls --config={{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml docker://{{ registry_public_url_with_path_redhat }}" - "Log File ..................... {{ mirror_working_dir }}/logs/mirror-direct-ocp{{ ocp_release }}.log" - name: "Mirror Red Hat content from source registry to target registry" shell: > - DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dest-skip-tls --config={{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml docker://{{ registry_public_url }} &> {{ mirror_working_dir }}/logs/mirror-direct-ocp{{ ocp_release }}.log + DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dest-skip-tls --config={{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml docker://{{ registry_public_url_with_path_redhat }} &> {{ mirror_working_dir }}/logs/mirror-direct-ocp{{ ocp_release }}.log diff --git a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml index 27d52d22e4..cfbad8bfa0 100644 --- a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml +++ b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml @@ -4,7 +4,7 @@ - name: "Fail if required properties are not provided" assert: that: - - registry_public_url is defined and registry_public_url != "" + - registry_public_url_with_path_redhat is defined and registry_public_url_with_path_redhat != "" - mirror_working_dir is defined and mirror_working_dir != "" fail_msg: "One or more required properties are missing" @@ -14,9 +14,9 @@ - name: "Debug Information" debug: msg: - - "Command ...................... DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dest-skip-tls --max-per-registry=1 --from={{ mirror_working_dir }}/mirror_seq1_000000.tar docker://{{ registry_public_url }}" + - "Command ...................... DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dest-skip-tls --max-per-registry=1 --from={{ mirror_working_dir }}/mirror_seq1_000000.tar docker://{{ registry_public_url_with_path_redhat }}" - "Log File ..................... {{ mirror_working_dir }}/logs/mirror-from-filesystem-ocp{{ ocp_release }}.log" - name: "Mirror Red Hat content from filesystem to target registry" shell: > - DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dest-skip-tls --max-per-registry=1 --from={{ mirror_working_dir }}/mirror_seq1_000000.tar docker://{{ registry_public_url }} &> {{ mirror_working_dir }}/logs/mirror-from-filesystem-ocp{{ ocp_release }}.log + DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dest-skip-tls --max-per-registry=1 --from={{ mirror_working_dir }}/mirror_seq1_000000.tar docker://{{ registry_public_url_with_path_redhat }} &> {{ mirror_working_dir }}/logs/mirror-from-filesystem-ocp{{ ocp_release }}.log diff --git a/ibm/mas_devops/roles/mirror_ocp/templates/imagesetconfiguration.yml.j2 b/ibm/mas_devops/roles/mirror_ocp/templates/imagesetconfiguration.yml.j2 index 438e451ede..4e7ebaca45 100644 --- a/ibm/mas_devops/roles/mirror_ocp/templates/imagesetconfiguration.yml.j2 +++ b/ibm/mas_devops/roles/mirror_ocp/templates/imagesetconfiguration.yml.j2 @@ -21,9 +21,6 @@ mirror: # certified-operators - catalog: registry.redhat.io/redhat/certified-operator-index:v{{ ocp_release }} packages: - - name: crunchy-postgres-operator # Required by ibm.mas_devops.uds role - channels: - - name: v5 - name: gpu-operator-certified # Required by ibm.mas_devops.nvidia_gpu role channels: - name: v23.3 @@ -47,7 +44,9 @@ mirror: packages: - name: grafana-operator # Required by ibm.mas_devops.grafana role channels: +{% if ocp_release != "4.16" %} - name: v4 +{% endif %} - name: v5 - name: opentelemetry-operator # Required by ibm.mas_devops.opentelemetry role channels: diff --git a/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml b/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml index 96b3f7d2a4..598db8a3be 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml +++ b/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml @@ -133,4 +133,3 @@ when: - gpu_clusterpolicy_result.resources | length == 0 - gpu_clusterpolicy_result.resources[0].status is not defined - diff --git a/ibm/mas_devops/roles/ocp_cluster_monitoring/README.md b/ibm/mas_devops/roles/ocp_cluster_monitoring/README.md index 113839c0eb..472c1b9b1e 100644 --- a/ibm/mas_devops/roles/ocp_cluster_monitoring/README.md +++ b/ibm/mas_devops/roles/ocp_cluster_monitoring/README.md @@ -2,8 +2,8 @@ ocp_cluster_monitoring =============================================================================== Configures the OpenShift Container Platform Cluster Monitoring enabling two settings: -- [OpenShift user defined project monitoring](https://access.redhat.com/documentation/en-us/openshift_container_platform/4.12/html/monitoring/enabling-monitoring-for-user-defined-projects) is enabled (`openshift-monitoring` namespace) -- [OpenShift monitoring stack](https://access.redhat.com/documentation/en-us/openshift_container_platform/4.12/html/monitoring/index) is configured to use persistent storage (`openshift-monitoring` namespace) +- [OpenShift user defined project monitoring](https://access.redhat.com/documentation/en-us/openshift_container_platform/4.16/html/monitoring/enabling-monitoring-for-user-defined-projects) is enabled (`openshift-monitoring` namespace) +- [OpenShift monitoring stack](https://access.redhat.com/documentation/en-us/openshift_container_platform/4.16/html/monitoring/index) is configured to use persistent storage (`openshift-monitoring` namespace) Role Variables diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/defaults/main.yml b/ibm/mas_devops/roles/ocp_contentsourcepolicy/defaults/main.yml deleted file mode 100644 index 78106d6fea..0000000000 --- a/ibm/mas_devops/roles/ocp_contentsourcepolicy/defaults/main.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# Get Registry facts from environment variables if they have not been passed to the role -registry_private_host: "{{ lookup('env', 'REGISTRY_PRIVATE_HOST') }}" -registry_private_port: "{{ lookup('env', 'REGISTRY_PRIVATE_PORT') }}" -registry_private_url: "{{ registry_private_host }}:{{ registry_private_port }}" - -registry_private_ca_file: "{{ lookup('env', 'REGISTRY_PRIVATE_CA_FILE') }}" - -registry_username: "{{ lookup('env', 'REGISTRY_USERNAME') }}" -registry_password: "{{ lookup('env', 'REGISTRY_PASSWORD') }}" -registry_auth: "{{ registry_username }}:{{ registry_password }}" - -setup_redhat_catalogs: "{{ lookup('env', 'SETUP_REDHAT_CATALOGS') | default('False', true) | bool }}" -setup_redhat_release: "{{ lookup('env', 'SETUP_REDHAT_RELEASE') | default('False', true) | bool }}" -ocp_release: "{{ lookup('env', 'OCP_RELEASE') }}" - -# Only used in development to add an extra entry to the pull secret enabling pulls -# from a second mirror registry -fvt_image_registry: "{{ lookup('env', 'FVT_IMAGE_REGISTRY') }}" -artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" -artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" -artifactory_auth: "{{ artifactory_username }}:{{ artifactory_token }}" - -#Optional redhat catalog prefix settings -env_redhat_catalogs_prefix: "{{ lookup('env', 'REDHAT_CATALOGS_PREFIX') | default('', true) }}" -redhat_catalogs_prefix: "{% if env_redhat_catalogs_prefix|length > 0 %}{{ env_redhat_catalogs_prefix }}-{% endif %}" # If the prefix is not empty then add a dash diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/main.yml b/ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/main.yml deleted file mode 100644 index 89bf57e26f..0000000000 --- a/ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/main.yml +++ /dev/null @@ -1,66 +0,0 @@ ---- -# Configure the Cluster to pull images from the mirror -# - Configure the Image Content Source Policy -# - wait for nodes to restart - - -# 1. Param check -# ----------------------------------------------------------------------------- -- name: "Check for required parameters" - assert: - that: - - registry_private_host is defined and registry_private_host != "" - - registry_private_port is defined and registry_private_port != "" - - registry_private_ca_file is defined and registry_private_ca_file != "" - - -# 2. Configure CA trust -# ----------------------------------------------------------------------------- -- name: Set up trust for the registry - include_tasks: "tasks/trust.yml" - - -# 3. Update default image pull secret -# ----------------------------------------------------------------------------- -- name: Update default image pull secret - when: - - registry_username != "" - - registry_password != "" - include_tasks: "tasks/update-pull-secret.yml" - -- name: Update default image pull secret (dev) - when: - - artifactory_username != "" - - artifactory_token != "" - include_tasks: "tasks/update-pull-secret-dev.yml" - - -# 4. Create ImageContentSourcePolicy for MAS & dependencies -# ----------------------------------------------------------------------------- -- name: Create ImageContentSourcePolicy - kubernetes.core.k8s: - apply: yes - template: 'templates/imagecontentsourcepolicy.yml.j2' - register: content_source_policy_result - - -# 5. Create Catalog Sources and ContentSourcePolicy -# ----------------------------------------------------------------------------- -- name: "Create Catalog Sources and ContentSourcePolicy" - when: setup_redhat_catalogs - kubernetes.core.k8s: - apply: yes - template: "templates/redhat-catalogs.yml.j2" - -- name: "Create ContentSourcePolicy for OCP release" - when: setup_redhat_release - kubernetes.core.k8s: - apply: yes - template: "templates/redhat-release.yml.j2" - - -# 6. Wait until the nodes have applied the updates -# ----------------------------------------------------------------------------- -- name: Wait for Machine Configs to update - when: content_source_policy_result.changed - include_tasks: "{{ role_path }}/../../common_tasks/wait-machine-config-update.yml" diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/redhat-catalogs.yml.j2 b/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/redhat-catalogs.yml.j2 deleted file mode 100644 index 8231e1cc61..0000000000 --- a/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/redhat-catalogs.yml.j2 +++ /dev/null @@ -1,127 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: CatalogSource -metadata: - name: certified-operator-index - namespace: openshift-marketplace -spec: - displayName: Certified Operators - publisher: Red Hat - image: {{ registry_private_url }}/redhat/{{ redhat_catalogs_prefix }}certified-operator-index:v{{ ocp_release }} - sourceType: grpc ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: CatalogSource -metadata: - name: community-operator-index - namespace: openshift-marketplace -spec: - displayName: Community Operators - publisher: Red Hat - image: {{ registry_private_url }}/redhat/{{ redhat_catalogs_prefix }}community-operator-index:v{{ ocp_release }} - sourceType: grpc ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: CatalogSource -metadata: - name: redhat-operator-index - namespace: openshift-marketplace -spec: - displayName: Red Hat Operators - publisher: Red Hat - image: {{ registry_private_url }}/redhat/{{ redhat_catalogs_prefix }}redhat-operator-index:v{{ ocp_release }} - sourceType: grpc ---- -apiVersion: operator.openshift.io/v1alpha1 -kind: ImageContentSourcePolicy -metadata: - name: ibm-mas-redhat-catalogs - labels: - operators.openshift.org/catalog: "true" -spec: - repositoryDigestMirrors: - - mirrors: - - {{ registry_private_url }}/cpopen - source: icr.io/cpopen - - mirrors: - - {{ registry_private_url }}/rhel8 - source: registry.redhat.io/rhel8 - - mirrors: - - {{ registry_private_url }}/rhel9 - source: registry.redhat.io/rhel9 - - mirrors: - - {{ registry_private_url }}/crunchydata - source: registry.connect.redhat.com/crunchydata - - mirrors: - - {{ registry_private_url }}/nvidia - source: registry.connect.redhat.com/nvidia - - mirrors: - - {{ registry_private_url }}/grafana - source: ghcr.io/grafana - - mirrors: - - {{ registry_private_url }}/grafana - source: docker.io/grafana - - mirrors: - # This is for the grafana v5 operator bundle image - - {{ registry_private_url }}/community-operator-pipeline-prod - source: quay.io/community-operator-pipeline-prod - - mirrors: - - {{ registry_private_url }}/open-telemetry - source: ghcr.io/open-telemetry - - mirrors: - - {{ registry_private_url }}/source-to-image - source: registry.redhat.io/source-to-image - - mirrors: - - {{ registry_private_url }}/odf4 - source: registry.redhat.io/odf4 - - mirrors: - - {{ registry_private_url }}/operator-pipeline-prod - source: quay.io/operator-pipeline-prod - - mirrors: - - {{ registry_private_url }}/strimzi - source: quay.io/strimzi - - mirrors: - - {{ registry_private_url }}/rhceph - source: registry.redhat.io/rhceph - - mirrors: - - {{ registry_private_url }}/amq-streams - source: registry.redhat.io/amq-streams - - mirrors: - - {{ registry_private_url }}/nvidia - source: nvcr.io/nvidia - - mirrors: - - {{ registry_private_url }}/openshift4 - source: registry.redhat.io/openshift4 - - mirrors: - - {{ registry_private_url }}/openshift-community-operators - source: quay.io/openshift-community-operators - - mirrors: - - {{ registry_private_url }}/kubebuilder - source: gcr.io/kubebuilder - - mirrors: - - {{ registry_private_url }}/ubi8 - source: registry.redhat.io/ubi8 - - mirrors: - - {{ registry_private_url }}/ubi9 - source: registry.redhat.io/ubi9 - - mirrors: - - {{ registry_private_url }}/openshift-pipelines - source: registry.redhat.io/openshift-pipelines - - mirrors: - - {{ registry_private_url }}/openshift-serverless-1 - source: registry.redhat.io/openshift-serverless-1 - - mirrors: - - {{ registry_private_url }}/turbonomic - source: registry.connect.redhat.com/turbonomic - - mirrors: - - {{ registry_private_url }}/rh-marketplace - source: quay.io/rh-marketplace - - mirrors: - - {{ registry_private_url }}/rh-marketplace - source: registry.connect.redhat.com/rh-marketplace - - mirrors: - - {{ registry_private_url }}/cert-manager - source: registry.redhat.io/cert-manager - - mirrors: - - {{ registry_private_url }}/lvms4 - source: registry.redhat.io/lvms4 diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/redhat-release.yml.j2 b/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/redhat-release.yml.j2 deleted file mode 100644 index b36e13912e..0000000000 --- a/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/redhat-release.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: operator.openshift.io/v1alpha1 -kind: ImageContentSourcePolicy -metadata: - name: ibm-mas-redhat-release -spec: - repositoryDigestMirrors: - - mirrors: - - {{ registry_private_url }}/openshift/release - source: quay.io/openshift-release-dev/ocp-v4.0-art-dev - - mirrors: - - {{ registry_private_url }}/openshift/release-images - source: quay.io/openshift-release-dev/ocp-release diff --git a/ibm/mas_devops/roles/ocp_deprovision/tasks/providers/rosa.yml b/ibm/mas_devops/roles/ocp_deprovision/tasks/providers/rosa.yml index 075ce02a07..0f8a79b9ca 100644 --- a/ibm/mas_devops/roles/ocp_deprovision/tasks/providers/rosa.yml +++ b/ibm/mas_devops/roles/ocp_deprovision/tasks/providers/rosa.yml @@ -73,8 +73,8 @@ shell: rosa describe cluster -c {{ cluster_name }} -o json register: cluster_lookup failed_when: "cluster_lookup.rc > 1" - retries: 60 - delay: 60 # 60s * 60 retries = 1 hour + retries: 90 + delay: 60 # 60s * 90 retries = 1.5 hours until: cluster_lookup.rc == 1 diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/README.md b/ibm/mas_devops/roles/ocp_idms/README.md similarity index 67% rename from ibm/mas_devops/roles/ocp_contentsourcepolicy/README.md rename to ibm/mas_devops/roles/ocp_idms/README.md index 866924b014..9771ca39ad 100644 --- a/ibm/mas_devops/roles/ocp_contentsourcepolicy/README.md +++ b/ibm/mas_devops/roles/ocp_idms/README.md @@ -1,9 +1,10 @@ -ocp_contentsourcepolicy +ocp_idms =============================================================================== -Installs an ImageContentSourcePolicy for IBM Maximo Application Suite's Maximo Operator Catalog. Optionally can also install a second ContentSourcePolicy suitable for the Red Hat Operator Catalogs created by [mirror_ocp](mirror_ocp.md). +Installs an **ImageDigestMirrorSet** (IDMS)for IBM Maximo Application Suite's Maximo Operator Catalog. Optionally can also install a second IDMS suitable for the Red Hat Operator Catalogs created by [mirror_ocp](mirror_ocp.md). If there are legacy **ImageContentSourcePolicies** installed by previous versions of this role, they will be deleted. !!! warning - This role doesn't work on IBMCloud ROKS. IBM Cloud RedHat OpenShift Service does not implement support for `ImageContentSourcePolicies`. If you want to use image mirroring you must manually configure each worker node individually using the IBM Cloud command line tool. + This role doesn't work on IBMCloud ROKS. IBM Cloud RedHat OpenShift Service does not implement support for `ImageDigestMirrorSet`. If you want to use image mirroring you must manually configure each worker node individually using the IBM Cloud command line tool. + IBM Maximo Operator Catalog Content ------------------------------------------------------------------------------- @@ -59,26 +60,19 @@ If you are managing the Red Hat Operator Catalogs yourself the content therein m Role Variables ------------------------------------------------------------------------------- ### setup_redhat_release -Instruct the role to setup **ContentSourcePolicy** for the mirrored release content generated by [mirror_ocp](mirror_ocp.md). This will create an additional policy named `ibm-mas-redhat-release`. +Instruct the role to setup **ImageDigestMirrorSet** for the mirrored release content generated by [mirror_ocp](mirror_ocp.md). This will create an additional policy named `ibm-mas-redhat-release`. - **Required** - Environment Variable: `SETUP_REDHAT_RELEASE` - Default: `False` ### setup_redhat_catalogs -Instruct the role to setup **CatalogSources** and **ContentSourcePolicy** for the mirror catalogs generated by [mirror_ocp](mirror_ocp.md). This will create an additional policy named `ibm-mas-redhat-catalogs`. +Instruct the role to setup **CatalogSources** and **ImageDigestMirrorSet** for the mirror catalogs generated by [mirror_ocp](mirror_ocp.md). This will create an additional policy named `ibm-mas-redhat-catalogs`. - **Required** - Environment Variable: `SETUP_REDHAT_CATALOGS` - Default: `False` -### ocp_release -The Red Hat release you are configuring an image content source policy for, e.g. `4.15`. - -- **Required** if `setup_redhat_catalogs` is enabled (not required if only `setup_redhat_release` is used) -- Environment Variable: `OCP_RELEASE` -- Default: None - Role Variables - Target Registry ------------------------------------------------------------------------------- @@ -92,7 +86,7 @@ The private hostname for the target registry ### registry_private_port The private port number for the target registry -- **Required** +- Optional - Environment Variable: `REGISTRY_PRIVATE_PORT` - Default: None @@ -117,8 +111,22 @@ The password for the target registry. - Environment Variable: `REGISTRY_PASSWORD` - Default: None +### registry_prefix +Optional additional path prefixed to all image repositories related to the IBM Maximo Operator Catalog in the target registry. We recommend the use of the catalog datestamp for this prefix to organize your registry, e.g. `mas-241107`, `mas-241205`. This should match the value used when you mirrored the images with [mirror_images](mirror_images.md). + +- Optional +- Environment Variable: `REGISTRY_PREFIX` +- Default: None + +### registry_prefix_redhat +Optional additional path prefixed to all image repositories related to the Red Hat Release and Operator Catalogs in the target registry. We recommend the use of the OpenShift release for this prefix to organize your registry, e.g. `ocp-412`, `ocp-414`. This should match the value used when you mirrored the images with [mirror_ocp](mirror_ocp.md). + +- **Required** +- Environment Variable: `REGISTRY_PREFIX_REDHAT` +- Default: None + ### redhat_catalogs_prefix -The prefix amended to the catalog sources names. E.g: With a `redhat_catalogs_prefix` of "ibm-mas" then `redhat/certified-operator-index` would instead be created as `redhat/ibm-mas-certified-operator-index` +An optional prefix to apply to the catalog sources names for the 3 Red Hat catalogs supported by this role. For example, setting a value of `ibm-mas` will result in three catalog sources named `ibm-mas-certified-operator-index`, `ibm-mas-community-operator-index`, `ibm-mas-redhat-operator-index` being created. - Optional - Environment Variable: `REDHAT_CATALOGS_PREFIX` diff --git a/ibm/mas_devops/roles/ocp_idms/defaults/main.yml b/ibm/mas_devops/roles/ocp_idms/defaults/main.yml new file mode 100644 index 0000000000..cc2e52d2a9 --- /dev/null +++ b/ibm/mas_devops/roles/ocp_idms/defaults/main.yml @@ -0,0 +1,40 @@ +--- +# Get Registry facts from environment variables if they have not been passed to the role +registry_private_host: "{{ lookup('env', 'REGISTRY_PRIVATE_HOST') }}" +registry_private_port: "{{ lookup('env', 'REGISTRY_PRIVATE_PORT') }}" +registry_private_url: "{{ registry_private_host }}{% if registry_private_port != '' %}:{{ registry_private_port }}{% endif %}" + +registry_private_ca_file: "{{ lookup('env', 'REGISTRY_PRIVATE_CA_FILE') }}" + +registry_username: "{{ lookup('env', 'REGISTRY_USERNAME') }}" +registry_password: "{{ lookup('env', 'REGISTRY_PASSWORD') }}" +registry_auth: "{{ registry_username }}:{{ registry_password }}" + +setup_redhat_catalogs: "{{ lookup('env', 'SETUP_REDHAT_CATALOGS') | default('False', true) | bool }}" +setup_redhat_release: "{{ lookup('env', 'SETUP_REDHAT_RELEASE') | default('False', true) | bool }}" + +# Only used in development to add an extra entry to the pull secret enabling pulls +# from a second mirror registry +fvt_image_registry: "{{ lookup('env', 'FVT_IMAGE_REGISTRY') }}" +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" +artifactory_auth: "{{ artifactory_username }}:{{ artifactory_token }}" + + +# Optional Registry Prefix - IBM Content +# ----------------------------------------------------------------------------- +registry_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') | default('', true) }}" +registry_private_url_with_path: "{{ registry_private_url }}{% if registry_prefix | length > 0 %}/{{ registry_prefix }}{% endif %}" +idms_suffix: "{% if registry_prefix | length > 0 %}-{{ registry_prefix }}{% endif %}" + + +# Optional Registry Prefix - Red Hat Content +# ----------------------------------------------------------------------------- +registry_prefix_redhat: "{{ lookup('env', 'REGISTRY_PREFIX_REDHAT') | default('', true) }}" +registry_private_url_with_path_redhat: "{{ registry_private_url }}{% if registry_prefix_redhat | length > 0 %}/{{ registry_prefix_redhat }}{% endif %}" +idms_suffix_redhat: "{% if registry_prefix_redhat | length > 0 %}-{{ registry_prefix_redhat }}{% endif %}" + + +# Optional Red Hat CatalogSource Name Prefix +# ----------------------------------------------------------------------------- +redhat_catalogs_prefix: "{{ lookup('env', 'REDHAT_CATALOGS_PREFIX') | default('', true) }}" diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/meta/main.yml b/ibm/mas_devops/roles/ocp_idms/meta/main.yml similarity index 100% rename from ibm/mas_devops/roles/ocp_contentsourcepolicy/meta/main.yml rename to ibm/mas_devops/roles/ocp_idms/meta/main.yml diff --git a/ibm/mas_devops/roles/ocp_idms/tasks/main.yml b/ibm/mas_devops/roles/ocp_idms/tasks/main.yml new file mode 100644 index 0000000000..6993686093 --- /dev/null +++ b/ibm/mas_devops/roles/ocp_idms/tasks/main.yml @@ -0,0 +1,135 @@ +--- +# Configure the Cluster to pull images from the mirror +# - Configure the Image Content Source Policy +# - wait for nodes to restart + + +# 1. Param check +# ----------------------------------------------------------------------------- +- name: "Check for required parameters" + assert: + that: + - registry_private_host is defined and registry_private_host != "" + - registry_private_ca_file is defined and registry_private_ca_file != "" + +- name: "Settings" + debug: + msg: + - "Registry Private Host ..................... {{ registry_private_host }}" + - "Registry Private Port ..................... {{ registry_private_port | default('') }}" + - "Registry Private URL ...................... {{ registry_private_url }}" + - "Registry Prefix (IBM) ..................... {{ registry_prefix }}" + - "Registry (IBM) ............................ {{ registry_private_url_with_path }}" + - "Registry Prefix (Red Hat) ................. {{ registry_prefix_redhat }}" + - "Registry (Red Hat) ........................ {{ registry_private_url_with_path_redhat }}" + + +# 2. Configure CA trust +# ----------------------------------------------------------------------------- +- name: Set up trust for the registry + include_tasks: "tasks/trust.yml" + + +# 3. Update default image pull secret +# ----------------------------------------------------------------------------- +- name: Update default image pull secret + when: + - registry_username != "" + - registry_password != "" + include_tasks: "tasks/update-pull-secret.yml" + +- name: Update default image pull secret (dev) + when: + - fvt_image_registry != "" + - artifactory_username != "" + - artifactory_token != "" + include_tasks: "tasks/update-pull-secret-dev.yml" + + +# 4. IBM Maximo Operator Catalog +# ----------------------------------------------------------------------------- +- name: Create ImageDigestMirrorSet + kubernetes.core.k8s: + apply: yes + template: 'templates/idms/mas-ibm-catalog.yml.j2' + register: idms_result + + +# 5. Red Hat Operator Catalogs +# ----------------------------------------------------------------------------- +- name: "Look up cluster version" + when: setup_redhat_catalogs + kubernetes.core.k8s_info: + api_version: config.openshift.io/v1 + name: "version" + kind: ClusterVersion + register: version_info + +- name: "Assert that we can obtain cluster version information" + when: setup_redhat_catalogs + assert: + that: version_info.resources[0] is defined + fail_msg: "Unable to determine OCP version information" + +- name: "Set OCP release" + when: setup_redhat_catalogs + set_fact: + ocp_version: "{{ version_info.resources[0].status.desired.version }}" + ocp_release: "{{ version_info.resources[0].status.desired.version | regex_search('^([0-9]+)\\.([0-9]+)') }}" + +- name: "Debug OCP release" + when: setup_redhat_catalogs + debug: + msg: + - "OCP Version ............................ {{ ocp_version }}" + - "OCP Release ............................ {{ ocp_release }}" + +- name: "Create Catalog Sources and ImageDigestMirrorSet" + when: setup_redhat_catalogs + kubernetes.core.k8s: + apply: yes + template: "templates/idms/mas-redhat-catalogs.yml.j2" + + +# 6. Red Hat Release Catalog +# ----------------------------------------------------------------------------- +- name: "Create ImageDigestMirrorSet for OCP release" + when: setup_redhat_release + kubernetes.core.k8s: + apply: yes + template: "templates/idms/mas-redhat-release.yml.j2" + + +# 7. Delete the old ICSPs +# ----------------------------------------------------------------------------- +- name: Delete old ibm-mas-and-dependencies ImageContentSourcePolicy + kubernetes.core.k8s: + state: absent + api_version: operator.openshift.io/v1alpha1 + kind: ImageContentSourcePolicy + name: ibm-mas-and-dependencies + wait: true + wait_timeout: 600 # 10 minutes +- name: Delete old ibm-mas-redhat-catalogs ImageContentSourcePolicy + kubernetes.core.k8s: + state: absent + api_version: operator.openshift.io/v1alpha1 + kind: ImageContentSourcePolicy + name: ibm-mas-redhat-catalogs + wait: true + wait_timeout: 600 # 10 minutes +- name: Delete old ibm-mas-redhat-release ImageContentSourcePolicy + kubernetes.core.k8s: + state: absent + api_version: operator.openshift.io/v1alpha1 + kind: ImageContentSourcePolicy + name: ibm-mas-redhat-release + wait: true + wait_timeout: 600 # 10 minutes + + +# 7. Wait until the nodes have applied the updates +# ----------------------------------------------------------------------------- +- name: Wait for Machine Configs to update + when: idms_result.changed + include_tasks: "{{ role_path }}/../../common_tasks/wait-machine-config-update.yml" diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/trust.yml b/ibm/mas_devops/roles/ocp_idms/tasks/trust.yml similarity index 100% rename from ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/trust.yml rename to ibm/mas_devops/roles/ocp_idms/tasks/trust.yml diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/update-pull-secret-dev.yml b/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret-dev.yml similarity index 100% rename from ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/update-pull-secret-dev.yml rename to ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret-dev.yml diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/update-pull-secret.yml b/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret.yml similarity index 100% rename from ibm/mas_devops/roles/ocp_contentsourcepolicy/tasks/update-pull-secret.yml rename to ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret.yml diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/configmap.yml.j2 b/ibm/mas_devops/roles/ocp_idms/templates/configmap.yml.j2 similarity index 59% rename from ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/configmap.yml.j2 rename to ibm/mas_devops/roles/ocp_idms/templates/configmap.yml.j2 index ff9dfd0853..cef4e6f62a 100644 --- a/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/configmap.yml.j2 +++ b/ibm/mas_devops/roles/ocp_idms/templates/configmap.yml.j2 @@ -5,5 +5,12 @@ metadata: name: registry-config namespace: openshift-config data: +{%- if registry_private_port == "" %} + + {{ registry_private_host }}: | + {{ registry_private_ca_crt | indent(4, False) }} +{%- else %} + {{ registry_private_host }}..{{ registry_private_port }}: | {{ registry_private_ca_crt | indent(4, False) }} +{%- endif %} diff --git a/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/imagecontentsourcepolicy.yml.j2 b/ibm/mas_devops/roles/ocp_idms/templates/idms/mas-ibm-catalog.yml.j2 similarity index 51% rename from ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/imagecontentsourcepolicy.yml.j2 rename to ibm/mas_devops/roles/ocp_idms/templates/idms/mas-ibm-catalog.yml.j2 index 8ebd9cc688..4ac4c13589 100644 --- a/ibm/mas_devops/roles/ocp_contentsourcepolicy/templates/imagecontentsourcepolicy.yml.j2 +++ b/ibm/mas_devops/roles/ocp_idms/templates/idms/mas-ibm-catalog.yml.j2 @@ -1,57 +1,74 @@ -apiVersion: operator.openshift.io/v1alpha1 -kind: ImageContentSourcePolicy +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet metadata: - name: ibm-mas-and-dependencies + name: mas-ibm-catalog{{ idms_suffix }} + labels: + mas.ibm.com/idmsContent: ibm + annotations: + mas.ibm.com/idmsRegistry: "{{ registry_private_url_with_path }}" + mas.ibm.com/idmsRegistryHost: "{{ registry_private_host }}" + mas.ibm.com/idmsRegistryPort: "{{ registry_private_port }}" + mas.ibm.com/idmsRegistryPrefix: "{{ registry_prefix }}" spec: - repositoryDigestMirrors: + imageDigestMirrors: # 1. IBM Container Registry # ------------------------------------------------------------------------- # All IBM operators live in the cpopen namespace - source: icr.io/cpopen mirrors: - - {{ registry_private_url }}/cpopen + - {{ registry_private_url_with_path }}/cpopen + mirrorSourcePolicy: NeverContactSource # IBM truststoremanager worker image lives in ibm-truststore-mgr namespace - source: icr.io/ibm-truststore-mgr mirrors: - - {{ registry_private_url }}/ibm-truststore-mgr + - {{ registry_private_url_with_path }}/ibm-truststore-mgr + mirrorSourcePolicy: NeverContactSource # IBM SLS content live in ibm-sls namespace - source: icr.io/ibm-sls mirrors: - - {{ registry_private_url }}/ibm-sls + - {{ registry_private_url_with_path }}/ibm-sls + mirrorSourcePolicy: NeverContactSource # IBM UDS content live in ibm-uds namespace - source: icr.io/ibm-uds mirrors: - - {{ registry_private_url }}/ibm-uds + - {{ registry_private_url_with_path }}/ibm-uds + mirrorSourcePolicy: NeverContactSource # IBM Db2 Universal operator content live in db2u namespace - source: icr.io/db2u mirrors: - - {{ registry_private_url }}/db2u + - {{ registry_private_url_with_path }}/db2u + mirrorSourcePolicy: NeverContactSource # 2. IBM Entitled Container Registry # ------------------------------------------------------------------------- # All IBM entitled container images live under cp namespace - source: cp.icr.io/cp mirrors: - - {{ registry_private_url }}/cp + - {{ registry_private_url_with_path }}/cp + mirrorSourcePolicy: NeverContactSource # 3. Red Hat Quay.io Container Registry # ------------------------------------------------------------------------- # IBM common services live here - source: quay.io/opencloudio mirrors: - - {{ registry_private_url }}/opencloudio + - {{ registry_private_url_with_path }}/opencloudio + mirrorSourcePolicy: NeverContactSource # MongoDb Community Edition Operator & associated container images - source: quay.io/mongodb mirrors: - - {{ registry_private_url }}/mongodb + - {{ registry_private_url_with_path }}/mongodb + mirrorSourcePolicy: NeverContactSource # Eclipse Amlen - Message Broker for IoT/Mobile/Web. Mainly uses MQTT v3.x and v5. - source: quay.io/amlen mirrors: - - {{ registry_private_url }}/amlen - # Non-product IBM Maximo Application Suite images (e.g. db2 backup operator & mirror of dockerhub mongodb image) + - {{ registry_private_url_with_path }}/amlen + mirrorSourcePolicy: NeverContactSource + # Non-product IBM Maximo Application Suite images (e.g CLI & mirror of dockerhub mongodb image) - source: quay.io/ibmmas mirrors: - - {{ registry_private_url }}/ibmmas + - {{ registry_private_url_with_path }}/ibmmas + mirrorSourcePolicy: NeverContactSource {% if artifactory_username != "" %} # 4. Artifactory @@ -59,4 +76,5 @@ spec: - source: docker-na-public.artifactory.swg-devops.com/wiotp-docker-local mirrors: - docker-us-south-edge-public.artifactory.swg-devops.com/wiotp-docker-local + mirrorSourcePolicy: NeverContactSource {% endif %} diff --git a/ibm/mas_devops/roles/ocp_idms/templates/idms/mas-redhat-catalogs.yml.j2 b/ibm/mas_devops/roles/ocp_idms/templates/idms/mas-redhat-catalogs.yml.j2 new file mode 100644 index 0000000000..a5b7e8f875 --- /dev/null +++ b/ibm/mas_devops/roles/ocp_idms/templates/idms/mas-redhat-catalogs.yml.j2 @@ -0,0 +1,161 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: {% if redhat_catalogs_prefix | length > 0 %}{{ redhat_catalogs_prefix }}-{% endif %}certified-operator-index + namespace: openshift-marketplace +spec: + displayName: Certified Operators + publisher: Red Hat + image: {{ registry_private_url_with_path_redhat}}/redhat/certified-operator-index:v{{ ocp_release }} + sourceType: grpc +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: {% if redhat_catalogs_prefix | length > 0 %}{{ redhat_catalogs_prefix }}-{% endif %}community-operator-index + namespace: openshift-marketplace +spec: + displayName: Community Operators + publisher: Red Hat + image: {{ registry_private_url_with_path_redhat }}/redhat/community-operator-index:v{{ ocp_release }} + sourceType: grpc +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: {% if redhat_catalogs_prefix | length > 0 %}{{ redhat_catalogs_prefix }}-{% endif %}redhat-operator-index + namespace: openshift-marketplace +spec: + displayName: Red Hat Operators + publisher: Red Hat + image: {{ registry_private_url_with_path_redhat }}/redhat/redhat-operator-index:v{{ ocp_release }} + sourceType: grpc +--- +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: mas-redhat-catalogs{{ idms_suffix_redhat }} + labels: + mas.ibm.com/idmsContent: redhat-catalogs + annotations: + mas.ibm.com/idmsRegistry: "{{ registry_private_url_with_path_redhat }}" + mas.ibm.com/idmsRegistryHost: "{{ registry_private_host }}" + mas.ibm.com/idmsRegistryPort: "{{ registry_private_port }}" + mas.ibm.com/idmsRegistryPrefix: "{{ registry_prefix_redhat }}" + operators.openshift.org/catalog: "true" +spec: + imageDigestMirrors: + - mirrors: + - {{ registry_private_url_with_path_redhat }}/cpopen + source: icr.io/cpopen + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/rhel8 + source: registry.redhat.io/rhel8 + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/rhel9 + source: registry.redhat.io/rhel9 + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/crunchydata + source: registry.connect.redhat.com/crunchydata + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/nvidia + source: registry.connect.redhat.com/nvidia + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/grafana + source: ghcr.io/grafana + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/grafana + source: docker.io/grafana + mirrorSourcePolicy: NeverContactSource + - mirrors: + # This is for the grafana v5 operator bundle image + - {{ registry_private_url_with_path_redhat }}/community-operator-pipeline-prod + source: quay.io/community-operator-pipeline-prod + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/open-telemetry + source: ghcr.io/open-telemetry + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/source-to-image + source: registry.redhat.io/source-to-image + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/odf4 + source: registry.redhat.io/odf4 + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/operator-pipeline-prod + source: quay.io/operator-pipeline-prod + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/strimzi + source: quay.io/strimzi + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/rhceph + source: registry.redhat.io/rhceph + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/amq-streams + source: registry.redhat.io/amq-streams + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/nvidia + source: nvcr.io/nvidia + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/openshift4 + source: registry.redhat.io/openshift4 + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/openshift-community-operators + source: quay.io/openshift-community-operators + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/kubebuilder + source: gcr.io/kubebuilder + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/ubi8 + source: registry.redhat.io/ubi8 + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/ubi9 + source: registry.redhat.io/ubi9 + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/openshift-pipelines + source: registry.redhat.io/openshift-pipelines + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/openshift-serverless-1 + source: registry.redhat.io/openshift-serverless-1 + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/turbonomic + source: registry.connect.redhat.com/turbonomic + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/rh-marketplace + source: quay.io/rh-marketplace + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/rh-marketplace + source: registry.connect.redhat.com/rh-marketplace + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/cert-manager + source: registry.redhat.io/cert-manager + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/lvms4 + source: registry.redhat.io/lvms4 + mirrorSourcePolicy: NeverContactSource diff --git a/ibm/mas_devops/roles/ocp_idms/templates/idms/mas-redhat-release.yml.j2 b/ibm/mas_devops/roles/ocp_idms/templates/idms/mas-redhat-release.yml.j2 new file mode 100644 index 0000000000..303d6ba21e --- /dev/null +++ b/ibm/mas_devops/roles/ocp_idms/templates/idms/mas-redhat-release.yml.j2 @@ -0,0 +1,21 @@ +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: mas-redhat-release{{ idms_suffix_redhat }} + labels: + mas.ibm.com/idmsContent: redhat-release + annotations: + mas.ibm.com/idmsRegistry: "{{ registry_private_url_with_path_redhat }}" + mas.ibm.com/idmsRegistryHost: "{{ registry_private_host }}" + mas.ibm.com/idmsRegistryPort: "{{ registry_private_port }}" + mas.ibm.com/idmsRegistryPrefix: "{{ registry_prefix_redhat}}" +spec: + imageDigestMirrors: + - mirrors: + - {{ registry_private_url_with_path_redhat }}/openshift/release + source: quay.io/openshift-release-dev/ocp-v4.0-art-dev + mirrorSourcePolicy: NeverContactSource + - mirrors: + - {{ registry_private_url_with_path_redhat }}/openshift/release-images + source: quay.io/openshift-release-dev/ocp-release + mirrorSourcePolicy: NeverContactSource diff --git a/ibm/mas_devops/roles/ocp_provision/defaults/main.yml b/ibm/mas_devops/roles/ocp_provision/defaults/main.yml index be7c38fc8d..af9053f465 100644 --- a/ibm/mas_devops/roles/ocp_provision/defaults/main.yml +++ b/ibm/mas_devops/roles/ocp_provision/defaults/main.yml @@ -7,7 +7,7 @@ cluster_platform: "{{lookup('env', 'CLUSTER_PLATFORM') | default('x',true)}}" ocp_version: "{{ lookup('env', 'OCP_VERSION') }}" ocp_fips_enabled: "{{ lookup('env', 'OCP_FIPS_ENABLED') | default('false', true) | bool }}" -default_ocp_version: 4.15 +default_ocp_version: 4.16 supported_cluster_types: - fyre diff --git a/ibm/mas_devops/roles/ocp_provision/tasks/main.yml b/ibm/mas_devops/roles/ocp_provision/tasks/main.yml index 288a907bd5..47aaadb8fd 100644 --- a/ibm/mas_devops/roles/ocp_provision/tasks/main.yml +++ b/ibm/mas_devops/roles/ocp_provision/tasks/main.yml @@ -34,8 +34,8 @@ Wednesday: 4.13 Thursday: 4.12 Friday: 4.15 - Saturday: 4.15 - Sunday: 4.15 + Saturday: 4.16 + Sunday: 4.16 - name: "Set default OCP version" when: ocp_version == "default" diff --git a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml index aa9f04f2d1..c3fcb0851e 100644 --- a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml @@ -1,13 +1,14 @@ --- -# # Install Operator & create entitlement openshift-odh -# # ----------------------------------------------------------------------------- -# - name: "Install Openshift odh Operator" -# ibm.mas_devops.apply_subscription: -# namespace: "{{ openshift_namespace }}" -# package_name: "opendatahub-operator" -# package_channel: "{{ odh_channel }}" -# catalog_source: "{{ odh_catalog_source }}" -# register: subscription +# Install Operator & create entitlement openshift-odh +# ----------------------------------------------------------------------------- +- name: "Install Openshift odh Operator" + ibm.mas_devops.apply_subscription: + namespace: "{{ openshift_namespace }}" + package_name: "opendatahub-operator" + package_channel: "{{ odh_channel }}" + catalog_source: "{{ odh_catalog_source }}" + register: subscription + # # Wait until the Opendata Hub CRD is available @@ -29,7 +30,7 @@ prompt: "Waiting for ODH install plan" - name: Gather facts about Install Plans - k8s_info: + kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 kind: InstallPlan namespace: "{{ openshift_namespace }}" diff --git a/ibm/mas_devops/roles/odh/tasks/pipeline-operator.yml b/ibm/mas_devops/roles/odh/tasks/pipeline-operator.yml index 7fa054424a..193f5074cf 100644 --- a/ibm/mas_devops/roles/odh/tasks/pipeline-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/pipeline-operator.yml @@ -1,6 +1,6 @@ --- - name: Check for pipeline subscriptions if exists - k8s_info: + kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 kind: Subscription namespace: "{{ openshift_namespace }}" diff --git a/ibm/mas_devops/roles/suite_app_upgrade/tasks/check_app_compatibility.yml b/ibm/mas_devops/roles/suite_app_upgrade/tasks/check_app_compatibility.yml index 85d2dbba67..174df98786 100644 --- a/ibm/mas_devops/roles/suite_app_upgrade/tasks/check_app_compatibility.yml +++ b/ibm/mas_devops/roles/suite_app_upgrade/tasks/check_app_compatibility.yml @@ -11,12 +11,13 @@ - "Application Kind ....................... {{ app_info[mas_app_id].kind }}" - "Application CR Identifier .............. {{ mas_instance_id }}/{{ app_info[mas_app_id].kind }}.{{ app_info[mas_app_id].api_version }}" -- name: "{{ mas_app_id }} : Get subscription for ibm-mas" +- name: "{{ mas_app_id }} : Get subscription for {{ mas_app_fqn }}" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "{{ mas_app_fqn }}" - namespace: "{{ mas_app_namespace }}" kind: Subscription + namespace: "{{ mas_app_namespace }}" + label_selectors: + - "operators.coreos.com/{{ mas_app_fqn }}.{{ mas_app_namespace }}" register: app_sub_info - name: "{{ mas_app_id }} : Debug existing Subscription" @@ -46,7 +47,9 @@ fail_msg: "Unable to upgrade from {{ app_sub_info.resources[0].spec.channel }} to {{ mas_app_upgrade_target_channel }}" - name: "{{ mas_app_id }} : Check that install plan approvals are set to 'Automatic'" - when: app_sub_info.resources[0].spec.channel != mas_app_upgrade_target_channel + when: + - app_sub_info.resources[0].spec.channel != mas_app_upgrade_target_channel + - app_sub_info.resources[0].spec.installPlanApproval is defined assert: that: app_sub_info.resources[0].spec.installPlanApproval == 'Automatic' fail_msg: "Automatic install plan approvals must be enabled to upgrade via this role" diff --git a/ibm/mas_devops/roles/suite_app_upgrade/tasks/check_core_compatibility.yml b/ibm/mas_devops/roles/suite_app_upgrade/tasks/check_core_compatibility.yml index 7065a46b67..871b3ade48 100644 --- a/ibm/mas_devops/roles/suite_app_upgrade/tasks/check_core_compatibility.yml +++ b/ibm/mas_devops/roles/suite_app_upgrade/tasks/check_core_compatibility.yml @@ -4,9 +4,10 @@ - name: "core : Get subscription for ibm-mas" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "ibm-mas-operator" - namespace: "{{ mas_namespace }}" kind: Subscription + namespace: "{{ mas_namespace }}" + label_selectors: + - "operators.coreos.com/ibm-mas.{{ mas_namespace }}" register: core_sub_info - name: "core : Debug existing Subscription" @@ -22,7 +23,9 @@ fail_msg: "Unable to upgrade to {{ mas_app_upgrade_target_channel }} because {{ mas_app_fqn }} channel {{ mas_app_upgrade_target_channel }} is not supported with MAS Core {{ core_sub_info.resources[0].spec.channel }}. Supported application channels are: {{ compatibility_matrix[core_sub_info.resources[0].spec.channel][mas_app_id] }}" - name: "core : Check that install plan approvals are set to 'Automatic'" - when: core_sub_info.resources is defined and core_sub_info.resources | length == 1 + when: + - core_sub_info.resources is defined and core_sub_info.resources | length == 1 + - core_sub_info.resources[0].spec.installPlanApproval is defined assert: that: core_sub_info.resources[0].spec.installPlanApproval == 'Automatic' fail_msg: "Automatic install plan approvals must be enabled to upgrade via this role" diff --git a/ibm/mas_devops/roles/suite_app_upgrade/tasks/main.yml b/ibm/mas_devops/roles/suite_app_upgrade/tasks/main.yml index dac1ca2911..42daa0a99b 100644 --- a/ibm/mas_devops/roles/suite_app_upgrade/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_app_upgrade/tasks/main.yml @@ -17,9 +17,10 @@ when: mas_app_channel is not defined or mas_app_channel == "" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "ibm-mas-operator" - namespace: "{{ mas_namespace }}" kind: Subscription + namespace: "{{ mas_namespace }}" + label_selectors: + - "operators.coreos.com/ibm-mas.{{ mas_namespace }}" register: core_sub_info @@ -46,9 +47,10 @@ - name: "{{ mas_app_id }} : Get subscription for {{ mas_app_fqn }}" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "{{ mas_app_fqn }}" - namespace: "{{ mas_app_namespace }}" kind: Subscription + namespace: "{{ mas_app_namespace }}" + label_selectors: + - "operators.coreos.com/{{ mas_app_fqn }}.{{ mas_app_namespace }}" register: app_sub_info diff --git a/ibm/mas_devops/roles/suite_app_upgrade/tasks/upgrade.yml b/ibm/mas_devops/roles/suite_app_upgrade/tasks/upgrade.yml index 6998043faa..924687f255 100644 --- a/ibm/mas_devops/roles/suite_app_upgrade/tasks/upgrade.yml +++ b/ibm/mas_devops/roles/suite_app_upgrade/tasks/upgrade.yml @@ -5,9 +5,10 @@ - name: "Lookup subscription" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "{{ mas_app_fqn }}" - namespace: "{{ mas_app_namespace }}" kind: Subscription + namespace: "{{ mas_app_namespace }}" + label_selectors: + - "operators.coreos.com/{{ mas_app_fqn }}.{{ mas_app_namespace }}" register: suite_app_sub - name: "Assert subscription exists" @@ -25,6 +26,9 @@ definition: spec: channel: "{{ mas_app_upgrade_target_channel }}" + name: "{{ app_sub_info.resources[0].spec.name }}" + source: "{{ app_sub_info.resources[0].spec.source }}" + sourceNamespace: "{{ app_sub_info.resources[0].spec.sourceNamespace }}" apply: true - name: "Pause for 5 minutes before checking upgrade status..." @@ -58,9 +62,9 @@ # ----------------------------------------------------------------------------- # OperatorCondition names are in the format {packageName}.{packageVersion} # We want to strip off the "v" prefix from the version while we do this -- name: "upgrade : Lookup operator version for ibm-mas" +- name: "upgrade : Lookup operator version for {{ mas_app_fqn }}" set_fact: - updated_opcon_version: "{{ updated_opcon.resources[0].metadata.name.split('.v')[1] | ibm.mas_devops.format_pre_version_without_buildid }}" + updated_opcon_version: "{{ updated_opcon.resources[0].metadata.name.split('.v')[1] | ibm.mas_devops.format_pre_version_with_buildid }}" - name: "upgrade : Debug Operator Version" debug: diff --git a/ibm/mas_devops/roles/suite_dns/defaults/main.yaml b/ibm/mas_devops/roles/suite_dns/defaults/main.yaml index 50caf61ff3..b589385b15 100644 --- a/ibm/mas_devops/roles/suite_dns/defaults/main.yaml +++ b/ibm/mas_devops/roles/suite_dns/defaults/main.yaml @@ -6,7 +6,7 @@ supported_dns_providers: - cis - cloudflare - route53 -mas_manual_cert_mgmt: "{{ lookup('env', 'MAS_MANUAL_CERT_MGMT')| default(False, true) }}" +mas_manual_cert_mgmt: "{{ lookup('env', 'MAS_MANUAL_CERT_MGMT')| default('false', true) | bool }}" # Certificate Manager # ----------------------------------------------------------------------------- @@ -50,7 +50,7 @@ cis_email: "{{ lookup('env', 'CIS_EMAIL') }}" cis_apikey: "{{ lookup('env', 'CIS_APIKEY') }}" cis_crn: "{{ lookup('env', 'CIS_CRN') }}" cis_subdomain: "{{ lookup('env', 'CIS_SUBDOMAIN') }}" -cis_enhanced_security: "{{ lookup('env', 'CIS_ENHANCED_SECURITY') | default(false, false) }}" +cis_enhanced_security: "{{ lookup('env', 'CIS_ENHANCED_SECURITY') | default('false', false) | bool }}" # Enhanced IBM CIS DNS Integration Security # ----------------------------------------------------------------------------- @@ -59,11 +59,11 @@ cis_proxy: "{{ lookup('env', 'CIS_PROXY') | default(false, true) }}" cis_service_name: "{{ lookup('env', 'CIS_SERVICE_NAME')}}" # Update DNS entry if it already exists -update_dns: "{{ lookup('env', 'UPDATE_DNS_ENTRIES') | default(true, true) }}" +update_dns: "{{ lookup('env', 'UPDATE_DNS_ENTRIES') | default('true', true) | bool }}" # Force deletion of wildcard dns entries in cis -delete_wildcards: "{{ lookup('env', 'DELETE_WILDCARDS') | default(false, true)}}" +delete_wildcards: "{{ lookup('env', 'DELETE_WILDCARDS') | default('false', true)| bool }}" # Override and delete any existing edge certificates in cis instance -override_edge_certs: "{{ lookup('env', 'OVERRIDE_EDGE_CERTS') | default(true, true)}}" +override_edge_certs: "{{ lookup('env', 'OVERRIDE_EDGE_CERTS') | default('true', true) | bool }}" cis_apiservice: group_name: acme.cis.ibm.com diff --git a/ibm/mas_devops/roles/suite_uninstall/tasks/main.yml b/ibm/mas_devops/roles/suite_uninstall/tasks/main.yml index 181250bc09..88b633eead 100644 --- a/ibm/mas_devops/roles/suite_uninstall/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_uninstall/tasks/main.yml @@ -153,7 +153,7 @@ kind: Subscription namespace: "mas-{{ mas_instance_id }}-core" label_selectors: - - "operators.coreos.com/ibm-mas.mas.mas-{{ mas_instance_id }}-core" + - "operators.coreos.com/ibm-mas.mas-{{ mas_instance_id }}-core" - name: "Delete ibm-truststore-mgr Subscription" kubernetes.core.k8s: diff --git a/ibm/mas_devops/roles/suite_upgrade/tasks/check_app_compatibility.yml b/ibm/mas_devops/roles/suite_upgrade/tasks/check_app_compatibility.yml index ce78793fcb..5a5fa3249e 100644 --- a/ibm/mas_devops/roles/suite_upgrade/tasks/check_app_compatibility.yml +++ b/ibm/mas_devops/roles/suite_upgrade/tasks/check_app_compatibility.yml @@ -8,9 +8,10 @@ - name: "{{ check_app.id }} : Get subscription for ibm-mas-{{ check_app.id }}" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "ibm-mas-{{ check_app.id }}" - namespace: "{{ check_app_namespace }}" kind: Subscription + namespace: "{{ check_app_namespace }}" + label_selectors: + - "operators.coreos.com/ibm-mas-{{ check_app.id }}.{{ check_app_namespace }}" register: checkapp_sub_info - name: "{{ check_app.id }} : Debug existing Subscription" @@ -34,7 +35,9 @@ fail_msg: "Unable to upgrade to {{ mas_channel }} because ibm-mas-{{ check_app.id }} is subscribed to a channel that is not compatible with this release ({{ checkapp_sub_info.resources[0].spec.channel }}). Supported channels are: {{ compatibility_matrix[mas_channel][check_app.id] }}" - name: "{{ check_app.id }} : Check that install plan approvals are set to 'Automatic'" - when: checkapp_sub_info.resources is defined and checkapp_sub_info.resources | length == 1 + when: + - checkapp_sub_info.resources is defined and checkapp_sub_info.resources | length == 1 + - checkapp_sub_info.resources[0].spec.installPlanApproval is defined assert: that: checkapp_sub_info.resources[0].spec.installPlanApproval == 'Automatic' fail_msg: "Automatic install plan approvals must be enabled to upgrade via this role" diff --git a/ibm/mas_devops/roles/suite_upgrade/tasks/check_core_compatibility.yml b/ibm/mas_devops/roles/suite_upgrade/tasks/check_core_compatibility.yml index 7a2c2696b6..22eb13a6ef 100644 --- a/ibm/mas_devops/roles/suite_upgrade/tasks/check_core_compatibility.yml +++ b/ibm/mas_devops/roles/suite_upgrade/tasks/check_core_compatibility.yml @@ -4,9 +4,10 @@ - name: "core : Get subscription for ibm-mas" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "ibm-mas-operator" - namespace: "{{ mas_namespace }}" kind: Subscription + namespace: "{{ mas_namespace }}" + label_selectors: + - "operators.coreos.com/ibm-mas.{{ mas_namespace }}" register: suite_sub_info - name: "core : Debug existing Subscription" @@ -36,7 +37,9 @@ fail_msg: "Unable to upgrade from {{ suite_sub_info.resources[0].spec.channel }} to {{ mas_channel }}" - name: "core : Check that install plan approvals are set to 'Automatic'" - when: suite_sub_info.resources[0].spec.channel != mas_channel + when: + - suite_sub_info.resources[0].spec.channel != mas_channel + - suite_sub_info.resources[0].spec.installPlanApproval is defined assert: that: suite_sub_info.resources[0].spec.installPlanApproval == 'Automatic' fail_msg: "Automatic install plan approvals must be enabled to upgrade via this role" diff --git a/ibm/mas_devops/roles/suite_upgrade/tasks/main.yml b/ibm/mas_devops/roles/suite_upgrade/tasks/main.yml index 0631ada51d..77fce4e358 100644 --- a/ibm/mas_devops/roles/suite_upgrade/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_upgrade/tasks/main.yml @@ -19,9 +19,10 @@ when: mas_channel is not defined or mas_channel == "" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "ibm-mas-operator" - namespace: "{{ mas_namespace }}" kind: Subscription + namespace: "{{ mas_namespace }}" + label_selectors: + - "operators.coreos.com/ibm-mas.{{ mas_namespace }}" register: core_sub_info - name: "Set default upgrade target based on installed version of MAS core" diff --git a/ibm/mas_devops/roles/suite_upgrade/tasks/upgrade.yml b/ibm/mas_devops/roles/suite_upgrade/tasks/upgrade.yml index efa48eda89..0e44906897 100644 --- a/ibm/mas_devops/roles/suite_upgrade/tasks/upgrade.yml +++ b/ibm/mas_devops/roles/suite_upgrade/tasks/upgrade.yml @@ -4,12 +4,15 @@ - name: "upgrade : Update ibm-mas subscription channel" kubernetes.core.k8s: api_version: operators.coreos.com/v1alpha1 - name: "ibm-mas-operator" - namespace: "{{ mas_namespace }}" kind: Subscription + name: "{{ core_sub_info.resources[0].metadata.name }}" + namespace: "{{ mas_namespace }}" definition: spec: channel: "{{ mas_channel }}" + name: "{{ core_sub_info.resources[0].spec.name }}" + source: "{{ core_sub_info.resources[0].spec.source }}" + sourceNamespace: "{{ core_sub_info.resources[0].spec.sourceNamespace }}" apply: true @@ -18,9 +21,10 @@ - name: "upgrade : Get updated subscription for ibm-mas" kubernetes.core.k8s_info: api_version: operators.coreos.com/v1alpha1 - name: "ibm-mas-operator" - namespace: "{{ mas_namespace }}" kind: Subscription + namespace: "{{ mas_namespace }}" + label_selectors: + - "operators.coreos.com/ibm-mas.{{ mas_namespace }}" register: updated_suite_sub_info retries: 20 # about 10 minutes delay: 30 # seconds diff --git a/ibm/mas_devops/roles/uds/tasks/install/udscfg.yml b/ibm/mas_devops/roles/uds/tasks/install/udscfg.yml index 91ed560fe4..c5fe982b71 100644 --- a/ibm/mas_devops/roles/uds/tasks/install/udscfg.yml +++ b/ibm/mas_devops/roles/uds/tasks/install/udscfg.yml @@ -13,7 +13,7 @@ # 1. Lookup the endpoint route # ----------------------------------------------------------------------------- - name: "udscfg : Lookup UDS endpoint Route" - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Route name: uds-endpoint @@ -28,7 +28,7 @@ # 2. Lookup the UDS API key # ----------------------------------------------------------------------------- - name: "udscfg : Lookup uds-api-key Secret" - k8s_info: + kubernetes.core.k8s_info: api_version: v1 kind: Secret name: uds-api-key diff --git a/mkdocs.yml b/mkdocs.yml index 1fad7b80f4..4dd50dab73 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -72,7 +72,7 @@ nav: - "mirror_extras_prepare": roles/mirror_extras_prepare.md - "mirror_images": roles/mirror_images.md - "mirror_ocp": roles/mirror_ocp.md - - "ocp_contentsourcepolicy": roles/ocp_contentsourcepolicy.md + - "ocp_idms": roles/ocp_idms.md - "ocp_simulate_disconnected_network": roles/ocp_simulate_disconnected_network.md - "registry": roles/registry.md - "Roles: Suite Mgmt": From 7616c334c2b1d1e431099942a39e31a4be2ca10e Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 19 Dec 2024 15:59:48 +0000 Subject: [PATCH 041/169] [patch] update odh to install specific version --- .../roles/odh/tasks/serverless-operator.yml | 36 +++++++++++-------- .../templates/serverless/subscription.yml.j2 | 17 +++++++++ 2 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 ibm/mas_devops/roles/odh/templates/serverless/subscription.yml.j2 diff --git a/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml b/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml index 727860b114..c048848367 100644 --- a/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml @@ -1,18 +1,24 @@ --- -# 1. Install Operator & create entitlement openshift-serverless -# ----------------------------------------------------------------------------- -- name: "Install Openshift Serverless Operator" - ibm.mas_devops.apply_subscription: - namespace: "{{ serverless_namespace }}" - package_name: "serverless-operator" - package_channel: "{{ serverless_channel }}" - catalog_source: "{{ serverless_catalog_source }}" - register: subscription +# # 1. Install Operator & create entitlement openshift-serverless +# # ----------------------------------------------------------------------------- +# - name: "Install Openshift Serverless Operator" +# ibm.mas_devops.apply_subscription: +# namespace: "{{ serverless_namespace }}" +# package_name: "serverless-operator" +# package_channel: "{{ serverless_channel }}" +# catalog_source: "{{ serverless_catalog_source }}" +# register: subscription + +# # 2. Wait until the Serverless CRD is available +# # ----------------------------------------------------------------------------- +# - name: "Wait until the Serverless CRD is available" +# include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" +# vars: +# crd_name: knativeservings.operator.knative.dev -# 2. Wait until the Serverless CRD is available -# ----------------------------------------------------------------------------- -- name: "Wait until the Serverless CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: knativeservings.operator.knative.dev +# enable following role temporary until find a way to use openshift-serverless and switch to apply_subscription role +- name: "Install Openshift Serverless Operator" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/serverless/subscription.yml.j2') }}" diff --git a/ibm/mas_devops/roles/odh/templates/serverless/subscription.yml.j2 b/ibm/mas_devops/roles/odh/templates/serverless/subscription.yml.j2 new file mode 100644 index 0000000000..62c2f67856 --- /dev/null +++ b/ibm/mas_devops/roles/odh/templates/serverless/subscription.yml.j2 @@ -0,0 +1,17 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: operatorgroup + namespace: openshift-serverless +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: serverless-operator + namespace: openshift-serverless +spec: + channel: stable + installPlanApproval: Automatic + name: serverless-operator + source: redhat-operators + sourceNamespace: openshift-marketplace From 802445418385e63147a4f2dd7352f73ace083471 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 20 Dec 2024 10:44:15 +0000 Subject: [PATCH 042/169] [patch] refactor storage classes for kmodel --- ibm/mas_devops/roles/kmodels/defaults/main.yml | 3 +++ .../roles/kmodels/tasks/determine-storage-classes.yml | 10 +++++----- .../kmodels/templates/kmodels-controller/pvc.yml.j2 | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ibm/mas_devops/roles/kmodels/defaults/main.yml b/ibm/mas_devops/roles/kmodels/defaults/main.yml index bb27ac82e7..c1c468d5b6 100644 --- a/ibm/mas_devops/roles/kmodels/defaults/main.yml +++ b/ibm/mas_devops/roles/kmodels/defaults/main.yml @@ -21,3 +21,6 @@ mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" #km-s3-secret variables aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + +# storage class +primary_storage_class: "{{ lookup('env', 'PRIMARY_STORAGE_CLASS') }}" diff --git a/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml index 1bfe8994a2..a8661d67f2 100644 --- a/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml @@ -8,15 +8,15 @@ # 2. Set Primary Storage (Required) # ----------------------------------------------------------------------------- - name: "determine-storage-classes : Default Primary Storage if not set by user" - when: cpd_primary_storage_class is not defined or cpd_primary_storage_class == "" + when: primary_storage_class is not defined or primary_storage_class == "" set_fact: - cpd_primary_storage_class: "{{ defaultStorageClasses.rwx }}" + primary_storage_class: "{{ defaultStorageClasses.rwx }}" - name: "determine-storage-classes : Assert that primary storage class has been defined" assert: - that: cpd_primary_storage_class is defined and cpd_primary_storage_class != "" - fail_msg: "cpd_primary_storage_class must be defined" + that: primary_storage_class is defined and primary_storage_class != "" + fail_msg: "primary_storage_class must be defined" - name: Set primary StorageClass debug: - msg: "Primary storage class ................. {{ cpd_primary_storage_class }}" + msg: "Primary storage class ................. {{ primary_storage_class }}" diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/pvc.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/pvc.yml.j2 index 167d97bfec..a130bd3e11 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/pvc.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/pvc.yml.j2 @@ -11,4 +11,4 @@ spec: resources: requests: storage: 1Gi - storageClassName: "{{ cpd_primary_storage_class }}" + storageClassName: "{{ primary_storage_class }}" From c7019395aa9d532c3bc59a761eb7dac22d96c94c Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 2 Jan 2025 13:11:08 +0000 Subject: [PATCH 043/169] [patch] add specific ODH version deployment --- .../roles/odh/tasks/odh-operator.yml | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml index c3fcb0851e..21badf8ebf 100644 --- a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml @@ -1,15 +1,13 @@ --- -# Install Operator & create entitlement openshift-odh -# ----------------------------------------------------------------------------- -- name: "Install Openshift odh Operator" - ibm.mas_devops.apply_subscription: - namespace: "{{ openshift_namespace }}" - package_name: "opendatahub-operator" - package_channel: "{{ odh_channel }}" - catalog_source: "{{ odh_catalog_source }}" - register: subscription - - +# # Install Operator & create entitlement openshift-odh +# # ----------------------------------------------------------------------------- +# - name: "Install Openshift odh Operator" +# ibm.mas_devops.apply_subscription: +# namespace: "{{ openshift_namespace }}" +# package_name: "opendatahub-operator" +# package_channel: "{{ odh_channel }}" +# catalog_source: "{{ odh_catalog_source }}" +# register: subscription # # Wait until the Opendata Hub CRD is available # # ----------------------------------------------------------------------------- From b6a7115c550cd608305b5cb7f2c1db425ffe1d07 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 2 Jan 2025 14:41:40 +0000 Subject: [PATCH 044/169] [patch] update kmodels versions --- ibm/mas_devops/roles/kmodels/vars/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ibm/mas_devops/roles/kmodels/vars/main.yml b/ibm/mas_devops/roles/kmodels/vars/main.yml index 64d85b6b5d..7225fe182c 100644 --- a/ibm/mas_devops/roles/kmodels/vars/main.yml +++ b/ibm/mas_devops/roles/kmodels/vars/main.yml @@ -2,13 +2,13 @@ # vars file for kmodels pullSecretName: "ibm-entitlement" # controller -mas_aibroker_controller_tag: 1.0.2-pre.dev10x +mas_aibroker_controller_tag: 1.0.2 controller: "aibroker/controller:{{ mas_aibroker_controller_tag }}" # store -mas_aibroker_store_tag: 1.0.1 +mas_aibroker_store_tag: 1.0.2 store: "aibroker/store:{{ mas_aibroker_store_tag }}" # watcher -mas_aibroker_watcher_tag: 1.0.1 +mas_aibroker_watcher_tag: 1.0.2 watcher: "aibroker/watcher:{{ mas_aibroker_watcher_tag }}" mas_aibroker_connector_tag: 1.0.0 From 2886bb0fdd1453ce494aab255284987272623c09 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 2 Jan 2025 16:43:16 +0000 Subject: [PATCH 045/169] [patch] update docs --- docs/playbooks/oneclick-aibroker.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index 9695f0e1ec..4bb83b3798 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -7,9 +7,9 @@ You will need a RedHat OpenShift v4.14 or above with IBM Maximo Application Suit ### Dependencies: * Object Storage - + Minio ( if customer does not want to use AWS S3 bucket) - + AWS S3 ( if customer use AWS S3 bucket bucket) -* MariaDB database (installed in cluster where aibroker instance) + + Minio (installed on the same cluster what aibroker) + + AWS S3 (if customer use AWS S3 bucket bucket) buckets needs to have unique names +* MariaDB database (installed in cluster where aibroker instance) or in AWS * IBM Maximo Application Suite Core v9.x ## Overview @@ -174,7 +174,7 @@ export MAS_CONFIG_DIR="config_path_location" export MAS_AIBROKER_SAAS="true" export MAS_AIBROKER_DOMAIN="apps.domain" export MAS_AIBROKER_SLS_URL="https://sls.ibm-sls.ibm-sls."${MAS_AIBROKER_DOMAIN} -export MAS_AIBROKER_SLS_REGISTRATION_KEY="xxxxxxx" +export MAS_AIBROKER_SLS_REGISTRATION_KEY="xxxxxxx" export MAS_AIBROKER_DRO_URL="https://ibm-data-reporter-redhat-marketplace."${MAS_AIBROKER_DOMAIN} export MAS_AIBROKER_DRO_TOKEN="xxx" export DB2_INSTANCE_NAME="aibroker" From 099c23110ed837eda13f8da8334995cab15860f6 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 3 Jan 2025 09:58:12 +0000 Subject: [PATCH 046/169] [patch] update doc for AI Broker --- docs/playbooks/oneclick-aibroker.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index 4bb83b3798..5ba17b998e 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -202,6 +202,10 @@ oc login --token=xxxx --server=https://myocpserver ansible-playbook playbooks/oneclick_add_aibroker.yml ``` +- `MAS_AIBROKER_SLS_REGISTRATION_KEY` - value can be found in `ibm-sls` namespace, in pod `sls-api-licensing-85699fb57-9lmrq` please look in environments tab, then value `REGISTRATION_KEY` +- `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` + + ## NOTICE: playbook oneclick_add_aibroker.yml will run three roles: ### Role: odh From c40f78d6d5c2ca7255135e70370749c44d7d95c2 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 3 Jan 2025 10:58:00 +0000 Subject: [PATCH 047/169] [patch] update aibroker docs --- docs/playbooks/oneclick-aibroker.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index 5ba17b998e..8b4ff94d3e 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -101,6 +101,7 @@ python3 -m pip install boto3 #### Run playbooks for deploy AI Broker from internal registry ex. `docker-na-public.artifactory.swg-devops.com` ```bash +export MAS_CATALOG_VERSION="" export ARTIFACTORY_USERNAME="" export ARTIFACTORY_TOKEN="" export MAS_ICR_CP="" @@ -134,6 +135,7 @@ ansible-playbook playbooks/oneclick_add_aibroker.yml #### Run playbooks for deploy AI Broker from public registry ex. `icr.io` ```bash +export MAS_CATALOG_VERSION="" export MAS_ENTITLEMENT_USERNAME="" export MAS_ENTITLEMENT_KEY="" export MAS_INSTANCE_ID="" @@ -165,6 +167,7 @@ ansible-playbook playbooks/oneclick_add_aibroker.yml #### Run playbooks for deploy AI Broker on SAAS ```bash +export MAS_CATALOG_VERSION="v9-master-amd64" export MAS_ENTITLEMENT_USERNAME="" export MAS_ENTITLEMENT_KEY="" export IBM_ENTITLEMENT_KEY=${MAS_ENTITLEMENT_KEY} From 2557d5531f107b5e66fc02df485aae14a2c96200 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 3 Jan 2025 12:06:31 +0000 Subject: [PATCH 048/169] [patch] update fixed version of watcher --- ibm/mas_devops/roles/kmodels/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/kmodels/vars/main.yml b/ibm/mas_devops/roles/kmodels/vars/main.yml index 7225fe182c..43f34dece6 100644 --- a/ibm/mas_devops/roles/kmodels/vars/main.yml +++ b/ibm/mas_devops/roles/kmodels/vars/main.yml @@ -8,7 +8,7 @@ controller: "aibroker/controller:{{ mas_aibroker_controller_tag }}" mas_aibroker_store_tag: 1.0.2 store: "aibroker/store:{{ mas_aibroker_store_tag }}" # watcher -mas_aibroker_watcher_tag: 1.0.2 +mas_aibroker_watcher_tag: 1.0.2-pre.dev10x watcher: "aibroker/watcher:{{ mas_aibroker_watcher_tag }}" mas_aibroker_connector_tag: 1.0.0 From cf8ca7bbafc5ac3c73493fd2ffc1d1965a30feb5 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 3 Jan 2025 12:20:15 +0000 Subject: [PATCH 049/169] [patch] update watcher config map --- .../kmodels/templates/kmodels-watcher/configmap.yml.j2 | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 index 5d8cc93e59..b8893fd874 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 @@ -7,12 +7,6 @@ metadata: labels: app: km-watcher data: - EXCLUDE_POD_NAMESPACES: "openshift-operators,openshift-pipelines" WATCHER_SENDER_DELAY: "5" - EXCLUDE_JOB_NAMESPACES: "ibm-cpd" PIPELINES: mas-{{ mas_instance_id }}-aibroker -{% if mas_aibroker_saas %} - TENANTS: provision-tenant -{% else %} - TENANTS: {{ tenantNamespace }} -{% endif %} + TENANTS_BUCKET: {{ mas_aibroker_storage_tenants_bucket }} From 4e6ed0b0b4f29b55eccdd2d5948ba589ba6708c9 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 09:50:52 +0000 Subject: [PATCH 050/169] [patch] comment out creation of server runtime --- .../aibroker/tasks/tenant/install/main.yml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index d930f93408..1499a15065 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -33,25 +33,25 @@ apply: yes definition: "{{ lookup('template', 'templates/tenant/km-s3-sa.yml.j2') }}" -- name: 'Create huggingfaceserver server runtime' - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-huggingfaceserver.yml.j2') }}" - -- name: 'Create lgbserver server runtime' - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-lgbserver.yml.j2') }}" - -- name: 'Create sklearnserver server runtime' - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-sklearnserver.yml.j2') }}" - -- name: 'Create xgbserver server runtime' - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-xgbserver.yml.j2') }}" +# - name: 'Create huggingfaceserver server runtime' +# kubernetes.core.k8s: +# apply: yes +# definition: "{{ lookup('template', 'templates/tenant/kserve-huggingfaceserver.yml.j2') }}" + +# - name: 'Create lgbserver server runtime' +# kubernetes.core.k8s: +# apply: yes +# definition: "{{ lookup('template', 'templates/tenant/kserve-lgbserver.yml.j2') }}" + +# - name: 'Create sklearnserver server runtime' +# kubernetes.core.k8s: +# apply: yes +# definition: "{{ lookup('template', 'templates/tenant/kserve-sklearnserver.yml.j2') }}" + +# - name: 'Create xgbserver server runtime' +# kubernetes.core.k8s: +# apply: yes +# definition: "{{ lookup('template', 'templates/tenant/kserve-xgbserver.yml.j2') }}" # Applay RBAC roles - name: 'Apply RBAC for tenant' From 4400f3f6f9a952494cffc9ed6f5b13286c796930 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 14:53:10 +0000 Subject: [PATCH 051/169] [patch] update create sls script --- .../roles/aibroker/files/alm_sample_sls_use.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py index cd88322c75..c80fe83f68 100644 --- a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py +++ b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py @@ -99,7 +99,7 @@ def generate_api_key(): headers = {'X-Registration-Key': slsRegistrationKey} # TODO: Rawa - Swagger docs don't document the need for this critical header #print ("Making initial registration request for client "+slsClientId) - response = requests.post(slsUrl+"/api/registrations",verify=caCertificateForSSL,json=data, headers=headers ) + response = requests.post(slsUrl+"/api/registrations",verify=False,json=data, headers=headers ) response.raise_for_status() registrationId = response.json()['registrationId'] @@ -111,7 +111,7 @@ def generate_api_key(): provisioningWaiting = True while provisioningWaiting: headers = {'X-Registration-Key': slsRegistrationKey} - response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=caCertificateForSSL,headers=headers ) + response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=False,headers=headers ) response.raise_for_status() status = response.json()['state'] print("Waiting for client provisioning to be completed. Current status: %s" %status) @@ -133,7 +133,7 @@ def generate_api_key(): # The next step is to confirm the client. Poll status until complete. confirmationWaiting = True while confirmationWaiting: - response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL, cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=False, cert=(clientTlsCrtPath,clientTlsKeyPath)) #print (response) status = response.json()['state'] #print("Waiting for client registration to be confirmed. Current status: %s" %status) @@ -152,12 +152,12 @@ def generate_api_key(): def spendAppPoints(quantity): #print ("Let's spend "+str(quantity)+" AppPoints") data = {'validity': 86400, 'quantity': quantity} - response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() # String representation of AppPoint usage by your client. TODO: SLS will shortly be providing an update to correctly report usage for your client. At present, it returns the overall usage across all clients. def getUsage(): - response = requests.get(slsUrl+"/api/tokens/",verify=caCertificateForSSL,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner + response = requests.get(slsUrl+"/api/tokens/",verify=False,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner response.raise_for_status() j = json.loads(response.content.decode('utf-8')) return j[0]['used'] @@ -240,12 +240,12 @@ def getUsage(): # At deprovision, return your licenses and unregister the client print ("Preparing for deprovision: returning any AppPoints to the pool") - response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() print("Usage after deprovisioning: "+str(getUsage())) print ("Preparing for deprovision: unregistering the SLS client") - response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() #print ("Deleting redundant client certificates") os.remove(clientTlsCrtPath) From e80087dc5918eb89466afee1d69db057ab3931d9 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 15:11:51 +0000 Subject: [PATCH 052/169] [patch] add debug print --- ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py index c80fe83f68..f0238f1f3b 100644 --- a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py +++ b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py @@ -117,7 +117,8 @@ def generate_api_key(): print("Waiting for client provisioning to be completed. Current status: %s" %status) if status=="AWAITING_CONFIRMATION": # We are now ready to proceed; client certs are available - #print ("Extracting client certificates") + print ("Extracting client certificates") + print(clientTlsCrtPath) clientTlskey= base64.b64decode(response.json()['certs']['tls.key']) clientTlscrt= base64.b64decode(response.json()['certs']['tls.crt']) clientCacrt= base64.b64decode(response.json()['certs']['ca.crt']) From d8d8049b138d5148d765f50a8478dc40c5f3e5a6 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 15:15:38 +0000 Subject: [PATCH 053/169] [patch] update path temp --- ibm/mas_devops/roles/aibroker/vars/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 98a1bc7935..a984813088 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -48,4 +48,5 @@ mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', tr mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" -mas_aibroker_path_ca_crt: "./" +# mas_aibroker_path_ca_crt: "./" +mas_aibroker_path_ca_crt: "/home/karol/git/ansible-devops/ibm/mas_devops/playbooks/certs" From a89da6b83de47615c7b12e51f524cfab97299f99 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 15:29:22 +0000 Subject: [PATCH 054/169] [patch] add debug --- ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py | 2 ++ ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py index f0238f1f3b..b3507164e0 100644 --- a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py +++ b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py @@ -69,6 +69,8 @@ def generate_api_key(): # Prepare the SLS clientId to use, and prepare for storing client certificates that are generated at registration time. slsClientId=offeringName+"-"+instanceIdentifier certs_dir=f"{os.getcwd()}/certs" +print("xxxxxxx") +print(certs_dir) clientCaCrtPath=certs_dir+"/"+slsClientId+"-ca.crt" clientTlsCrtPath=certs_dir+"/"+slsClientId+"-tls.crt" clientTlsKeyPath=certs_dir+"/"+slsClientId+"-tls.key" diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index e3debc887b..df018ea236 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -13,6 +13,8 @@ if [ -z ${TENANT} ]; then fi cwd=$(pwd) +echo "#####" +echo "location pwd in script create secret " echo $cwd mkdir -p certs From f5095322fec902ec71b3f710abccc59c6ef24e0b Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 15:34:49 +0000 Subject: [PATCH 055/169] [patch] update --- ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index df018ea236..87932f6aa5 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -39,6 +39,11 @@ SECRET_NAME=${TENANT}----sls-secret CA_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-ca.crt TLS_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.crt KEY_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.key +echo "#####" +echo $SECRET_NAME +echo $CA_FILE_PATH +echo $TLS_FILE_PATH +echo $KEY_FILE_PATH # Create the secret oc create secret generic $SECRET_NAME -n $NAMESPACE \ From d63acca8742c7f10c1776370184471dd95d21751 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 15:37:59 +0000 Subject: [PATCH 056/169] [patch] update --- ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index 87932f6aa5..e50c827e34 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -27,7 +27,7 @@ echo "SLS registration is created successfully." registrationKey=$4 - +echo "----------" echo $instanceIdentifier # Set variables # Need to change From bd3cf446115f0bf7ef7d0bfbb9c2b19c63eef905 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 15:44:47 +0000 Subject: [PATCH 057/169] [patch] update --- .../roles/aibroker/files/alm_sample_sls_use.py | 18 +++++++++--------- .../roles/aibroker/files/create_sls_secret.sh | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py index b3507164e0..95d05f102f 100644 --- a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py +++ b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py @@ -17,10 +17,10 @@ slsRegistrationKey = sys.argv[2] caCertificateForSSL = sys.argv[3] -print('slsUrl='+slsUrl) -print(slsRegistrationKey) -print("caCertificateForSSL") -print(caCertificateForSSL) +# print('slsUrl='+slsUrl) +# print(slsRegistrationKey) +# print("caCertificateForSSL") +# print(caCertificateForSSL) # -------------------------------------------------------------------------------------------------------------------------------- # - # - Configuration section - you need to complete steps 1-5 @@ -69,8 +69,8 @@ def generate_api_key(): # Prepare the SLS clientId to use, and prepare for storing client certificates that are generated at registration time. slsClientId=offeringName+"-"+instanceIdentifier certs_dir=f"{os.getcwd()}/certs" -print("xxxxxxx") -print(certs_dir) +# print("xxxxxxx") +# print(certs_dir) clientCaCrtPath=certs_dir+"/"+slsClientId+"-ca.crt" clientTlsCrtPath=certs_dir+"/"+slsClientId+"-tls.crt" clientTlsKeyPath=certs_dir+"/"+slsClientId+"-tls.key" @@ -96,7 +96,7 @@ def generate_api_key(): pass # Make initial registration request - print("will use certificate for client provisioning ") + # print("will use certificate for client provisioning ") data = {'clientId': slsClientId, 'description': 'ALM sample client registration', 'role': 'manage'} # TODO: Rawa - Swagger docs incorrectly state 'id' rather than 'clientId' headers = {'X-Registration-Key': slsRegistrationKey} # TODO: Rawa - Swagger docs don't document the need for this critical header @@ -119,8 +119,8 @@ def generate_api_key(): print("Waiting for client provisioning to be completed. Current status: %s" %status) if status=="AWAITING_CONFIRMATION": # We are now ready to proceed; client certs are available - print ("Extracting client certificates") - print(clientTlsCrtPath) + # print ("Extracting client certificates") + # print(clientTlsCrtPath) clientTlskey= base64.b64decode(response.json()['certs']['tls.key']) clientTlscrt= base64.b64decode(response.json()['certs']['tls.crt']) clientCacrt= base64.b64decode(response.json()['certs']['ca.crt']) diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index e50c827e34..ff9612db3e 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -29,6 +29,7 @@ registrationKey=$4 echo "----------" echo $instanceIdentifier +echo "----------" # Set variables # Need to change NAMESPACE=mas-${instance_id}-aibroker From b4187128f6900bc2887893b77a43b922738d53cc Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 15:49:04 +0000 Subject: [PATCH 058/169] [patch] remove debug commands --- ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py index 95d05f102f..666dc461ef 100644 --- a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py +++ b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py @@ -116,7 +116,7 @@ def generate_api_key(): response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=False,headers=headers ) response.raise_for_status() status = response.json()['state'] - print("Waiting for client provisioning to be completed. Current status: %s" %status) + # print("Waiting for client provisioning to be completed. Current status: %s" %status) if status=="AWAITING_CONFIRMATION": # We are now ready to proceed; client certs are available # print ("Extracting client certificates") @@ -141,7 +141,7 @@ def generate_api_key(): status = response.json()['state'] #print("Waiting for client registration to be confirmed. Current status: %s" %status) if status=="REGISTERED": - print ("Client registration confirmed.") + # print ("Client registration confirmed.") confirmationWaiting = False time.sleep(1) From 58c990e244e9c9281d699d80e4d26c6e4459d00c Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 15:59:53 +0000 Subject: [PATCH 059/169] [patch] cleanup and added fixes --- .../roles/aibroker/files/create_sls_secret.sh | 12 ++---------- ibm/mas_devops/roles/aibroker/vars/main.yml | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index ff9612db3e..d6c0e7dac6 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -13,9 +13,7 @@ if [ -z ${TENANT} ]; then fi cwd=$(pwd) -echo "#####" echo "location pwd in script create secret " -echo $cwd mkdir -p certs @@ -26,12 +24,11 @@ echo "SLS registration is created successfully." registrationKey=$4 - echo "----------" echo $instanceIdentifier echo "----------" # Set variables -# Need to change +# Need to change NAMESPACE=mas-${instance_id}-aibroker slsClientId="aibroker"-$instanceIdentifier echo $slsClientId @@ -40,15 +37,10 @@ SECRET_NAME=${TENANT}----sls-secret CA_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-ca.crt TLS_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.crt KEY_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.key -echo "#####" -echo $SECRET_NAME -echo $CA_FILE_PATH -echo $TLS_FILE_PATH -echo $KEY_FILE_PATH # Create the secret oc create secret generic $SECRET_NAME -n $NAMESPACE \ - --from-file=$CA_FILE_PATH \ + --from-file=$CA_FILE_PATH \ --from-file=$TLS_FILE_PATH \ --from-file=$KEY_FILE_PATH \ --from-literal=SLS_CLIENT_ID=${slsClientId} \ diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index a984813088..63e2970550 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -49,4 +49,4 @@ mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', tru mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" # mas_aibroker_path_ca_crt: "./" -mas_aibroker_path_ca_crt: "/home/karol/git/ansible-devops/ibm/mas_devops/playbooks/certs" +mas_aibroker_path_ca_crt: "./certs" From d8d80a01e94bc655ca68ad7b1731b469826c55f3 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 16:14:28 +0000 Subject: [PATCH 060/169] [patch] remove echos --- .../roles/aibroker/files/create_sls_secret.sh | 10 +++++----- ibm/mas_devops/roles/aibroker/vars/main.yml | 11 ++++------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh index d6c0e7dac6..f942f3e026 100644 --- a/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker/files/create_sls_secret.sh @@ -20,18 +20,18 @@ mkdir -p certs echo "creating SLS registration, please wait....." # instanceIdentifier=`python3 {{ role_path }}/files/alm_sample_sls_use.py $3 $4` instanceIdentifier=$(python3 ../roles/aibroker/files/alm_sample_sls_use.py $3 $4 $5) -echo "SLS registration is created successfully." +# echo "SLS registration is created successfully." registrationKey=$4 -echo "----------" -echo $instanceIdentifier -echo "----------" +# echo "----------" +# echo $instanceIdentifier +# echo "----------" # Set variables # Need to change NAMESPACE=mas-${instance_id}-aibroker slsClientId="aibroker"-$instanceIdentifier -echo $slsClientId +# echo $slsClientId SECRET_NAME=${TENANT}----sls-secret CA_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-ca.crt diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 63e2970550..28fdd4c4c6 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -1,18 +1,17 @@ --- # aibroker vars -aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" +aibroker_namespace: 'mas-{{ mas_instance_id }}-aibroker' mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" -pullSecretName: "ibm-entitlement" - +pullSecretName: 'ibm-entitlement' # Whether to perform an install or remove tenant tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" # Create Api Key vars mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" -tenantNamespace: "aibroker-{{ mas_aibroker_tenant_name }}" +tenantNamespace: 'aibroker-{{ mas_aibroker_tenant_name }}' # app_domain: "{{ lookup('env', 'APP_DOMAIN') }}" mas_aibroker_apikey_action: "{{ lookup('env', 'MAS_AIBROKER_APIKEY_ACTION') | default('install', true) }}" @@ -44,9 +43,7 @@ mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | defa mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" -# mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" -# mas_aibroker_path_ca_crt: "./" -mas_aibroker_path_ca_crt: "./certs" +mas_aibroker_path_ca_crt: './certs' From 6fb1b6b23dac4af961f5de0a536b94bfcf6be326 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 7 Jan 2025 17:16:51 +0000 Subject: [PATCH 061/169] [patch] update starting year --- ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 1499a15065..1c351cc4a5 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -177,7 +177,7 @@ Content-Type: "application/json" tenantid: "provision-tenant" body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"similarity\", \"entitlement_start_date\": \"2024-01-01\", \"entitlement_end_date\": \"2025-01-01\"}" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"similarity\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" status_code: [200] timeout: 30 register: add_tenant_entitlements_to_db2 From 3cc5dda952dfd0009c32615e019ecd515d7ce10f Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 8 Jan 2025 10:25:20 +0000 Subject: [PATCH 062/169] [patch] add km-s3 secret to watcher deployment --- .../roles/kmodels/templates/kmodels-watcher/deployment.yml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/deployment.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/deployment.yml.j2 index fc5d80c6b9..9bad336b60 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/deployment.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/deployment.yml.j2 @@ -62,3 +62,5 @@ spec: envFrom: - configMapRef: name: km-watcher + - secretRef: + name: km-s3-secret From 476f5680c658f88bed1759ce07889123130eda00 Mon Sep 17 00:00:00 2001 From: roshi-dubey Date: Thu, 9 Jan 2025 15:49:08 +0530 Subject: [PATCH 063/169] [patch] update km-s3-secret (#1617) --- .../roles/aibroker/tasks/tenant/install/main.yml | 4 ++-- ibm/mas_devops/roles/kmodels/tasks/main.yml | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 1c351cc4a5..317afba945 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -138,7 +138,7 @@ tenantid: "provision-tenant" body_format: "json" body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"sls_url\": \"{{ mas_aibroker_sls_url }}\", \"dro_url\": \"{{ mas_aibroker_dro_url }}\"}" - status_code: [200] + status_code: [201] timeout: 30 register: add_tenant_to_db2 @@ -178,7 +178,7 @@ tenantid: "provision-tenant" body_format: "json" body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"similarity\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" - status_code: [200] + status_code: [201] timeout: 30 register: add_tenant_entitlements_to_db2 diff --git a/ibm/mas_devops/roles/kmodels/tasks/main.yml b/ibm/mas_devops/roles/kmodels/tasks/main.yml index 26aac36322..3a2256e58a 100644 --- a/ibm/mas_devops/roles/kmodels/tasks/main.yml +++ b/ibm/mas_devops/roles/kmodels/tasks/main.yml @@ -13,6 +13,15 @@ kubernetes.core.k8s: template: "templates/namespace.yml.j2" +- name: "Delete km-s3-secret secret if it exists" + kubernetes.core.k8s: + state: absent + api_version: v1 + kind: Secret + name: km-s3-secret + namespace: mas-{{ mas_instance_id }}-aibroker + ignore_errors: true + - name: "Create km-s3-secret secret" kubernetes.core.k8s: definition: From 406f9d3000ce1bd90d313fa7fa7d107ace5bf1f3 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 9 Jan 2025 14:35:50 +0000 Subject: [PATCH 064/169] [patch] add provision tenant debug for investigation --- ibm/mas_devops/roles/aibroker/tasks/saas/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml index c6e12dc92c..89a4894a1f 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml @@ -52,6 +52,10 @@ set_fact: provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" +- name: Debug provision-tenant api key #TODO: remove on finall commit + debug: + msg: "Provision tenant api key: {{ provision_tenant_api_key }}" + # - name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" # shell: | # curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ From 5dc11640b6752fd433492b21e0862a626afb8a09 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 10 Jan 2025 13:13:17 +0000 Subject: [PATCH 065/169] [patch] update docs, and wait for service ready --- docs/playbooks/oneclick-aibroker.md | 37 +++++++++++++++++++ .../aibroker/tasks/tenant/install/main.yml | 5 +++ 2 files changed, 42 insertions(+) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index 8b4ff94d3e..fb161be730 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -207,6 +207,7 @@ ansible-playbook playbooks/oneclick_add_aibroker.yml - `MAS_AIBROKER_SLS_REGISTRATION_KEY` - value can be found in `ibm-sls` namespace, in pod `sls-api-licensing-85699fb57-9lmrq` please look in environments tab, then value `REGISTRATION_KEY` - `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` +- in `AWS` for `MAS_AIBROKER_STORAGE_PIPELINES_BUCKET`, `MAS_AIBROKER_STORAGE_TENANTS_BUCKET`, `MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET` user need to create S3 buckets with unique name ## NOTICE: playbook oneclick_add_aibroker.yml will run three roles: @@ -338,3 +339,39 @@ export MAS_AIBROKER_WATSONX_ACTION="remove" export ROLE_NAME="aibroker" oc login --token=xxxx --server=https://myocpserver ansible-playbook playbooks/run_role.yml +``` + +# create tanant SAAS + +```bash +export MAS_AIBROKER_TENANT_NAME="tenant1" +export TENANT_ACTION="install" +export MAS_INSTANCE_ID="aibdev" +export MAS_AIBROKER_DOMAIN="apps.aibrokerdev.cp.fyre.ibm.com" +export MAS_AIBROKER_SLS_URL="https://sls.ibm-sls.ibm-sls."${MAS_AIBROKER_DOMAIN} +export MAS_AIBROKER_DRO_URL="https://ibm-data-reporter-redhat-marketplace."${MAS_AIBROKER_DOMAIN} +export MAS_AIBROKER_SLS_REGISTRATION_KEY="" +export MAS_AIBROKER_DRO_TOKEN="" +export MAS_AIBROKER_SAAS="true" +export MAS_AIBROKER_STORAGE_ACCESSKEY="" +export MAS_AIBROKER_STORAGE_SECRETKEY="" +export MAS_AIBROKER_STORAGE_HOST="" +export MAS_AIBROKER_STORAGE_SSL="true or false" +export MAS_AIBROKER_STORAGE_REGION="" +export MAS_AIBROKER_STORAGE_PROVIDER="" +export MAS_AIBROKER_STORAGE_PORT="" +export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" +export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" +export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" +export MAS_AIBROKER_WATSONXAI_APIKEY="" +export MAS_AIBROKER_WATSONXAI_URL="" +export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" +export ROLE_NAME="aibroker" +ansible-playbook playbooks/run_role.yml +``` + +- `MAS_AIBROKER_SLS_REGISTRATION_KEY` - value can be found in `ibm-sls` namespace, in pod `sls-api-licensing-85699fb57-9lmrq` please look in environments tab, then value `REGISTRATION_KEY` +- `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` + + +**NOTE:** for create addidional tenants we don't need to specify buckets diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 317afba945..542b07536b 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -126,6 +126,11 @@ # --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' # register: add_tenant_to_db2 +- name: Wait for service /service/rest/api/v1/tenant + pause: + minutes: 2 + prompt: "Waiting for for service /service/rest/api/v1/tenant" + - name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri uri: url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" From 8e7f1d5875dc8de1a6cb8b08abce140923c3ef36 Mon Sep 17 00:00:00 2001 From: roshi-dubey Date: Mon, 13 Jan 2025 15:54:43 +0530 Subject: [PATCH 066/169] [patch] update dns entries for aibroker (#1614) --- ibm/mas_devops/roles/suite_certs/templates/dnsentries.yml.j2 | 3 +++ ibm/mas_devops/roles/suite_dns/templates/dnsentries.yml.j2 | 3 +++ .../roles/suite_dns/templates/edge_certificate_routes.yml.j2 | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/ibm/mas_devops/roles/suite_certs/templates/dnsentries.yml.j2 b/ibm/mas_devops/roles/suite_certs/templates/dnsentries.yml.j2 index 0429aba26c..59b4078d68 100644 --- a/ibm/mas_devops/roles/suite_certs/templates/dnsentries.yml.j2 +++ b/ibm/mas_devops/roles/suite_certs/templates/dnsentries.yml.j2 @@ -53,6 +53,7 @@ nowildcard: - arcgis - {{ mas_workspace_id }}.arcgis - reportdb + - aibroker wildcard: - "" - "*" @@ -74,3 +75,5 @@ wildcard: - "*.arcgis" - "cis.test.cert" - "cp4d" + - "*.aibroker" + - "*.api.aibroker" diff --git a/ibm/mas_devops/roles/suite_dns/templates/dnsentries.yml.j2 b/ibm/mas_devops/roles/suite_dns/templates/dnsentries.yml.j2 index 1453fe05ee..af49a14b16 100644 --- a/ibm/mas_devops/roles/suite_dns/templates/dnsentries.yml.j2 +++ b/ibm/mas_devops/roles/suite_dns/templates/dnsentries.yml.j2 @@ -53,6 +53,7 @@ nowildcard: - arcgis - reportdb - cp4d + - aibroker wildcard: - "" - "*" @@ -74,3 +75,5 @@ wildcard: - "*.visualinspection" - "cis.test.cert" - "cp4d" + - "*.aibroker" + - "*.api.aibroker" diff --git a/ibm/mas_devops/roles/suite_dns/templates/edge_certificate_routes.yml.j2 b/ibm/mas_devops/roles/suite_dns/templates/edge_certificate_routes.yml.j2 index e1b3125149..17f394e0ad 100644 --- a/ibm/mas_devops/roles/suite_dns/templates/edge_certificate_routes.yml.j2 +++ b/ibm/mas_devops/roles/suite_dns/templates/edge_certificate_routes.yml.j2 @@ -50,3 +50,7 @@ edge_cert_routes: - arcgis.{{mas_domain}} - {{ mas_workspace_id }}.argis.{{mas_domain}} - reportdb.{{mas_domain}} + - aibroker.{{mas_domain}} + - {{ mas_workspace_id }}.aibroker.{{mas_domain}} + - api.aibroker.{{mas_domain}} + - {{ mas_workspace_id }}.api.aibroker.{{mas_domain}} From 9558d1110350ee0d30373cbe34df077b07726bef Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 13 Jan 2025 14:17:44 +0000 Subject: [PATCH 067/169] [patch] added new services --- .../aibroker/tasks/tenant/install/main.yml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 542b07536b..3d2b416d56 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -190,3 +190,43 @@ - name: Debug tenant entitlement #TODO: remove on final commit debug: msg: "{{ add_tenant_entitlements_to_db2 }}" + +- name: Create tenant {{ tenantNamespace }} and add to db2 for pcc - using uri + uri: + url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + validate_certs: no + method: POST + headers: + apikey: "{{ provision_tenant_api_key }}" + accept: "*/*" + Content-Type: "application/json" + tenantid: "provision-tenant" + body_format: "json" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"pcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" + status_code: [201] + timeout: 30 + register: add_tenant_entitlements_to_db2_pcc + +- name: Debug tenant entitlement #TODO: remove on final commit + debug: + msg: "{{ add_tenant_entitlements_to_db2_pcc }}" + +- name: Create tenant {{ tenantNamespace }} and add to db2 for mcc - using uri + uri: + url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + validate_certs: no + method: POST + headers: + apikey: "{{ provision_tenant_api_key }}" + accept: "*/*" + Content-Type: "application/json" + tenantid: "provision-tenant" + body_format: "json" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"mcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" + status_code: [201] + timeout: 30 + register: add_tenant_entitlements_to_db2_mcc + +- name: Debug tenant entitlement #TODO: remove on final commit + debug: + msg: "{{ add_tenant_entitlements_to_db2_mcc }}" From bb1dbee1ca7416da1e0c2304bdeb53596a3022c4 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 15 Jan 2025 15:43:49 +0000 Subject: [PATCH 068/169] [patch] rollback old way install AiBroker --- .../playbooks/oneclick_add_aibroker.yml | 18 +++++++++--------- ibm/mas_devops/roles/aibroker/tasks/main.yml | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index b4eb8aed3f..73a6c25842 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -65,15 +65,15 @@ fail_msg: "One or more required environment variables are not defined" when: mas_aibroker_saas == true - - include_role: - name: ibm.mas_devops.db2 - when: mas_aibroker_saas == true - - include_role: - name: ibm.mas_devops.suite_config - when: mas_aibroker_saas == true + # - include_role: + # name: ibm.mas_devops.db2 + # when: mas_aibroker_saas == true + # - include_role: + # name: ibm.mas_devops.suite_config + # when: mas_aibroker_saas == true roles: - - ibm.mas_devops.odh - - ibm.mas_devops.kmodels - - ibm.mas_devops.suite_app_install + # - ibm.mas_devops.odh + # - ibm.mas_devops.kmodels + # - ibm.mas_devops.suite_app_install - ibm.mas_devops.aibroker diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 9a57222a1a..62eca7b763 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -3,7 +3,7 @@ include_vars: "vars/main.yml" # install AI Broker api -# - include_tasks: tasks/aibroker/main.yml +- include_tasks: tasks/aibroker/main.yml # run api calls for aibroker saas - include_tasks: tasks/saas/main.yml From 6ca7b6ae4a871f0e3a1433c615fe68516e12838e Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 15 Jan 2025 15:56:28 +0000 Subject: [PATCH 069/169] [patch] add ibm catalog to playbook --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 73a6c25842..e8cf55e322 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -76,4 +76,5 @@ # - ibm.mas_devops.odh # - ibm.mas_devops.kmodels # - ibm.mas_devops.suite_app_install + - ibm.mas_devops.ibm_catalogs - ibm.mas_devops.aibroker From de0be60ec0872385ab57ca69c190027ee3624a74 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 16 Jan 2025 10:05:34 +0000 Subject: [PATCH 070/169] [patch] update operator and playbook --- .../playbooks/oneclick_add_aibroker.yml | 56 +++++++++---------- .../templates/aibroker/subscription.yml.j2 | 4 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index e8cf55e322..35546d74e6 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -17,11 +17,11 @@ mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" # Control if saas release - mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" + # mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" # DB2 # ----------------------------------------------------------------------------- - db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" + # db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" pre_tasks: # For the full set of supported environment variables refer to the playbook documentation @@ -30,22 +30,22 @@ that: - lookup('env', 'MAS_ENTITLEMENT_KEY') != "" - lookup('env', 'MAS_INSTANCE_ID') != "" - - lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') != "" - - lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') != "" - - lookup('env', 'MAS_AIBROKER_STORAGE_HOST') != "" - - lookup('env', 'MAS_AIBROKER_STORAGE_SSL') != "" - - lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') != "" - - lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') != "" - - lookup('env', 'MAS_AIBROKER_STORAGE_TENANTS_BUCKET') != "" - - lookup('env', 'MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET') != "" - - lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY') != "" - - lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') != "" - - lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') != "" - - lookup('env', 'MAS_AIBROKER_DB_HOST') != "" - - lookup('env', 'MAS_AIBROKER_DB_PORT') != "" - - lookup('env', 'MAS_AIBROKER_DB_USER') != "" - - lookup('env', 'MAS_AIBROKER_DB_DATABASE') != "" - - lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') != "" + # - lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') != "" + # - lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') != "" + # - lookup('env', 'MAS_AIBROKER_STORAGE_HOST') != "" + # - lookup('env', 'MAS_AIBROKER_STORAGE_SSL') != "" + # - lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') != "" + # - lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') != "" + # - lookup('env', 'MAS_AIBROKER_STORAGE_TENANTS_BUCKET') != "" + # - lookup('env', 'MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET') != "" + # - lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY') != "" + # - lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') != "" + # - lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') != "" + # - lookup('env', 'MAS_AIBROKER_DB_HOST') != "" + # - lookup('env', 'MAS_AIBROKER_DB_PORT') != "" + # - lookup('env', 'MAS_AIBROKER_DB_USER') != "" + # - lookup('env', 'MAS_AIBROKER_DB_DATABASE') != "" + # - lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') != "" - lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') != "" # - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" # - lookup('env', 'MAS_AIBROKER_SLS_URL') != "" @@ -54,16 +54,16 @@ # - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" # db2 support fail_msg: "One or more required environment variables are not defined" - - name: Check for required saas environment variables - assert: - that: - - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" - - lookup('env', 'MAS_AIBROKER_SLS_URL') != "" - - lookup('env', 'MAS_AIBROKER_DRO_URL') != "" - - lookup('env', 'MAS_CONFIG_DIR') != "" - - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" - fail_msg: "One or more required environment variables are not defined" - when: mas_aibroker_saas == true + # - name: Check for required saas environment variables + # assert: + # that: + # - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" + # - lookup('env', 'MAS_AIBROKER_SLS_URL') != "" + # - lookup('env', 'MAS_AIBROKER_DRO_URL') != "" + # - lookup('env', 'MAS_CONFIG_DIR') != "" + # - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" + # fail_msg: "One or more required environment variables are not defined" + # when: mas_aibroker_saas == true # - include_role: # name: ibm.mas_devops.db2 diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/subscription.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/subscription.yml.j2 index b76ea4d55b..dc7a429078 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/subscription.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/subscription.yml.j2 @@ -2,10 +2,10 @@ apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: - name: ibm-mas-aibroker-operator + name: ibm-mas-aibroker namespace: "{{ aibroker_namespace }}" spec: - channel: "{{ aibroker_channel }}" + channel: "{{ mas_app_channel }}" installPlanApproval: Automatic name: ibm-mas-aibroker source: "{{ mas_catalog_source }}" From d132675443f5e6196341a913ec456f49c1d0fe81 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 16 Jan 2025 10:32:53 +0000 Subject: [PATCH 071/169] [patch] fix to select correct operator --- .../playbooks/oneclick_add_aibroker.yml | 2 +- .../roles/aibroker/defaults/main.yml | 3 +++ .../roles/aibroker/tasks/aibroker/main.yml | 21 ++++++++++++------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 35546d74e6..262f2b8d55 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -46,7 +46,7 @@ # - lookup('env', 'MAS_AIBROKER_DB_USER') != "" # - lookup('env', 'MAS_AIBROKER_DB_DATABASE') != "" # - lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') != "" - - lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') != "" + # - lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') != "" # - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" # - lookup('env', 'MAS_AIBROKER_SLS_URL') != "" # - lookup('env', 'MAS_AIBROKER_DRO_URL') != "" diff --git a/ibm/mas_devops/roles/aibroker/defaults/main.yml b/ibm/mas_devops/roles/aibroker/defaults/main.yml index c3b5aee6c6..9105dc703b 100644 --- a/ibm/mas_devops/roles/aibroker/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker/defaults/main.yml @@ -2,6 +2,9 @@ mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" +# Which MAS channel to subscribe to +# mas_app_channel: "{{ lookup('env', 'MAS_APP_CHANNEL') | default('8.x', true) }}" + # Custom Labels # ----------------------------------------------------------------------------- custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string | ibm.mas_devops.string2dict() }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index dd93accf78..45360836c7 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -5,7 +5,7 @@ debug: msg: - "Namespace ...................... {{ aibroker_namespace }}" - - "Channel ........................ {{ aibroker_channel }}" + - "Channel ........................ {{ mas_app_channel }}" - "MAS Instance Id ................ {{ mas_instance_id }}" # Create IBM Maximo IBM Maximo AI Broker @@ -28,13 +28,18 @@ artifactory_username: "{{ artifactory_username }}" artifactory_password: "{{ artifactory_token }}" -- name: "Create ibm-mas-aibroker Subscription" - ibm.mas_devops.apply_subscription: - namespace: "{{ aibroker_namespace }}" - package_name: ibm-mas-aibroker - package_channel: "{{ aibroker_channel }}" - catalog_source: "{{ mas_catalog_source }}" - register: subscription +# - name: "Create ibm-mas-aibroker Subscription" +# ibm.mas_devops.apply_subscription: +# namespace: "{{ aibroker_namespace }}" +# package_name: ibm-mas-aibroker +# package_channel: "{{ aibroker_channel }}" +# catalog_source: "{{ mas_catalog_source }}" +# register: subscription + +- name: "Install Openshift odh Operator specific version" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/aibroker/subscription.yml.j2') }}" # Wait until the IBM Maximo AI Broker CRD is available From cc49f176e36d581ea1c7d4402244275136358467 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 16 Jan 2025 14:20:08 +0000 Subject: [PATCH 072/169] [patch] added cert manager role --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 1 + ibm/mas_devops/roles/aibroker/defaults/main.yml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 262f2b8d55..18dd078710 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -77,4 +77,5 @@ # - ibm.mas_devops.kmodels # - ibm.mas_devops.suite_app_install - ibm.mas_devops.ibm_catalogs + - ibm.mas_devops.cert_manager - ibm.mas_devops.aibroker diff --git a/ibm/mas_devops/roles/aibroker/defaults/main.yml b/ibm/mas_devops/roles/aibroker/defaults/main.yml index 9105dc703b..760a9ec042 100644 --- a/ibm/mas_devops/roles/aibroker/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker/defaults/main.yml @@ -2,8 +2,6 @@ mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" -# Which MAS channel to subscribe to -# mas_app_channel: "{{ lookup('env', 'MAS_APP_CHANNEL') | default('8.x', true) }}" # Custom Labels # ----------------------------------------------------------------------------- @@ -11,8 +9,10 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # Source container registry # ----------------------------------------------------------------------------- -mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" -mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" +# mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +# mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" +icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" # MAS Entitlement # ----------------------------------------------------------------------------- From 1d43feb7dd7f23d7887683d0ff06444755135de4 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 16 Jan 2025 15:01:47 +0000 Subject: [PATCH 073/169] [patch] comment out other aibroker deps --- .../roles/aibroker/tasks/aibroker/main.yml | 2 +- ibm/mas_devops/roles/aibroker/tasks/main.yml | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 45360836c7..13c630fca8 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -36,7 +36,7 @@ # catalog_source: "{{ mas_catalog_source }}" # register: subscription -- name: "Install Openshift odh Operator specific version" +- name: "Install AI Broker Operator" kubernetes.core.k8s: apply: yes definition: "{{ lookup('template', 'templates/aibroker/subscription.yml.j2') }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 62eca7b763..4a90d9dde5 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -5,18 +5,18 @@ # install AI Broker api - include_tasks: tasks/aibroker/main.yml -# run api calls for aibroker saas -- include_tasks: tasks/saas/main.yml - when: mas_aibroker_saas == true +# # run api calls for aibroker saas +# - include_tasks: tasks/saas/main.yml +# when: mas_aibroker_saas == true -# create AI Broker tenant -- include_tasks: tasks/tenant/main.yml +# # create AI Broker tenant +# - include_tasks: tasks/tenant/main.yml -# create AI Broker api key -- include_tasks: tasks/apikey/main.yml +# # create AI Broker api key +# - include_tasks: tasks/apikey/main.yml -# create s3 api key -- include_tasks: tasks/s3/main.yml +# # create s3 api key +# - include_tasks: tasks/s3/main.yml -# create wx api -- include_tasks: tasks/watsonx/main.yml +# # create wx api +# - include_tasks: tasks/watsonx/main.yml From 35a6237ee9bfedd4788c7a30592d8cfe035d3315 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 17 Jan 2025 11:52:24 +0000 Subject: [PATCH 074/169] [patch] add icr registry to CR --- .../roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index d018c87b45..9a18da45fe 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -9,4 +9,7 @@ metadata: mas.ibm.com/instanceId: "{{ mas_instance_id }}" spec: bindings: {} - # jdbc: system + settings: + icr: + cp: "{{ mas_icr_cp }}" + cpopen: "{{ mas_icr_cpopen }}" \ No newline at end of file From 0ff62cdfd0160c7e356897be3df09702c07b43a4 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 17 Jan 2025 13:43:50 +0000 Subject: [PATCH 075/169] [patch] add condition to wait for pod ready --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 13c630fca8..a551429396 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -75,3 +75,11 @@ - aibrokerapp_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 retries: 45 # approx 45 minutes before we give up delay: 60 # 1 minute + +- name: "Wait for aibroker-api pod to be ready (10s delay)" + shell: > + oc get pod -l app=aibroker-api -n mas-apmdevops-aibroker --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l + register: _is_not_ready + until: _is_not_ready.stdout|int == 0 + retries: 60 + delay: 10 From 4b67bd6ad7212680119790ebacd6783bccad2648 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 17 Jan 2025 13:54:35 +0000 Subject: [PATCH 076/169] [patch] add condition to wait for pod --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index a551429396..dd25ff6631 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -78,8 +78,12 @@ - name: "Wait for aibroker-api pod to be ready (10s delay)" shell: > - oc get pod -l app=aibroker-api -n mas-apmdevops-aibroker --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l + oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l register: _is_not_ready until: _is_not_ready.stdout|int == 0 retries: 60 delay: 10 + +- name: Display pod status + debug: + msg: " Status ..... {{ _is_not_ready }}" From c372148b6e913e4b630bbf105e4dcacd0b085a8a Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 17 Jan 2025 13:56:20 +0000 Subject: [PATCH 077/169] [patch] remove debug --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index dd25ff6631..893c0c8394 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -83,7 +83,3 @@ until: _is_not_ready.stdout|int == 0 retries: 60 delay: 10 - -- name: Display pod status - debug: - msg: " Status ..... {{ _is_not_ready }}" From fb76dbb2cfca61a3b32efdccb0c71a051e50ef83 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 17 Jan 2025 14:10:44 +0000 Subject: [PATCH 078/169] [patch] add wait for pod --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 893c0c8394..48068bc762 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -76,6 +76,9 @@ retries: 45 # approx 45 minutes before we give up delay: 60 # 1 minute +- name: "Wait for aibroker-api pod" + shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} + - name: "Wait for aibroker-api pod to be ready (10s delay)" shell: > oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l From 1f61ce62b3df7d04ec7bd576624d598e5bb0444d Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 17 Jan 2025 14:54:48 +0000 Subject: [PATCH 079/169] [patch] added wait --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 48068bc762..c224e793cc 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -78,6 +78,10 @@ - name: "Wait for aibroker-api pod" shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} + register: _is_not_exists + until: _is_not_exists.stdout|int == 0 + retries: 60 + delay: 10 - name: "Wait for aibroker-api pod to be ready (10s delay)" shell: > From 3c0f0262d9e4d9da53d08c24d2b8036d3a15045b Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 17 Jan 2025 15:09:24 +0000 Subject: [PATCH 080/169] [patch] added wait --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index c224e793cc..53d4f1a1d9 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -76,6 +76,11 @@ retries: 45 # approx 45 minutes before we give up delay: 60 # 1 minute +- name: Await to aibroker-api pod created + pause: + minutes: 3 + prompt: "Waiting for aibroker-api pod" + - name: "Wait for aibroker-api pod" shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} register: _is_not_exists From 65d8f979310c650eb422f35cf86bc7af1f0a1ad6 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 20 Jan 2025 09:28:45 +0000 Subject: [PATCH 081/169] [patch] added other roles needed for Aibroker to playbook --- .../playbooks/oneclick_add_aibroker.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 18dd078710..6c29f2912f 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -65,17 +65,14 @@ # fail_msg: "One or more required environment variables are not defined" # when: mas_aibroker_saas == true - # - include_role: - # name: ibm.mas_devops.db2 - # when: mas_aibroker_saas == true - # - include_role: - # name: ibm.mas_devops.suite_config - # when: mas_aibroker_saas == true - roles: - # - ibm.mas_devops.odh - # - ibm.mas_devops.kmodels - # - ibm.mas_devops.suite_app_install - ibm.mas_devops.ibm_catalogs - ibm.mas_devops.cert_manager + - ibm.mas_devops.sls + - ibm.mas_devops.dro + - ibm.mas_devops.db2 + # - ibm.mas_devops.minio + # - ibm.mas_devops.mariadb + # - ibm.mas_devops.odh + # - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker From 179f1873780dcd35f6741bc4feb6d619d4ce5d78 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 20 Jan 2025 09:59:55 +0000 Subject: [PATCH 082/169] Squashed commit of the following: commit 5bd2cccff32b80f0e71a096596250bd65a52268a Author: KAROL CZARNECKI Date: Fri Jan 17 15:11:21 2025 +0000 [ptch] added wait condition commit d53d59bc8f06f3794da9ae6d3cc796604ecd5b39 Author: KAROL CZARNECKI Date: Fri Jan 17 14:33:24 2025 +0000 [patch] restore old condition commit d632e9e625e83cd0e7ce8c734dc340db44c1c15d Author: KAROL CZARNECKI Date: Fri Jan 17 13:58:33 2025 +0000 [patch] add more accurate task for wait on aibroker-api pod ready commit 603cf5f2a43f345a1181e2ed216be0deb4594ad9 Author: KAROL CZARNECKI Date: Wed Jan 15 16:49:05 2025 +0000 [patch] added wait time commit d9511886dba2673644975149cb8837803e36bfbc Author: KAROL CZARNECKI Date: Wed Jan 15 16:37:34 2025 +0000 [patch] remove prefix cpd_ from detect storage class task commit 31ed39cbc111bddf792083752379891b929b9cd4 Author: KAROL CZARNECKI Date: Wed Jan 15 15:38:47 2025 +0000 [patch] update temp subscription commit 9221f859f8e23b98043fb050e0fa7d10cebacf44 Author: KAROL CZARNECKI Date: Wed Jan 15 12:25:54 2025 +0000 Squashed commit of the following: commit 9c33df9b7da2167dd9154d87b09472d020ad8375 Author: rene-oromtz <157750458+rene-oromtz@users.noreply.github.com> Date: Mon Jan 13 10:26:02 2025 -0600 [patch] Add missing api_version to k8s_info lookup (#1616) commit 0417879bd9fe9912ff71a148b5c53ae0aadab3da Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Mon Jan 13 21:52:43 2025 +0530 [patch] Adds some deletion waits in suite_app_uninstall (#1615) Co-authored-by: Harsh Tamakuwala commit 9a0893d48a2fa727325f02722799517058e67441 Author: David Parker Date: Mon Jan 13 16:05:34 2025 +0000 [patch] Run prettier on yaml files (#1621) commit d50047368037b203bd672bac0d7b838fcffa087f Author: Tremaine Hart Date: Fri Jan 10 15:32:13 2025 -0600 [patch] [MASCORE-5063] Problems with Restoring PV data - DT420492 (#1618) commit 0fa5980090b7ea4887bfc59289b26b02ded49f22 Author: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Date: Thu Jan 9 02:28:38 2025 +0530 [minor] Support January Catalog Update (#1595) Co-authored-by: Parveen Kumar commit 5653ebdafa56e9cdf040a0ff82fb2e0dd3ed6d01 Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Mon Jan 6 16:24:12 2025 +0530 [patch] Fixes manage namespace uninstall stuck in terminating state (#1611) Co-authored-by: Harsh Tamakuwala commit 83d83650cb75eaccaf6460c9b0fe6fba03fe1b8f Author: leo-miran <105313348+leo-miran@users.noreply.github.com> Date: Thu Dec 19 17:09:45 2024 -0300 [patch] Remove OCP 4.12 & 4.13 from rotation (#1609) commit 13770e9343341a5963d2918f1814006a6f326964 Author: sekharcvalluri <54737838+sekharcvalluri@users.noreply.github.com> Date: Thu Dec 19 19:19:48 2024 +0530 [patch] Fix cos_bucket picking incorrect resourcegroup (#1593) Co-authored-by: Harsh Tamakuwala Co-authored-by: leo-miran <105313348+leo-miran@users.noreply.github.com> commit 0a3b97bce5c3ed58d10995c773305eeb376067f2 Author: David Parker Date: Thu Dec 19 13:47:57 2024 +0000 [major] Replace ICSP with IDMS (#1594) commit b08cda4327ab30d03326dbc5c7b35bee86bf16ba Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Thu Dec 19 18:49:47 2024 +0530 [patch] Update supported kafka versions (#1608) Co-authored-by: Harsh Tamakuwala commit cf057173082d9a94137cf3aa3c63d5bb9aa7b3bd Author: David Parker Date: Thu Dec 19 11:39:55 2024 +0000 [patch] Lock mas-devops less than v2 (#1607) commit 6ff397e628583040bc4e7aad56ae5558d7796db2 Author: David Parker Date: Tue Dec 17 22:35:41 2024 +0000 [patch] Enable HMAC by default for IBM COS (#1606) commit 49b2099ff65f2322fb8c670311dc5ea7030ae2ea Author: Rawa <61942902+rawa-resul@users.noreply.github.com> Date: Thu Dec 12 16:28:49 2024 +0000 [patch] Fix subscription lookup in upgrade (#1589) Co-authored-by: Rawa Resul commit 79c5991d608369f36eed2c50ea2edb69225b5ac7 Author: leo-miran <105313348+leo-miran@users.noreply.github.com> Date: Thu Dec 12 13:26:06 2024 -0300 [patch] fix default booleans for suite_dns (#1590) commit 1e91ba5f6361f484e67c1ab6181f399eebd4269c Author: Andrew Whitfield Date: Thu Dec 12 07:34:31 2024 +0000 [patch] Increase rosa deprovision wait time (#1592) commit 91bd67beb1637d11677e8d835594eb26dbd07da3 Author: Ian Boden <82514609+IanBoden@users.noreply.github.com> Date: Mon Dec 9 10:32:54 2024 +0000 [patch] reinstate zenmetastore and couchdb workarounds for cpd 4.8 (#1580) commit aa05d8a589d8334149fcd53c66d5fcea01977d07 Author: Rawa <61942902+rawa-resul@users.noreply.github.com> Date: Fri Dec 6 16:32:24 2024 +0000 [patch] Fix label_selectors pattern (#1586) Co-authored-by: Rawa Resul commit e0ba0decf64579d7b021e16e68d866f9704c6a5e Author: Parveen Kumar Date: Fri Dec 6 19:23:46 2024 +0530 [minor] Support December catalog (#1562) Co-authored-by: Anil Prajapati Co-authored-by: Jon Levell Co-authored-by: Jon Levell Co-authored-by: Ian Boden <82514609+IanBoden@users.noreply.github.com> Co-authored-by: Ian Boden Co-authored-by: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Co-authored-by: Terence Quinn Co-authored-by: David Parker Co-authored-by: Rawa Resul Co-authored-by: Rawa <61942902+rawa-resul@users.noreply.github.com> commit 9206656d7384f91469b03841606d73c06682bd18 Author: David Parker Date: Tue Dec 3 14:36:31 2024 +0000 [major] Use new OLM functions in mas-devops (#1573) commit 3ccdbef982c83989942445ef077d4117d81fec48 Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Mon Dec 2 18:30:04 2024 +0530 [patch] Fixes ibmcloud_api_key issue in COS deprovision for ibmcloud (#1581) Co-authored-by: Harsh Tamakuwala commit 1283c03b65c60cfc0095969abc878264db98724f Author: Terence Quinn Date: Fri Nov 29 03:52:51 2024 -0600 [patch] create unique certificate for each db2 instance (#1579) commit e04138afb8c7ffdfe4be159323d98d404426bb88 Author: David Parker Date: Tue Nov 26 21:40:39 2024 +0000 [patch] get_default_storage_classes should always set failed=False (#1578) commit 7014f49fc63410d44808a7082e8a1ce922e93879 Author: Sanjay Prabhakar Date: Tue Nov 26 16:32:42 2024 +0000 [patch] Add estimates for cpd in readme (#1577) commit 9933bc23fb272ba7fe169ded7a54de71e952b55b Author: David Parker Date: Tue Nov 26 13:46:05 2024 +0000 [patch] Use correct variable for mongo extras (#1576) commit bb720a394502505f5c91849992fc276e0a4cc1c4 Author: David Parker Date: Tue Nov 26 13:32:30 2024 +0000 [patch] Set cpd_product_version from catalog metadata (#1575) commit ba23e83ae87fe889f6bf2fd6bd26da16d1034881 Author: Terence Quinn Date: Mon Nov 25 15:03:03 2024 -0600 [patch] Fix missing catalog_digest (#1574) Co-authored-by: David Parker commit 0f60c6150f2f4541673c22b8312113fe24ce67ac Author: Sanjay Prabhakar Date: Fri Nov 22 17:33:20 2024 +0000 [patch] Add back restore.yml.j2 (#1572) Co-authored-by: Sanjay Prabhakar commit 739c4d03781bca7c983d92e103e62e6542b72fa7 Author: David Parker Date: Fri Nov 22 16:02:24 2024 +0000 [major] Overhaul backup/restore capability (#1557) Co-authored-by: Sanjay Prabhakar commit d5ef4102841b56a9fa7045be01989a3056fed5ef Author: David Parker Date: Fri Nov 22 12:39:48 2024 +0000 [doc] Fix formatting commit 61272c972bc692df31bdd64104234257b1e41ef2 Author: Sanjay Prabhakar Date: Fri Nov 22 12:27:00 2024 +0000 [patch] fix restore for manage role (#1571) Co-authored-by: Sanjay Prabhakar commit 0a13abdc64bf2e458dcb42c9560965a5c107da9f Author: Sanjay Prabhakar Date: Thu Nov 21 10:40:05 2024 +0000 [patch] Db2 backup/restore: fix copy local to pod script (#1569) Co-authored-by: Sanjay Prabhakar commit f4745094e7a119e8fbf5eb2a341e856baaf757e3 Author: David Parker Date: Wed Nov 20 23:43:25 2024 +0000 [major] Move catalog & storage class metadata to python package (#1561) commit df4fe4c55d7a1f1d78723f1452b8a6c3f4dda3a9 Author: Lokesh <110647904+lokesh-sreedhara@users.noreply.github.com> Date: Wed Nov 20 08:49:28 2024 -0500 [patch] remove MIRRORLOGPATH folder creation (#1568) commit 5668f331e38307250c4bdb22e5e5dd2e1f687607 Author: Lokesh <110647904+lokesh-sreedhara@users.noreply.github.com> Date: Wed Nov 20 07:49:43 2024 -0500 [patch] fix conflicting DB config (#1567) commit 5c595c11aacfdb9a3044c257ba9042a08005f311 Author: Sanjay Prabhakar Date: Tue Nov 19 17:45:03 2024 +0000 [patch] can skip db2 in br_manage playbook (#1566) Co-authored-by: Sanjay Prabhakar commit 8c8f9933ce35e16862e3f328ee5fe79349a634ea Author: stonepd Date: Tue Nov 19 16:39:18 2024 +0000 [patch] update CP4D README, remove outdated fix code (#1565) Co-authored-by: Sanjay Prabhakar commit 5eb2cd4aa2ef8f9fcc12626e1f85b577b9219ef3 Author: Sanjay Prabhakar Date: Tue Nov 19 14:42:36 2024 +0000 [patch] fix restore for core and fix tar command in restore tasks (#1564) Co-authored-by: Sanjay Prabhakar commit e81d3a3e0ddd89ef0f7ed1289ff8a7f311599b33 Author: dclain Date: Sat Nov 16 15:39:54 2024 -0600 [patch] Address several issues w/ Nvidia playbook (#1558) commit 1792e98b77b32185c486da66a31c4c89fb11bfcf Author: Tremaine Hart Date: Thu Nov 14 13:19:26 2024 -0600 [patch] Fixes and updates for IBMCloud COS support (#1555) Co-authored-by: David Parker commit 495e26d52efbda11ccb94bbab990a00652e9cc0e Author: Daniel Istrate <16302386+istrate@users.noreply.github.com> Date: Thu Nov 14 13:02:17 2024 +0200 [doc] Document storage class access mode (#1554) Co-authored-by: Daniel Istrate commit 08c22d74515c90a46f19894a630cd2c8155bccef Author: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Date: Tue Nov 12 02:51:17 2024 +0530 [minor] Support November catalog (#1508) Co-authored-by: Parveen Kumar Co-authored-by: Jon Levell Co-authored-by: Jon Levell Co-authored-by: Ian Boden <82514609+IanBoden@users.noreply.github.com> Co-authored-by: Ian Boden commit 35017863e41ff139d567671a507800c49b4f1bc5 Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Thu Nov 7 09:53:24 2024 -0800 [patch] MAXMIS-897: check if attribute 'deployimagestitching' is defined (#1551) Co-authored-by: Brendan More O'Ferrall commit 00c50f15b3ba266f4ec42d0fd44bd81f95f9543c Author: caemar <44871769+caemar@users.noreply.github.com> Date: Thu Nov 7 11:47:38 2024 +0100 [minor] Add Kafka Cluster CA certificate validityDays (#1543) commit ee3a2272b4a3418339eb2c4df77598d56f1f0179 Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Wed Nov 6 11:57:00 2024 -0800 [patch] MAXMIS-886: disable stitching deploy when civil is not enabled2 (#1548) Co-authored-by: Brendan More O'Ferrall commit 3ee251228c962a156692391bceb063c22ebac459 Author: Tremaine Hart Date: Wed Nov 6 11:41:34 2024 -0600 [patch] [MASCORE-4344] Grafana role overwriting critical k8s configuration resource (#1549) commit 482b3868b31468f220da901d5d23458eb47d8afd Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Tue Nov 5 21:51:50 2024 -0800 [minor] #MAXMIS-748: add role to auto configure image stitching (#1530) Co-authored-by: Brendan More O'Ferrall Co-authored-by: Alexandre Quinteiro Co-authored-by: unnati-solanki-git <131964221+unnati-solanki-git@users.noreply.github.com> commit dd13898e98f238b724da63f476d59312c494f139 Author: Tremaine Hart Date: Tue Nov 5 14:45:45 2024 -0600 [patch] [MASCORE-4344] Grafana role overwriting critical k8s configuration resource (#1547) commit ade2a0ea54989b2b2ce55a47890f5d249d13484a Author: Andrew Whitfield Date: Tue Nov 5 17:18:24 2024 +0000 [patch] wait for olm to update operatorcondition (#1545) commit c267990a26a599d405fc78fe248181ab13ad258a Author: Sanjay Prabhakar Date: Tue Nov 5 16:33:07 2024 +0000 [patch] prevent mongo upgrade when featureCompatibilityVersion is not defined or set incorrectly (#1544) Co-authored-by: Sanjay Prabhakar commit f4c04d569cbad2268ad763b344fd297399c5e19a Author: Ian Boden <82514609+IanBoden@users.noreply.github.com> Date: Tue Nov 5 11:21:08 2024 +0000 [patch] Fix gpu-certified-operator default channel (#1542) commit 733b595cebbd00232c415a1d62d456266936ed58 Author: David Parker Date: Mon Nov 4 15:49:12 2024 +0000 [patch] Fix oc exec commands (#1541) commit 224a9a8707befe26612523397a2564a1f31b4071 Author: David Parker Date: Thu Oct 31 18:10:39 2024 +0000 [patch] Reduce useless logging (#1540) commit 0bdd755bc0e89e5da4ecea4edf30855e1b85c5b7 Author: KAROL CZARNECKI Date: Wed Jan 15 11:58:58 2025 +0000 [patch] clean playbooks not used --- .gitignore | 3 + .prettierignore | 52 + build/bin/build-collection.sh | 4 +- .../backup_restore/after_run_tasks.yml | 3 - .../backup_restore/before_run_tasks.yml | 5 - .../backup_restore/check_backup_vars.yml | 15 +- .../backup_restore/check_common_vars.yml | 1 - .../backup_restore/check_restore_vars.yml | 17 +- .../backup_restore/confirm_cluster_info.yml | 1 - .../copy_pod_files_to_storage.yml | 14 +- .../copy_storage_files_to_pod.yml | 29 +- .../backup_restore/create_cleanup_job.yml | 2 - .../rename_storage_job_folder.yml | 1 - .../backup_restore/update_job_status.yml | 3 - .../common_tasks/default_storage_classes.yml | 1 - ibm/mas_devops/common_tasks/detect_airgap.yml | 3 - ibm/mas_devops/common_tasks/detect_sno.yml | 7 +- .../common_tasks/get_version_from_channel.yml | 2 +- .../backup-namespace-resources.sh.j2 | 7 + .../wait-machine-config-update.yml | 5 +- ibm/mas_devops/common_tasks/wait_for_crd.yml | 2 +- ibm/mas_devops/common_vars/backup_restore.yml | 3 - .../common_vars/compatibility_matrix.yml | 16 +- ibm/mas_devops/meta/runtime.yml | 2 +- ibm/mas_devops/playbooks/br_db2.yml | 1 - ibm/mas_devops/playbooks/br_health.yml | 1 - ibm/mas_devops/playbooks/br_iot.yml | 1 - ibm/mas_devops/playbooks/br_manage.yml | 1 - ibm/mas_devops/playbooks/br_mongodb.yml | 1 - ibm/mas_devops/playbooks/br_monitor.yml | 1 - ibm/mas_devops/playbooks/br_optimizer.yml | 1 - .../playbooks/br_visualinspection.yml | 1 - ibm/mas_devops/playbooks/mirror_add_iot.yml | 1 - .../playbooks/mirror_add_monitor.yml | 1 - .../playbooks/mirror_dependencies.yml | 17 - .../playbooks/ocp_convert_to_disconnected.yml | 2 +- .../playbooks/oneclick_add_manage.yml | 2 - .../playbooks/oneclick_add_optimizer.yml | 1 - .../playbooks/oneclick_add_predict.yml | 10 +- .../oneclick_add_visualinspection.yml | 18 +- ibm/mas_devops/playbooks/oneclick_upgrade.yml | 2 - ibm/mas_devops/requirements.txt | 2 +- .../roles/aibroker/tasks/aibroker/main.yml | 1 - .../roles/aibroker/tasks/s3/remove/main.yml | 6 +- .../roles/aibroker/tasks/saas/main.yml | 74 +- .../aibroker/tasks/tenant/install/main.yml | 137 +- ibm/mas_devops/roles/aibroker/vars/main.yml | 1 + .../roles/appconnect/tasks/appconnectcfg.yml | 2 +- .../tasks/determine-storage-classes.yml | 1 - .../roles/appconnect/tasks/main.yml | 15 +- ibm/mas_devops/roles/arcgis/tasks/main.yml | 1 - .../aws_bucket_access_point/tasks/main.yml | 6 +- .../roles/aws_documentdb_user/tasks/main.yml | 3 - .../files/policy-template-sample.json | 22 +- .../roles/aws_user_creation/tasks/main.yml | 1 - .../tasks/provider/redhat/install.yml | 6 - .../cis/tasks/provider/ibm/provision.yml | 2 +- .../common_services/tasks/actions/upgrade.yml | 4 - .../defaults/main.yml | 2 +- .../tasks/main.yml | 4 +- .../cos/tasks/providers/ibm/provision.yml | 4 +- .../roles/cos_bucket/defaults/main.yml | 15 +- .../cos_bucket/tasks/providers/aws/delete.yml | 6 +- .../create_cross_region_location_bucket.yml | 45 +- .../ibm/create_region_location_bucket.yml | 45 +- .../ibm/create_service_credentials.yml | 2 +- .../cp4d/tasks/authorize-user-cpd-admin.yml | 1 - .../roles/cp4d/tasks/create-subscriptions.yml | 2 +- .../cp4d/tasks/determine-ibmcatalog-tag.yml | 2 - .../cp4d/tasks/determine-storage-classes.yml | 1 - .../roles/cp4d/tasks/entitlement.yml | 8 +- .../tasks/prereq-check/check-cpfs-version.yml | 1 - .../roles/cp4d/tasks/prereqs/install-cpfs.yml | 4 +- .../prereqs/install-ibm-namespace-scope.yml | 4 +- .../cp4d/tasks/wait/wait-zenmetastore-edb.yml | 10 +- .../cp4d/templates/catalog_sources/5.0.0.yml | 2 +- .../tasks/get-cp4d-bearer-token.yml | 2 +- .../cp4d_admin_pwd_update/tasks/main.yml | 4 +- .../tasks/determine-storage-classes.yml | 1 - .../cp4d_service/tasks/gencfg/gencfg-wd.yml | 9 +- .../cp4d_service/tasks/gencfg/gencfg-wsl.yml | 15 +- .../cp4d_service/tasks/prereqs/odf/main.yml | 3 +- .../roles/cp4d_service/tasks/wait/wait-ca.yml | 1 - .../cp4d_service/tasks/wait/wait-ccs.yml | 1 - .../cp4d_service/tasks/wait/wait-couchdb.yml | 7 +- .../tasks/wait/wait-elasticsearch.yml | 1 - .../cp4d_service/tasks/wait/wait-spark.yml | 1 - .../cp4d_service/tasks/wait/wait-wml.yml | 1 - .../cp4d_service/tasks/wait/wait-wsl.yml | 3 - .../templates/wd/storage/namespace.yaml | 2 +- .../db2/tasks/backup/backup-database.yml | 6 - .../roles/db2/tasks/backup/main.yml | 4 - .../roles/db2/tasks/before-backup-restore.yml | 5 - .../db2/tasks/install/create_ldap_user.yml | 1 - .../install/determine-storage-classes.yml | 6 - .../roles/db2/tasks/install/main.yml | 1 - .../db2/tasks/install/setup_norootsquash.yml | 5 +- .../roles/db2/tasks/install/suite_jdbccfg.yml | 7 +- .../tasks/restore/copy-db2-backup-file.yml | 10 +- .../roles/db2/tasks/restore/main.yml | 4 - .../db2/tasks/restore/restore-database.yml | 9 - .../roles/dro/tasks/gencfg/main.yml | 1 - .../install-dro/determine-storage-classes.yml | 2 - .../roles/dro/tasks/install-dro/main.yml | 11 - ibm/mas_devops/roles/eck/defaults/main.yml | 1 - ibm/mas_devops/roles/eck/tasks/install.yml | 8 - .../tasks/rotate-ibm-entitlement.yml | 4 +- .../tasks/rotate-wiotp-docker-local.yml | 4 +- .../roles/gencfg_watsonstudio/tasks/main.yml | 4 +- .../install/determine-storage-classes.yml | 2 - .../roles/grafana/tasks/install/main.yml | 6 +- .../grafana/tasks/uninstall/uninstall.yml | 4 - .../tasks/uninstall_grafanav4/main.yml | 1 - .../tasks/update/determine-storage-config.yml | 4 - .../roles/grafana/tasks/update/main.yml | 3 - .../tasks/install/development-catalog.yml | 8 +- .../roles/ibm_catalogs/tasks/install/main.yml | 3 - .../ibm_catalogs/tasks/uninstall/main.yml | 1 - .../ibmcloud_resource_key/tasks/main.yml | 1 - .../tasks/service_key_create.yml | 1 - .../tasks/service_key_delete.yml | 1 - .../kafka/tasks/determine-storage-classes.yml | 1 - .../kafka/tasks/provider/aws/install.yml | 3 - .../kafka/tasks/provider/ibm/install.yml | 2 +- .../redhat/run-kafka-instances-upgrade.yml | 1 - .../kafka/tasks/provider/redhat/upgrade.yml | 1 - .../redhat/grafana-json/kafka-exporter.json | 3279 +++++---- .../redhat/grafana-json/kafka-zookeeper.json | 3239 +++++---- .../templates/redhat/grafana-json/kafka.json | 6022 ++++++++--------- .../tasks/determine-storage-classes.yml | 1 - .../kmodels-controller/configmap.yml.j2 | 2 +- .../kmodels-watcher/configmap.yml.j2 | 2 +- ibm/mas_devops/roles/kmodels/vars/main.yml | 2 +- .../roles/mirror_case_prepare/tasks/main.yml | 2 - .../mirror_extras_prepare/defaults/main.yml | 2 - .../mirror_extras_prepare/tasks/catalog.yml | 4 - .../mirror_extras_prepare/tasks/extras.yml | 4 - .../mirror_extras_prepare/tasks/main.yml | 1 - .../roles/mirror_images/tasks/main.yml | 9 +- .../roles/mirror_ocp/defaults/main.yml | 3 - .../roles/mirror_ocp/tasks/actions/direct.yml | 3 +- .../tasks/actions/from-filesystem.yml | 1 - .../tasks/actions/to-filesystem.yml | 3 +- .../roles/mirror_ocp/tasks/main.yml | 7 +- .../roles/mongodb/defaults/main.yml | 1 - .../tasks/determine-ibmcatalog-tag.yml | 2 - .../tasks/providers/aws/destroy-data.yml | 2 +- .../providers/aws/docdb_secret_rotate.yml | 2 - .../backup-restore/backup-database.yml | 12 +- .../backup-restore/before-backup-restore.yml | 7 - .../backup-restore/create-role-user.yml | 2 - .../backup-restore/restore-database-patch.yml | 3 - .../restore-database-perform.yml | 3 - .../backup-restore/restore-database.yml | 6 - .../tasks/providers/community/backup.yml | 4 - .../community/determine-storage-classes.yml | 2 - .../providers/community/install-mongo.yml | 13 - .../tasks/providers/community/install.yml | 4 - .../tasks/providers/community/restore.yml | 4 - .../tasks/providers/community/uninstall.yml | 6 - .../providers/community/validate-upgrade.yml | 1 - .../mongodb/templates/community/ca-cert.yml | 6 +- .../json/mongodb-overview-grafana.json | 267 +- .../templates/community/server-cert.yml | 6 +- .../roles/nvidia_gpu/tasks/main.yml | 124 +- .../roles/nvidia_gpu/tasks/nfd_setup.yml | 109 +- .../roles/ocp_provision/tasks/main.yml | 8 +- .../roles/odh/tasks/odh-operator.yml | 4 +- .../tasks/restore-pv.yml | 2 +- .../roles/suite_app_uninstall/tasks/main.yml | 42 +- prettierrc.yaml | 3 + 171 files changed, 6743 insertions(+), 7377 deletions(-) create mode 100644 .prettierignore create mode 100644 prettierrc.yaml diff --git a/.gitignore b/.gitignore index 0116e96c41..067e4d4ecd 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ build/bin/downloads/*.tgz .pyenv cpd-cli-workspace/* /tmp +/node_modules +package-lock.json +package.json diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000..d606ef6b52 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,52 @@ +# Don't change markdown files +**/*.md + +# Exclude entirely the roles +# TODO: Gradually remove these rules and verify the formatting +/ibm/mas_devops/roles/ocp_cluster_monitoring/**/*.* +/ibm/mas_devops/roles/ocp_config/**/*.* +/ibm/mas_devops/roles/ocp_deprovision/**/*.* +/ibm/mas_devops/roles/ocp_efs/**/*.* +/ibm/mas_devops/roles/ocp_github_oauth/**/*.* +/ibm/mas_devops/roles/ocp_idms/**/*.* +/ibm/mas_devops/roles/ocp_login/**/*.* +/ibm/mas_devops/roles/ocp_node_config/**/*.* +/ibm/mas_devops/roles/ocp_provision/**/*.* +/ibm/mas_devops/roles/ocp_roks_upgrade_registry_storage/**/*.* +/ibm/mas_devops/roles/ocp_simulate_disconnected_network/**/*.* +/ibm/mas_devops/roles/ocp_upgrade/**/*.* +/ibm/mas_devops/roles/ocp_verify/**/*.* +/ibm/mas_devops/roles/ocs/**/*.* +/ibm/mas_devops/roles/odh/**/*.* +/ibm/mas_devops/roles/opentelemetry/**/*.* +/ibm/mas_devops/roles/registry/**/*.* +/ibm/mas_devops/roles/sls/**/*.* +/ibm/mas_devops/roles/smtp/**/*.* +/ibm/mas_devops/roles/suite_app_backup_restore/**/*.* +/ibm/mas_devops/roles/suite_app_config/**/*.* +/ibm/mas_devops/roles/suite_app_install/**/*.* +/ibm/mas_devops/roles/suite_app_rollback/**/*.* +/ibm/mas_devops/roles/suite_app_uninstall/**/*.* +/ibm/mas_devops/roles/suite_app_upgrade/**/*.* +/ibm/mas_devops/roles/suite_app_verify/**/*.* +/ibm/mas_devops/roles/suite_backup_restore/**/*.* +/ibm/mas_devops/roles/suite_certs/**/*.* +/ibm/mas_devops/roles/suite_config/**/*.* +/ibm/mas_devops/roles/suite_db2_setup_for_manage/**/*.* +/ibm/mas_devops/roles/suite_dns/**/*.* +/ibm/mas_devops/roles/suite_install/**/*.* +/ibm/mas_devops/roles/suite_manage_attachments_config/**/*.* +/ibm/mas_devops/roles/suite_manage_bim_config/**/*.* +/ibm/mas_devops/roles/suite_manage_birt_report_config/**/*.* +/ibm/mas_devops/roles/suite_manage_customer_files_config/**/*.* +/ibm/mas_devops/roles/suite_manage_imagestitching_config/**/*.* +/ibm/mas_devops/roles/suite_manage_import_certs_config/**/*.* +/ibm/mas_devops/roles/suite_manage_load_dbc_scripts/**/*.* +/ibm/mas_devops/roles/suite_manage_logging_config/**/*.* +/ibm/mas_devops/roles/suite_manage_pvc_config/**/*.* +/ibm/mas_devops/roles/suite_rollback/**/*.* +/ibm/mas_devops/roles/suite_uninstall/**/*.* +/ibm/mas_devops/roles/suite_upgrade/**/*.* +/ibm/mas_devops/roles/suite_verify/**/*.* +/ibm/mas_devops/roles/turbonomic/**/*.* +/ibm/mas_devops/roles/uds/**/*.* diff --git a/build/bin/build-collection.sh b/build/bin/build-collection.sh index 84a599cfb2..02425aebf7 100644 --- a/build/bin/build-collection.sh +++ b/build/bin/build-collection.sh @@ -13,8 +13,8 @@ cat $GITHUB_WORKSPACE/ibm/mas_devops/galaxy.yml # Update this when we have new catalog -MAS_PREVIOUS_CATALOG='v9-241107-amd64' -MAS_LATEST_CATALOG='v9-241205-amd64' +MAS_PREVIOUS_CATALOG='v9-241205-amd64' +MAS_LATEST_CATALOG='v9-250109-amd64' # Update all the placeholders in the playbooks diff --git a/ibm/mas_devops/common_tasks/backup_restore/after_run_tasks.yml b/ibm/mas_devops/common_tasks/backup_restore/after_run_tasks.yml index 78ad24ab0f..f8ae4c50da 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/after_run_tasks.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/after_run_tasks.yml @@ -5,7 +5,6 @@ when: _component_after_task_path is defined and _component_after_task_path | length > 0 include_tasks: "{{ _component_after_task_path }}" - # Copy Ansible log file to storage location # ----------------------------------------------------------------------------- - name: "Set fact: Ansible log path" @@ -38,7 +37,6 @@ - src_file: "log/{{ masbr_ansible_log_name }}-log.tar.gz" dest_folder: "log" - # Delete local job folder # ----------------------------------------------------------------------------- - name: "Delete local job folder" @@ -46,7 +44,6 @@ path: "{{ masbr_local_job_folder }}" state: absent - # Display summary of the running task results # ----------------------------------------------------------------------------- - name: "Summary" diff --git a/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml b/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml index 7d037d7744..ab977485f4 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml @@ -5,32 +5,27 @@ # Scenario 2 - when running a role: # 1. the role include this task - # Check common variables # ----------------------------------------------------------------------------- - name: "Check common variables" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/check_common_vars.yml" - # Confirm cluster information # ----------------------------------------------------------------------------- - name: "Confirm the currently connected cluster information" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/confirm_cluster_info.yml" - # Check common backup/restore variables # ----------------------------------------------------------------------------- - name: "Check common {{ _job_type }} variables" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/check_{{ _job_type }}_vars.yml" - # Before backup/restore component # ------------------------------------------------------------------------- - name: "Before {{ _job_type }} {{ masbr_job_component.name }}" when: _component_before_task_path is defined and _component_before_task_path | length > 0 include_tasks: "{{ _component_before_task_path }}" - # Set a flag to indicate these tasks are included # ----------------------------------------------------------------------------- - name: "Set fact: already included these tasks" diff --git a/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml b/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml index abb99d4b69..87ddb9b788 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml @@ -8,7 +8,6 @@ # masbr_backup_from # masbr_backup_from_yaml - # Backup environment variables # ----------------------------------------------------------------------------- - name: "Set fact: backup environment variables" @@ -23,7 +22,6 @@ # only used when masbr_backup_type='incr' masbr_backup_from_version: "{{ lookup('env', 'MASBR_BACKUP_FROM_VERSION') | default('', true) }}" - # Check 'masbr_job_component' # ----------------------------------------------------------------------------- - name: "Fail if masbr_job_component is not provided" @@ -36,7 +34,6 @@ - "masbr_job_component.name is required" - "masbr_job_component.namespace is required" - # Check 'masbr_job_data_list' # ----------------------------------------------------------------------------- - name: "Set fact: init masbr_job_data_list" @@ -80,7 +77,6 @@ set_fact: masbr_job_data_list: "{{ masbr_job_data_init }}" - # Set 'masbr_task_type' # ----------------------------------------------------------------------------- - name: "Set fact: backup job variables" @@ -112,13 +108,11 @@ # At this point, set it as the same value of masbr_job_name masbr_job_name_final: "{{ masbr_job_name }}" - # Create local job folder # ----------------------------------------------------------------------------- - name: "Create local job folder" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_local_job_folder.yml" - # Check incremental backup # ----------------------------------------------------------------------------- - name: "Checks for incremental backup" @@ -129,16 +123,16 @@ when: masbr_backup_from_version is not defined or masbr_backup_from_version | length == 0 include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/list_storage_job_folders.yml" vars: - - masbr_ls_job_type: "backup" - - masbr_ls_filter: "| grep -P '^{{ masbr_job_name_prefix }}-full-.*(? 0 include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/list_storage_job_folders.yml" vars: - - masbr_ls_job_type: "backup" - - masbr_ls_filter: "| grep '^{{ masbr_job_name_prefix }}-full-{{ masbr_backup_from_version }}$' | sort -r | head -1" + masbr_ls_job_type: "backup" + masbr_ls_filter: "| grep '^{{ masbr_job_name_prefix }}-full-{{ masbr_backup_from_version }}$' | sort -r | head -1" - name: "Fail if not found any previous Full backup job" assert: @@ -190,7 +184,6 @@ that: masbr_job_data_list_differ | length == 0 fail_msg: "The data list of backup from job does not cover current job: {{ masbr_job_data_list_differ }}" - # Show backup job information # ----------------------------------------------------------------------------- - name: "Debug: backup job information" diff --git a/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml b/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml index a29a1c995d..086cfab533 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml @@ -28,7 +28,6 @@ set_fact: masbr_job_version: "{{ lookup('env', 'MASBR_JOB_VERSION') | default(masbr_timestamp_format | strftime, true) }}" - # Storage location # ----------------------------------------------------------------------------- - name: "Fail if masbr_storage_local_folder is not provided" diff --git a/ibm/mas_devops/common_tasks/backup_restore/check_restore_vars.yml b/ibm/mas_devops/common_tasks/backup_restore/check_restore_vars.yml index 438061f9e2..e870c48b1c 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/check_restore_vars.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/check_restore_vars.yml @@ -11,7 +11,6 @@ # masbr_restore_to_diff_domain: true|false # masbr_restore_to_diff_instance: true|false - # Restore environment variables # ----------------------------------------------------------------------------- - name: "Set fact: restore environment variables" @@ -32,7 +31,6 @@ that: masbr_restore_from_version is defined and masbr_restore_from_version != "" fail_msg: "masbr_restore_from_version is required for running restore job" - # Check 'masbr_job_component' # ----------------------------------------------------------------------------- - name: "Fail if masbr_job_component is not provided" @@ -45,7 +43,6 @@ - "masbr_job_component.name is required" - "masbr_job_component.namespace is required" - # Check 'masbr_job_data_list' # ----------------------------------------------------------------------------- - name: "Set fact: init masbr_job_data_list" @@ -86,14 +83,13 @@ set_fact: masbr_job_data_list: "{{ masbr_job_data_init }}" - # Find restore-from job name # ----------------------------------------------------------------------------- - name: "Find the restore-from job name" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/list_storage_job_folders.yml" vars: - - masbr_ls_job_type: "backup" - - masbr_ls_filter: "| grep '^{{ masbr_job_component.name }}-.*-{{ masbr_restore_from_version }}$'" + masbr_ls_job_type: "backup" + masbr_ls_filter: "| grep '^{{ masbr_job_component.name }}-.*-{{ masbr_restore_from_version }}$'" - name: "Fail if not found the restore-from job name" assert: @@ -104,7 +100,6 @@ set_fact: masbr_restore_from: "{{ masbr_ls_results[0] }}" - # Set restore job variables # ----------------------------------------------------------------------------- - name: "Set fact: restore job variables" @@ -121,13 +116,11 @@ # At this point, set it as the same value of masbr_job_name masbr_job_name_final: "{{ masbr_job_name }}" - # Create local job folder # ----------------------------------------------------------------------------- - name: "Create local job folder" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_local_job_folder.yml" - # Get restore-from job information # ----------------------------------------------------------------------------- - name: "Get restore-from job information" @@ -167,7 +160,6 @@ {{ true if (masbr_job_component.instance is defined and masbr_job_component.instance | length > 0 and masbr_restore_from_yaml.source.instance != masbr_job_component.instance) else false }} - # Trying to restore from an incremental backup, also need to check the existance of the based on full backup # ----------------------------------------------------------------------------- - name: "Check the existence of the based on full backup job" @@ -185,8 +177,8 @@ - name: "Check the existence of the based on full backup job" include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/list_storage_job_folders.yml" vars: - - masbr_ls_job_type: "backup" - - masbr_ls_filter: "| grep {{ masbr_restore_basedon }}" + masbr_ls_job_type: "backup" + masbr_ls_filter: "| grep {{ masbr_restore_basedon }}" - name: "Fail if not found the based on full backup job" assert: @@ -195,7 +187,6 @@ Not found the based on full backup job folder: {{ masbr_storage_job_type_folder }}/{{ masbr_restore_basedon }} - # Show restore job information # ----------------------------------------------------------------------------- - name: "Debug: restore job information" diff --git a/ibm/mas_devops/common_tasks/backup_restore/confirm_cluster_info.yml b/ibm/mas_devops/common_tasks/backup_restore/confirm_cluster_info.yml index 00041570ba..097623b698 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/confirm_cluster_info.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/confirm_cluster_info.yml @@ -16,7 +16,6 @@ debug: msg: "Cluster domain ........................ {{ masbr_cluster_domain }}" - # Confirm the cluster information # ----------------------------------------------------------------------------- - name: "Confirm the connected cluster information" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml index 2601a6557b..44547d055c 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml @@ -6,9 +6,15 @@ masbr_storage_job_folder: >- {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} -- name: "Debug: local storage job folder" +- name: "Debug: All PV variables" debug: - msg: "Local storage job folder ......... {{ masbr_storage_job_folder }}" + msg: + - "Pod Name ................... {{ masbr_cf_pod_name }}" + - "Local storage job folder ......... {{ masbr_storage_job_folder }}" + - "Folder setup .................... {{ masbr_cf_paths }}" + - "Source Folder .................... {{ item.src_folder }}" + - "Destination Folder ............... {{ [masbr_storage_job_folder, item.dest_folder] | path_join }}" + loop: "{{ masbr_cf_paths }}" # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* - name: "Copy files from pod folder to local storage folder" @@ -30,7 +36,7 @@ shell: >- mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_file }} + {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ [item.src_folder, item.src_file] | path_join }} {{ [masbr_storage_job_folder, item.dest_folder, item.src_file|basename] | path_join }} loop: "{{ masbr_cf_paths }}" @@ -42,6 +48,6 @@ shell: >- mkdir -p {{ [masbr_storage_job_folder, item.dest_file|dirname] | path_join }} && oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_file }} + {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ [item.src_folder, item.src_file] | path_join }} {{ [masbr_storage_job_folder, item.dest_file] | path_join }} loop: "{{ masbr_cf_paths }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml index 6aa3eb0247..c6e3818f80 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml @@ -8,23 +8,28 @@ masbr_storage_job_folder: >- {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} -- name: "Debug: local storage job folder" +- name: "Debug: All PV variables" debug: - msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" + msg: + - "Local storage job folder ......... {{ masbr_storage_job_folder }}" + - "Folder setup .................... {{ masbr_cf_paths }}" + - "Source Folder .................... {{ [masbr_storage_job_folder, item.src_folder] | path_join }}" + - "Destination Folder ............... {{ item.dest_folder }}" + loop: "{{ masbr_cf_paths }}" # Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* # # - exec into masbr_cf_pod_name/masbr_cf_container_name, create temp folder # - cp from src_folder to temp folder inside masbr_cf_pod_name/masbr_cf_container_name -# - exec into masbr_cf_pod_name/masbr_cf_container_name, move temp_dest_folder to dest_folder and delete temp_dest_folder +# - exec into masbr_cf_pod_name/masbr_cf_container_name, move item.temp_dest_folder to dest_folder and delete temp_dest_folder - name: "Copy files from local storage folder to pod folder" when: - item.src_folder is defined and item.src_folder | length > 0 - item.dest_folder is defined and item.dest_folder | length > 0 shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'temp_dest_folder={{ [item.dest_folder, masbr_job_version] | path_join }} && mkdir -p ${temp_dest_folder}' \ - && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_folder] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:${temp_dest_folder} \ - && oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mv -f ${temp_dest_folder}/* {{ item.dest_folder }} && rm -rf ${temp_dest_folder}' + oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mkdir -p {{ [item.dest_folder, masbr_job_version] | path_join }}' \ + && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_folder] | path_join }}/* {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ [item.dest_folder, masbr_job_version] | path_join }} \ + && oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mv -f {{ [item.dest_folder, masbr_job_version] | path_join }}/* {{ item.dest_folder }} && rm -rf {{ [item.dest_folder, masbr_job_version] | path_join }}' loop: "{{ masbr_cf_paths }}" # Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file @@ -34,18 +39,18 @@ - item.dest_folder is defined and item.dest_folder | length > 0 shell: >- oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mkdir -p {{ item.dest_folder }}' \ - && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_file] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.dest_folder }} + && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_folder, item.src_file] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.dest_folder }} loop: "{{ masbr_cf_paths }}" # Condition 3. src_file -> dest_file # - exec into masbr_cf_pod_name/masbr_cf_container_name, create temp folder # - cp from src_folder to temp folder inside masbr_cf_pod_name/masbr_cf_container_name -# - exec into masbr_cf_pod_name/masbr_cf_container_name, move temp_dest_folder to dest_folder and delete temp_dest_folder +# - exec into masbr_cf_pod_name/masbr_cf_container_name, move temp_dest_folder to item.dest_folder and delete temp_dest_folder - name: "Copy file from local storage folder to pod file" when: - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 + - item.dest_folder is defined and item.dest_folder | length > 0 shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'temp_dest_folder={{ [item.dest_file|dirname, masbr_job_version] | path_join }} && mkdir -p ${temp_dest_folder}' \ - && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_file] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:${temp_dest_folder} \ - && oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mv -f ${temp_dest_folder}/{{ item.src_file|basename }} {{ item.dest_file }} && rm -rf ${temp_dest_folder}' + oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mkdir -p {{ [item.dest_folder, masbr_job_version] | path_join }}' \ + && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [temp_src_folder, item.src_file] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ [item.dest_folder, masbr_job_version] | path_join }} \ + && oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mv -f {{ [item.dest_folder, masbr_job_version] | path_join }}/{{ item.src_file|basename }} {{ item.dest_folder }} && rm -rf {{ [item.dest_folder, masbr_job_version] | path_join }}' diff --git a/ibm/mas_devops/common_tasks/backup_restore/create_cleanup_job.yml b/ibm/mas_devops/common_tasks/backup_restore/create_cleanup_job.yml index 5c4817d240..eacd6e9153 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/create_cleanup_job.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/create_cleanup_job.yml @@ -17,7 +17,6 @@ set_fact: masbr_cleanup_job_exists: true - # Create script configmap if cleanup Job not exists # ----------------------------------------------------------------------------- - name: "Create script configmap if cleanup Job not exists" @@ -45,7 +44,6 @@ script: "{{ _cleanup_sh_content.stdout }}" wait: true - # Create or update cleanup Job # ----------------------------------------------------------------------------- - name: "Set fact: cleanup Job variables" diff --git a/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml b/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml index fa271acc47..fe869342e1 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml @@ -3,7 +3,6 @@ set_fact: masbr_job_name_final: "{{ masbr_job_name }}-{{ masbr_job_status.phase }}" - # Rename the job folder in local storage # ----------------------------------------------------------------------------- - name: "Set fact: local storage job folder" diff --git a/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml b/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml index fd61b8675c..98f751b30f 100644 --- a/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml +++ b/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml @@ -124,7 +124,6 @@ 'completionTimestamp': '%Y-%m-%dT%H:%M:%S' | strftime }) }} - # Create job file # ----------------------------------------------------------------------------- - name: "Debug: update job variables" @@ -139,7 +138,6 @@ src: "{{ role_path }}/../../common_tasks/templates/backup_restore/{{ masbr_job_type }}.yml.j2" dest: "{{ masbr_local_job_folder }}/{{ masbr_job_type }}.yml" - # Copy local job files to specified storage location # ----------------------------------------------------------------------------- - name: "Copy local job files to specified storage location" @@ -151,7 +149,6 @@ - src_file: "{{ masbr_job_type }}.yml" dest_folder: "" - # Append job final status to the job folder name # ----------------------------------------------------------------------------- - name: "Append job final status to the job folder name" diff --git a/ibm/mas_devops/common_tasks/default_storage_classes.yml b/ibm/mas_devops/common_tasks/default_storage_classes.yml index f31bc5c7cd..7b805de008 100644 --- a/ibm/mas_devops/common_tasks/default_storage_classes.yml +++ b/ibm/mas_devops/common_tasks/default_storage_classes.yml @@ -1,5 +1,4 @@ --- - # Lookup & use default supported storage class # ----------------------------------------------------------------------------- # See: ibm/mas_devops/plugins/action/get_default_storage_classes.py diff --git a/ibm/mas_devops/common_tasks/detect_airgap.yml b/ibm/mas_devops/common_tasks/detect_airgap.yml index ecbf958465..f58e883a1e 100644 --- a/ibm/mas_devops/common_tasks/detect_airgap.yml +++ b/ibm/mas_devops/common_tasks/detect_airgap.yml @@ -1,5 +1,4 @@ --- - # 1. Look for the legacy ICSP # ----------------------------------------------------------------------------- - name: "detect-airgap : Look for the MAS ImageContentSourcePolicy" @@ -9,7 +8,6 @@ name: ibm-mas-and-dependencies register: icsp_lookup - # 2. Look for the new IDMS # ----------------------------------------------------------------------------- - name: "detect-airgap : Look for the MAS ImageDigestMirrorSet" @@ -20,7 +18,6 @@ - mas.ibm.com/idmsContent=ibm register: idms_lookup - # 3. Set the airgap boolena and print debug # ----------------------------------------------------------------------------- - name: "detect-airgap : Set airgap_install property" diff --git a/ibm/mas_devops/common_tasks/detect_sno.yml b/ibm/mas_devops/common_tasks/detect_sno.yml index da2fc7c7e1..485b620dfe 100644 --- a/ibm/mas_devops/common_tasks/detect_sno.yml +++ b/ibm/mas_devops/common_tasks/detect_sno.yml @@ -1,10 +1,8 @@ --- - # 1. Verify if the cluster is single node - name: "detect-sno : Get the number of nodes" - shell: - oc get nodes --no-headers | wc -l + shell: oc get nodes --no-headers | wc -l register: nodes_count - debug: @@ -13,8 +11,7 @@ - name: "detect-sno : Set sno_mode Environment variable" set_fact: sno_mode: true - when: - nodes_count is defined and nodes_count.stdout|int == 1 + when: nodes_count is defined and nodes_count.stdout|int == 1 - name: "detect-sno : Debug sno detection" debug: diff --git a/ibm/mas_devops/common_tasks/get_version_from_channel.yml b/ibm/mas_devops/common_tasks/get_version_from_channel.yml index 8700f298f7..c6c7c08388 100644 --- a/ibm/mas_devops/common_tasks/get_version_from_channel.yml +++ b/ibm/mas_devops/common_tasks/get_version_from_channel.yml @@ -19,7 +19,7 @@ name: "{{ op_pm_name }}" namespace: openshift-marketplace register: op_pm_lookup - no_log: true # This generates a huge amount of (useless) logging + no_log: true # This generates a huge amount of (useless) logging - name: Assert that PackageManifest exists ansible.builtin.assert: diff --git a/ibm/mas_devops/common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2 b/ibm/mas_devops/common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2 index 29e7f12ef9..89bf2fd25a 100644 --- a/ibm/mas_devops/common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2 +++ b/ibm/mas_devops/common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2 @@ -47,6 +47,13 @@ function backupResources { done } +if command -v yq &> /dev/null; then + echo "yq is installed, continuing" +else + echo "yq not found! please install yq before performing a backup" + exit 1 +fi + {% if masbr_ns_backup_resources is defined and masbr_ns_backup_resources | length > 0 %} {% for ns_resources in masbr_ns_backup_resources %} diff --git a/ibm/mas_devops/common_tasks/wait-machine-config-update.yml b/ibm/mas_devops/common_tasks/wait-machine-config-update.yml index 77ac01e414..d895a167ba 100644 --- a/ibm/mas_devops/common_tasks/wait-machine-config-update.yml +++ b/ibm/mas_devops/common_tasks/wait-machine-config-update.yml @@ -11,14 +11,13 @@ wait_timeout: 15 # give 15 seconds for the machines to start updating, if necessary wait_condition: type: Updating - status: 'True' + status: "True" register: _mcp ignore_errors: true loop: - "worker" - "master" - # 2. Wait for node pools to finish updating # ----------------------------------------------------------------------------- - name: Wait for node pools to finish updating @@ -31,7 +30,7 @@ wait_timeout: 60 # 1 min wait wait_condition: type: Updated - status: 'True' + status: "True" register: _mcp retries: 30 # Approx 1 hour delay: 120 # 2 minutes diff --git a/ibm/mas_devops/common_tasks/wait_for_crd.yml b/ibm/mas_devops/common_tasks/wait_for_crd.yml index e78ecfecb5..d01774cfe6 100644 --- a/ibm/mas_devops/common_tasks/wait_for_crd.yml +++ b/ibm/mas_devops/common_tasks/wait_for_crd.yml @@ -25,4 +25,4 @@ until: - _crd_info.resources is defined - _crd_info.resources | length > 0 - no_log: true # This generates a huge amount of (useless) logging + no_log: true # This generates a huge amount of (useless) logging diff --git a/ibm/mas_devops/common_vars/backup_restore.yml b/ibm/mas_devops/common_vars/backup_restore.yml index 32f0140fb1..10c54bb15f 100644 --- a/ibm/mas_devops/common_vars/backup_restore.yml +++ b/ibm/mas_devops/common_vars/backup_restore.yml @@ -21,20 +21,17 @@ masbr_cleanup_ttl_sec: "{{ lookup('env', 'MASBR_CLEANUP_TTL_SEC') | default('604 # https://en.wikipedia.org/wiki/List_of_tz_database_time_zones masbr_job_timezone: "{{ lookup('env', 'MASBR_JOB_TIMEZONE') | default('', true) }}" - # Docker image tag # ----------------------------------------------------------------------------- masbr_mascli_image_tag: "{{ lookup('env', 'MASBR_MASCLI_IMAGE_TAG') | default('latest', true) }}" masbr_mascli_image_pull_policy: "{{ lookup('env', 'MASBR_MASCLI_IMAGE_PULL_POLICY') | default('', true) }}" - # Storage variables # ----------------------------------------------------------------------------- # Local temp folder for backup/restore masbr_local_temp_folder: "{{ lookup('env', 'MASBR_LOCAL_TEMP_FOLDER') | default('/tmp/masbr', true) }}" masbr_storage_local_folder: "{{ lookup('env', 'MASBR_STORAGE_LOCAL_FOLDER') }}" - # Notification variables # ----------------------------------------------------------------------------- # Supported notification levels: diff --git a/ibm/mas_devops/common_vars/compatibility_matrix.yml b/ibm/mas_devops/common_vars/compatibility_matrix.yml index b52044e900..3c40610bdd 100644 --- a/ibm/mas_devops/common_vars/compatibility_matrix.yml +++ b/ibm/mas_devops/common_vars/compatibility_matrix.yml @@ -1,13 +1,13 @@ --- compatibility_matrix: 9.1.x-feature: - assist: [9.0.x] - iot: [9.0.x] - manage: [9.0.x, 9.1.x-feature] - monitor: [9.0.x] - optimizer: [9.0.x, 9.1.x-feature] - predict: [9.0.x] - visualinspection: [9.0.x, 9.1.x-feature] + assist: [9.0.x] + iot: [9.0.x] + manage: [9.0.x, 9.1.x-feature] + monitor: [9.0.x] + optimizer: [9.0.x, 9.1.x-feature] + predict: [9.0.x] + visualinspection: [9.0.x, 9.1.x-feature] 9.0.x: assist: [8.8.x, 9.0.x] iot: [8.8.x, 9.0.x] @@ -43,7 +43,6 @@ compatibility_matrix: predict: [8.6.x, 8.7.x] visualinspection: [8.6.x, 8.7.x] - # There is probably a better way to do this by manipulating the channel string, but this is fast and cheap! upgrade_requirement: core: @@ -83,7 +82,6 @@ upgrade_requirement: 8.9.x: 8.8.x 8.8.x: 8.7.x - # There is probably a better way to do this by manipulating the channel string, but this is fast and cheap! upgrade_path: 9.0.x: 9.1.x-feature diff --git a/ibm/mas_devops/meta/runtime.yml b/ibm/mas_devops/meta/runtime.yml index 8accf7c4c5..1a82761f6e 100644 --- a/ibm/mas_devops/meta/runtime.yml +++ b/ibm/mas_devops/meta/runtime.yml @@ -1,2 +1,2 @@ --- -requires_ansible: '>=2.10.3' +requires_ansible: ">=2.10.3" diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index 94d7b9b9b1..597d66f992 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -38,7 +38,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" diff --git a/ibm/mas_devops/playbooks/br_health.yml b/ibm/mas_devops/playbooks/br_health.yml index 5c440bc50d..3690d3e585 100644 --- a/ibm/mas_devops/playbooks/br_health.yml +++ b/ibm/mas_devops/playbooks/br_health.yml @@ -44,7 +44,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" diff --git a/ibm/mas_devops/playbooks/br_iot.yml b/ibm/mas_devops/playbooks/br_iot.yml index cb5d2d2a56..4f40665b8e 100644 --- a/ibm/mas_devops/playbooks/br_iot.yml +++ b/ibm/mas_devops/playbooks/br_iot.yml @@ -44,7 +44,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" diff --git a/ibm/mas_devops/playbooks/br_manage.yml b/ibm/mas_devops/playbooks/br_manage.yml index a14e6f80d3..be69727303 100644 --- a/ibm/mas_devops/playbooks/br_manage.yml +++ b/ibm/mas_devops/playbooks/br_manage.yml @@ -39,7 +39,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml index c15a65af83..3f8200c514 100644 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -34,7 +34,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" diff --git a/ibm/mas_devops/playbooks/br_monitor.yml b/ibm/mas_devops/playbooks/br_monitor.yml index ef2c68f88a..abd9812fa5 100644 --- a/ibm/mas_devops/playbooks/br_monitor.yml +++ b/ibm/mas_devops/playbooks/br_monitor.yml @@ -44,7 +44,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" diff --git a/ibm/mas_devops/playbooks/br_optimizer.yml b/ibm/mas_devops/playbooks/br_optimizer.yml index 041d209d97..244a22acf4 100644 --- a/ibm/mas_devops/playbooks/br_optimizer.yml +++ b/ibm/mas_devops/playbooks/br_optimizer.yml @@ -44,7 +44,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" diff --git a/ibm/mas_devops/playbooks/br_visualinspection.yml b/ibm/mas_devops/playbooks/br_visualinspection.yml index feeddc1ca1..3599ff96d9 100644 --- a/ibm/mas_devops/playbooks/br_visualinspection.yml +++ b/ibm/mas_devops/playbooks/br_visualinspection.yml @@ -37,7 +37,6 @@ that: masbr_action in ["backup", "restore"] fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - # Common checks before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" diff --git a/ibm/mas_devops/playbooks/mirror_add_iot.yml b/ibm/mas_devops/playbooks/mirror_add_iot.yml index 17cbd83a4b..57cb170dbf 100644 --- a/ibm/mas_devops/playbooks/mirror_add_iot.yml +++ b/ibm/mas_devops/playbooks/mirror_add_iot.yml @@ -32,7 +32,6 @@ manifest_name: ibm-mas-iot manifest_version: "{{ lookup('env', 'MAS_IOT_VERSION') | default (mas_catalog_metadata.mas_iot_version[mas_channel], True) | replace('_', '.') }}" - # 2. Eclipse Amlen # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_extras_prepare diff --git a/ibm/mas_devops/playbooks/mirror_add_monitor.yml b/ibm/mas_devops/playbooks/mirror_add_monitor.yml index 3454e3a0c4..7b8d1c599e 100644 --- a/ibm/mas_devops/playbooks/mirror_add_monitor.yml +++ b/ibm/mas_devops/playbooks/mirror_add_monitor.yml @@ -32,7 +32,6 @@ manifest_name: ibm-data-dictionary manifest_version: "{{ lookup('env', 'DATA_DICTIONARY_VERSION') | default (mas_catalog_metadata.dd_version, True) | replace('_', '.') }}" - # 2. IBM Maximo Monitor # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare diff --git a/ibm/mas_devops/playbooks/mirror_dependencies.yml b/ibm/mas_devops/playbooks/mirror_dependencies.yml index 2c6c3843c7..ab4972e4ca 100644 --- a/ibm/mas_devops/playbooks/mirror_dependencies.yml +++ b/ibm/mas_devops/playbooks/mirror_dependencies.yml @@ -99,7 +99,6 @@ manifest_name: catalog manifest_version: "{{ catalog_tag }}" - # 2. MongoDb Community Edition # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_extras_prepare @@ -161,7 +160,6 @@ manifest_name: extras_mongoce manifest_version: "{{ mas_catalog_metadata.mongo_extras_version_6 }}" - # Mirror Mongo v7 specifically when requested - role: ibm.mas_devops.mirror_extras_prepare when: @@ -192,7 +190,6 @@ - ibm-cpp - ibm-cs-healthcheck - ibm-cs-monitoring - - ibm-events-operator ibmpak_skip_dependencies: false - role: ibm.mas_devops.mirror_images @@ -214,7 +211,6 @@ - ibm-cpp - ibm-cs-healthcheck - ibm-cs-monitoring - - ibm-events-operator ibmpak_skip_dependencies: false - role: ibm.mas_devops.mirror_images @@ -240,7 +236,6 @@ - ibm-cpp - ibm-cs-healthcheck - ibm-cs-monitoring - - ibm-events-operator ibmpak_skip_dependencies: false - role: ibm.mas_devops.mirror_images @@ -266,7 +261,6 @@ - ibm-cpp - ibm-cs-healthcheck - ibm-cs-monitoring - - ibm-events-operator ibmpak_skip_dependencies: false - role: ibm.mas_devops.mirror_images @@ -278,7 +272,6 @@ manifest_name: ibm-cp-common-services manifest_version: "{{ mas_catalog_metadata.common_svcs_version_2 }}" - # 4. IBM Suite License Service # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -297,7 +290,6 @@ manifest_name: ibm-sls manifest_version: "{{ lookup('env', 'SLS_VERSION') | default (mas_catalog_metadata.sls_version, True) }}" - # 5. IBM Truststore Manager # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -316,7 +308,6 @@ manifest_name: ibm-truststore-mgr manifest_version: "{{ lookup('env', 'TSM_VERSION') | default (mas_catalog_metadata.tsm_version, True) }}" - # 6.1 CP4D Platform # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -408,7 +399,6 @@ manifest_name: ibm-licensing manifest_version: "{{ mas_catalog_metadata.ibm_licensing_version }}" - # 6.3 IBM Watson Studio and Watson Machine Learning dependency - ibm-ccs # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -427,7 +417,6 @@ manifest_name: ibm-ccs manifest_version: "{{ mas_catalog_metadata.wsl_version }}" - # 7.1 IBM Watson Studio dependency - ibm-datarefinery # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -500,7 +489,6 @@ manifest_name: ibm-wsl manifest_version: "{{ mas_catalog_metadata.wsl_version }}" - # 8. IBM Watson Machine Learning # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -519,7 +507,6 @@ manifest_name: ibm-wml-cpd manifest_version: "{{ mas_catalog_metadata.wml_version }}" - # 9. Analytics Engine (Spark) # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -538,7 +525,6 @@ manifest_name: ibm-analyticsengine manifest_version: "{{ mas_catalog_metadata.spark_version }}" - # 10. IBM Db2u # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -571,7 +557,6 @@ manifest_name: extras_db2u manifest_version: "{{ mas_catalog_metadata.db2u_extras_version }}" - # 11. Cognos Analytics # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -590,7 +575,6 @@ manifest_name: ibm-cognos-analytics-prod manifest_version: "{{ mas_catalog_metadata.cognos_version }}" - # 12. App Connect # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare @@ -609,7 +593,6 @@ manifest_name: ibm-appconnect manifest_version: "{{ mas_catalog_metadata.appconnect_version }}" - # 13. ODF - some extras needed # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_extras_prepare diff --git a/ibm/mas_devops/playbooks/ocp_convert_to_disconnected.yml b/ibm/mas_devops/playbooks/ocp_convert_to_disconnected.yml index fe063536db..8f39fd92c3 100644 --- a/ibm/mas_devops/playbooks/ocp_convert_to_disconnected.yml +++ b/ibm/mas_devops/playbooks/ocp_convert_to_disconnected.yml @@ -17,5 +17,5 @@ roles: - ibm.mas_devops.ocp_config - - ibm.mas_devops.ocp_contentsourcepolicy + - ibm.mas_devops.ocp_idms - ibm.mas_devops.ocp_simulate_disconnected_network diff --git a/ibm/mas_devops/playbooks/oneclick_add_manage.yml b/ibm/mas_devops/playbooks/oneclick_add_manage.yml index de6756e9f9..02b72fe282 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_manage.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_manage.yml @@ -43,7 +43,6 @@ configure_manage_attachments: "{{ lookup('env', 'CONFIGURE_MANAGE_ATTACHMENTS') | default('False', true) | bool }}" configure_manage_bim: "{{ lookup('env', 'CONFIGURE_MANAGE_BIM') | default('False', true) | bool }}" - # Cloud Pak for Data Configuration # ------------------------------------------------------------------------- cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default('4.6.3', true) }}" @@ -94,7 +93,6 @@ - "Attachments ............................. {{ configure_manage_attachments }}" - "Building Information Models ............. {{ configure_manage_bim }}" - roles: # Optional Cloud Pak for Data Install - name: ibm.mas_devops.cp4d diff --git a/ibm/mas_devops/playbooks/oneclick_add_optimizer.yml b/ibm/mas_devops/playbooks/oneclick_add_optimizer.yml index ea7abe713c..6148d7f415 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_optimizer.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_optimizer.yml @@ -7,7 +7,6 @@ - hosts: localhost any_errors_fatal: true vars: - # Application Installation mas_app_channel: "{{ lookup('env', 'MAS_APP_CHANNEL') | default('9.0.x', true) }}" mas_app_id: optimizer diff --git a/ibm/mas_devops/playbooks/oneclick_add_predict.yml b/ibm/mas_devops/playbooks/oneclick_add_predict.yml index 023ce608ff..9e64f9a8c6 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_predict.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_predict.yml @@ -42,29 +42,29 @@ pre_tasks: - include_role: - # Cloud Pak for Data Platform (~1 1/2 hours) + # Cloud Pak for Data Platform (~1 1/2 hours) name: ibm.mas_devops.cp4d when: install_cp4d_platform == true - include_role: - # Watson Studio (~3 hours) + # Watson Studio (~3 hours) name: ibm.mas_devops.cp4d_service when: install_watson_studio == true vars: cpd_service_name: wsl - include_role: - # Watson Machine Learning (~2 1/2 hours) + # Watson Machine Learning (~2 1/2 hours) name: ibm.mas_devops.cp4d_service when: install_watson_machine_learning == true vars: cpd_service_name: wml - include_role: - # Analytics Engine Powered by Apache Spark (~30 minutes) + # Analytics Engine Powered by Apache Spark (~30 minutes) name: ibm.mas_devops.cp4d_service when: install_analytics_engine == true vars: cpd_service_name: spark - include_role: - # Watson OpenScale (~1 hour) + # Watson OpenScale (~1 hour) name: ibm.mas_devops.cp4d_service when: install_watson_openscale == true vars: diff --git a/ibm/mas_devops/playbooks/oneclick_add_visualinspection.yml b/ibm/mas_devops/playbooks/oneclick_add_visualinspection.yml index 47e76edb18..93866a428c 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_visualinspection.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_visualinspection.yml @@ -20,7 +20,7 @@ configure_cos_bucket: "{{ lookup('env', 'CONFIGURE_COS_BUCKET') | default('false', true) | bool }}" cos_type: "{{ lookup('env', 'COS_TYPE') }}" mas_app_settings_visualinspection_object_storage_enabled: "{{ lookup('env', 'MAS_APP_SETTINGS_VISUALINSPECTION_OBJECT_STORAGE_ENABLED') | default('false', true) | bool }}" - mas_app_settings_visualinspection_object_storage_bucket_default: 'mvi-bucket-{{ mas_instance_id }}-{{ mas_workspace_id }}' + mas_app_settings_visualinspection_object_storage_bucket_default: "mvi-bucket-{{ mas_instance_id }}-{{ mas_workspace_id }}" mas_app_settings_visualinspection_object_storage_workspace: "{{ lookup('env', 'MAS_APP_SETTINGS_VISUALINSPECTION_OBJECT_STORAGE_WORKSPACE') | default(mas_app_settings_visualinspection_object_storage_bucket_default, true) }}" pre_tasks: @@ -33,20 +33,20 @@ - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" fail_msg: "One or more required environment variables are not defined" - include_role: - # Configure Object Storage (15 min) - # With this we are allowing options for users to also provision Object Storage instance if they don't have one setup yet - # While using cos role, users will be able to configure either OCS/ODF or IBM Cloud Object Storage instances to be used - # This will also generate the ObjectStorageCfg file that will be used to configure the target Object Storage instance - # into the target MAS instance. + # Configure Object Storage (15 min) + # With this we are allowing options for users to also provision Object Storage instance if they don't have one setup yet + # While using cos role, users will be able to configure either OCS/ODF or IBM Cloud Object Storage instances to be used + # This will also generate the ObjectStorageCfg file that will be used to configure the target Object Storage instance + # into the target MAS instance. name: ibm.mas_devops.cos when: - mas_app_settings_visualinspection_object_storage_enabled - configure_cos - cos_type in ['ibm','ocs'] - include_role: - # Configure COS bucket in IBM Cloud Object Storage or AWS (10 min) - # While using the cos_bucket role, users will be able to create a bucket within the IBM cloud object storage instance - # or in AWS users can allocate an S3 bucket (no need of a instance created, s3 buckets are directly created in the AWS account) + # Configure COS bucket in IBM Cloud Object Storage or AWS (10 min) + # While using the cos_bucket role, users will be able to create a bucket within the IBM cloud object storage instance + # or in AWS users can allocate an S3 bucket (no need of a instance created, s3 buckets are directly created in the AWS account) name: ibm.mas_devops.cos_bucket when: - mas_app_settings_visualinspection_object_storage_enabled diff --git a/ibm/mas_devops/playbooks/oneclick_upgrade.yml b/ibm/mas_devops/playbooks/oneclick_upgrade.yml index 35836b69e2..805c19044f 100644 --- a/ibm/mas_devops/playbooks/oneclick_upgrade.yml +++ b/ibm/mas_devops/playbooks/oneclick_upgrade.yml @@ -1,5 +1,4 @@ --- - - hosts: localhost any_errors_fatal: true vars: @@ -33,7 +32,6 @@ - name: ibm.mas_devops.suite_verify when: upgrade_core - # 2. Applications # ------------------------------------------------------------------------- # 2.1 Assist diff --git a/ibm/mas_devops/requirements.txt b/ibm/mas_devops/requirements.txt index 6974c01c5a..2055408de0 100644 --- a/ibm/mas_devops/requirements.txt +++ b/ibm/mas_devops/requirements.txt @@ -1 +1 @@ -mas-devops >= 1.11.1 +mas-devops >= 2.0.0, < 3 diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 53d4f1a1d9..404ec57fe3 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -41,7 +41,6 @@ apply: yes definition: "{{ lookup('template', 'templates/aibroker/subscription.yml.j2') }}" - # Wait until the IBM Maximo AI Broker CRD is available # ----------------------------------------------------------------------------- - name: "Wait until the IBM Maximo AI Broker Operator CRD is available" diff --git a/ibm/mas_devops/roles/aibroker/tasks/s3/remove/main.yml b/ibm/mas_devops/roles/aibroker/tasks/s3/remove/main.yml index ddbd3f49f2..927a909810 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/s3/remove/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/s3/remove/main.yml @@ -22,7 +22,7 @@ file: path: "{{ ansible_env.HOME }}/.aws" state: directory - mode: '0700' + mode: "0700" - name: Create AWS credentials file copy: @@ -31,7 +31,7 @@ [default] aws_access_key_id = {{ mas_aibroker_storage_accesskey }} aws_secret_access_key = {{ mas_aibroker_storage_secretkey }} - mode: '0600' + mode: "0600" - name: Create AWS config file copy: @@ -39,7 +39,7 @@ content: | [default] region = {{ mas_aibroker_storage_region }} - mode: '0600' + mode: "0600" - name: Delete S3 bucket if script succeeds command: python3 {{ role_path }}/files/delete_s3.py "{{ decoded_secret_value.stdout }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml index 89a4894a1f..49b80c6dc6 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/saas/main.yml @@ -1,87 +1,25 @@ --- -# TODO: create route - -# Adding inference server runtimes -# - name: 'create controller route' -# kubernetes.core.k8s: -# apply: yes -# definition: "{{ lookup('template', 'templates/saas/create-controller-route.yml.j2') }}" - -# - name: Apply Kmodel api -# shell: | -# curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id=provision-tenant" \ -# --header 'accept: */*' \ -# --header 'Content-Type: application/json' \ -# --data '{"name": "string","public": true,"max_models": 10}' -# register: kmodel_api_output - -# - name: Apply Kmodel api - using uri -# uri: -# url: https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id=provision-tenant -# validate_certs: no -# method: POST -# headers: -# accept: "*/*" -# Content-Type: application/json -# body_format: "json" -# body: "{\"name\": \"string\",\"public\": true,\"max_models\": 10}" -# status_code: [200,409] -# timeout: 30 -# register: kmodel_api_output -# - name: Kmodel debug #TODO: remove before commit -# debug: -# msg: "POST output ...... {{ kmodel_api_output }}" -- name: "Check if API key secret exists in namespace: provision-tenant" +- name: "Check if API key secret exists in namespace: {{ mas_aibroker_provision_tenant }}" k8s_info: api_version: v1 kind: Secret - name: "provision-tenant----apikey-secret" + name: "{{ mas_aibroker_provision_tenant }}----apikey-secret" namespace: "{{ aibroker_namespace }}" register: apikey_secret_info - name: Create API Key - script: "{{ role_path }}/files/create_apikey.sh provision-tenant {{ aibroker_namespace }}" + script: "{{ role_path }}/files/create_apikey.sh {{ mas_aibroker_provision_tenant }} {{ aibroker_namespace }}" when: - apikey_secret_info.resources | length == 0 -- name: "Lookup tenant provision-tenant api key" - shell: oc get secret provision-tenant----apikey-secret -n {{ aibroker_namespace }} -o json | jq -r '.data.AIBROKER_APIKEY' | base64 -d +- name: "Lookup tenant {{ mas_aibroker_provision_tenant }} api key" + shell: oc get secret {{ mas_aibroker_provision_tenant }}----apikey-secret -n {{ aibroker_namespace }} -o json | jq -r '.data.AIBROKER_APIKEY' | base64 -d register: provision_tenant_api_key_output - name: "Set aibroker api key variable" set_fact: provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" -- name: Debug provision-tenant api key #TODO: remove on finall commit +- name: Debug {{ mas_aibroker_provision_tenant }} api key #TODO: remove on final commit debug: msg: "Provision tenant api key: {{ provision_tenant_api_key }}" - -# - name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" -# shell: | -# curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ -# --header 'apikey: {{ provision_tenant_api_key }}' \ -# --header 'accept: */*' \ -# --header 'Content-Type: application/json' \ -# --header 'tenantid: provision-tenant' \ -# --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' -# register: add_tenant_to_db2 - -# # - name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri -# # uri: -# # url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" -# # validate_certs: no -# # method: POST -# # headers: -# # apikey: "{{ provision_tenant_api_key }}" -# # accept: "*/*" -# # Content-Type: "application/json" -# # tenantid: "provision-tenant" -# # body_format: "json" -# # body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"sls_url\": \"{{ mas_aibroker_sls_url }}\", \"dro_url\": \"{{ mas_aibroker_dro_url }}\"}" -# # status_code: [200] -# # timeout: 30 -# # register: add_tenant_to_db2 - -# - name: Debug tenant #TODO: remove on finall commit -# debug: -# msg: "{{ add_tenant_to_db2 }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 3d2b416d56..b94d78ea4d 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -1,88 +1,64 @@ --- -- name: 'Check if tenant namespace: {{ tenantNamespace }} exists' +- name: "Check if tenant namespace: {{ tenantNamespace }} exists" kubernetes.core.k8s_info: api_version: v1 kind: Namespace - name: '{{ tenantNamespace }}' + name: "{{ tenantNamespace }}" register: namespace_info # Create tenant namespace -- name: 'Create a tenant namespace: {{ tenantNamespace }}' +- name: "Create a tenant namespace: {{ tenantNamespace }}" kubernetes.core.k8s: - name: '{{ tenantNamespace }}' + name: "{{ tenantNamespace }}" api_version: v1 kind: Namespace when: - namespace_info.resources | length == 0 # Copy secrets to tenant namespace -- name: 'Copy secrets to namespace: {{ tenantNamespace }}' +- name: "Copy secrets to namespace: {{ tenantNamespace }}" shell: 'oc get secret {{ item }} -n mas-{{ mas_instance_id }}-aibroker -o yaml | sed "s/namespace: .*/namespace: {{ tenantNamespace }}/" | oc apply --force -f -' with_items: - km-s3-secret - - '{{ pullSecretName }}' + - "{{ pullSecretName }}" # Adding inference server runtimes -- name: 'Create config map for connector config' +- name: "Create config map for connector config" kubernetes.core.k8s: apply: yes definition: "{{ lookup('template', 'templates/tenant/connector-configmap.yml.j2') }}" -- name: 'Create SA for server runtimes' +- name: "Create SA for server runtimes" kubernetes.core.k8s: apply: yes definition: "{{ lookup('template', 'templates/tenant/km-s3-sa.yml.j2') }}" -# - name: 'Create huggingfaceserver server runtime' -# kubernetes.core.k8s: -# apply: yes -# definition: "{{ lookup('template', 'templates/tenant/kserve-huggingfaceserver.yml.j2') }}" +- name: "Create huggingfaceserver server runtime" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/kserve-huggingfaceserver.yml.j2') }}" -# - name: 'Create lgbserver server runtime' -# kubernetes.core.k8s: -# apply: yes -# definition: "{{ lookup('template', 'templates/tenant/kserve-lgbserver.yml.j2') }}" +- name: "Create lgbserver server runtime" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/kserve-lgbserver.yml.j2') }}" -# - name: 'Create sklearnserver server runtime' -# kubernetes.core.k8s: -# apply: yes -# definition: "{{ lookup('template', 'templates/tenant/kserve-sklearnserver.yml.j2') }}" +- name: "Create sklearnserver server runtime" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/kserve-sklearnserver.yml.j2') }}" -# - name: 'Create xgbserver server runtime' -# kubernetes.core.k8s: -# apply: yes -# definition: "{{ lookup('template', 'templates/tenant/kserve-xgbserver.yml.j2') }}" +- name: "Create xgbserver server runtime" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/kserve-xgbserver.yml.j2') }}" # Applay RBAC roles -- name: 'Apply RBAC for tenant' +- name: "Apply RBAC for tenant" kubernetes.core.k8s: apply: yes definition: "{{ lookup('template', 'templates/tenant/rbac.yml.j2') }}" -# - name: "debug domain" -# debug: -# msg: "Domain name ......... {{ domain }}" - -# - name: "Remove prefix from domain" -# set_fact: -# hostname: "{{ domain | regex_replace('^[^.]+\\.', '') }}" - -# - name: Apply Kmodel api -# shell: | -# curl -k -X POST "https://controller-mas-{{ mas_instance_id }}-aibroker.{{ mas_aibroker_domain }}/api/v1/tenant?id={{ tenantNamespace }}" \ -# --header 'accept: */*' \ -# --header 'Content-Type: application/json' \ -# --data '{"name": "string","public": true,"max_models": 10}' - -# - name: Create sls secret sample for AI Broker -# command: python3 {{ role_path }}/files/alm_sample_sls_use.py {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} -# register: create_sls_secret_output -# ignore_errors: yes - -# - name: Debug Create sls secret for AI Broker -# debug: -# msg: "{{ create_sls_secret_output }}" - - name: "Check if SLS key secret exists in namespace: {{ aibroker_namespace }}" k8s_info: api_version: v1 @@ -115,21 +91,30 @@ debug: msg: "{{ dro_token_output }}" +# - name: Wait for service /service/rest/api/v1/tenant #need more accurate way to check if service working +# pause: +# minutes: 8 +# prompt: "Waiting for for service /service/rest/api/v1/tenant" -# - name: "Create tenant {{ tenantNamespace }} and add to db2 for saas" -# shell: | -# curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" \ -# --header 'apikey: {{ provision_tenant_api_key }}' \ -# --header 'accept: */*' \ -# --header 'Content-Type: application/json' \ -# --header 'tenantid: provision-tenant' \ -# --data ' {"tenant_name": "{{ tenantNamespace }}", "sls_url": "{{ mas_aibroker_sls_url }}", "dro_url": "{{ mas_aibroker_dro_url }}"}' -# register: add_tenant_to_db2 - -- name: Wait for service /service/rest/api/v1/tenant +- name: Await to aibroker-api pod created pause: - minutes: 2 - prompt: "Waiting for for service /service/rest/api/v1/tenant" + minutes: 3 + prompt: "Waiting for aibroker-api pod" + +- name: "Wait for aibroker-api pod" + shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} + register: _is_not_exists + until: _is_not_exists.stdout|int == 0 + retries: 60 + delay: 10 + +- name: "Wait for aibroker-api pod to be ready (10s delay)" + shell: > + oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l + register: _is_not_ready + until: _is_not_ready.stdout|int == 0 + retries: 60 + delay: 10 - name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri uri: @@ -140,7 +125,7 @@ apikey: "{{ provision_tenant_api_key }}" accept: "*/*" Content-Type: "application/json" - tenantid: "provision-tenant" + tenantid: "{{ mas_aibroker_provision_tenant }}" body_format: "json" body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"sls_url\": \"{{ mas_aibroker_sls_url }}\", \"dro_url\": \"{{ mas_aibroker_dro_url }}\"}" status_code: [201] @@ -151,26 +136,6 @@ debug: msg: "{{ add_tenant_to_db2 }}" - -# - name: Create tenant entitlement -# command: python3 {{ role_path }}/files/create_tenant_entitlement.py {{ tenantNamespace }} standard pcc 2024-01-01 2027-01-01 -# register: create_tenant_entitlement -# ignore_errors: yes - -# - name: Debug Create tenant entitlement -# debug: -# msg: "{{ create_tenant_entitlement }}" - -# - name: "Create tenantEntitlements {{ tenantNamespace }} and add to db2 for saas" -# shell: | -# curl -k -X POST "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" \ -# --header 'apikey: {{ provision_tenant_api_key }}' \ -# --header 'accept: */*' \ -# --header 'Content-Type: application/json' \ -# --header 'tenantid: provision-tenant' \ -# --data '{"tenant_name": "{{ tenantNamespace }}", "entitlement_type": "standard", "model_type": "similarity", "entitlement_start_date": "2024-01-01", "entitlement_end_date": "2025-01-01"}' -# register: add_tenant_entitlements_to_db2 - - name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri uri: url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" @@ -180,7 +145,7 @@ apikey: "{{ provision_tenant_api_key }}" accept: "*/*" Content-Type: "application/json" - tenantid: "provision-tenant" + tenantid: "{{ mas_aibroker_provision_tenant }}" body_format: "json" body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"similarity\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" status_code: [201] @@ -200,7 +165,7 @@ apikey: "{{ provision_tenant_api_key }}" accept: "*/*" Content-Type: "application/json" - tenantid: "provision-tenant" + tenantid: "{{ mas_aibroker_provision_tenant }}" body_format: "json" body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"pcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" status_code: [201] @@ -220,7 +185,7 @@ apikey: "{{ provision_tenant_api_key }}" accept: "*/*" Content-Type: "application/json" - tenantid: "provision-tenant" + tenantid: "{{ mas_aibroker_provision_tenant }}" body_format: "json" body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"mcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" status_code: [201] diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 28fdd4c4c6..29d86e90d3 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -47,3 +47,4 @@ mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', tru mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" mas_aibroker_path_ca_crt: './certs' +mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" diff --git a/ibm/mas_devops/roles/appconnect/tasks/appconnectcfg.yml b/ibm/mas_devops/roles/appconnect/tasks/appconnectcfg.yml index a06ba607e6..129a09bf19 100644 --- a/ibm/mas_devops/roles/appconnect/tasks/appconnectcfg.yml +++ b/ibm/mas_devops/roles/appconnect/tasks/appconnectcfg.yml @@ -5,4 +5,4 @@ ansible.builtin.template: src: appconnectcfg.yml.j2 dest: "{{ mas_config_dir }}/appconnect-{{mas_instance_id}}-addons.yml" - mode: '664' + mode: "664" diff --git a/ibm/mas_devops/roles/appconnect/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/appconnect/tasks/determine-storage-classes.yml index 112f885650..db87bf7314 100644 --- a/ibm/mas_devops/roles/appconnect/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/appconnect/tasks/determine-storage-classes.yml @@ -4,7 +4,6 @@ - name: "determine-storage-classes : Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Set storage class # ----------------------------------------------------------------------------- - name: "Use default storage class (RWX)" diff --git a/ibm/mas_devops/roles/appconnect/tasks/main.yml b/ibm/mas_devops/roles/appconnect/tasks/main.yml index 6466978d53..e243cc4676 100644 --- a/ibm/mas_devops/roles/appconnect/tasks/main.yml +++ b/ibm/mas_devops/roles/appconnect/tasks/main.yml @@ -1,5 +1,4 @@ --- - # 1. Check for required facts # ----------------------------------------------------------------------------- - name: "Fail if AppConnect entitlement username has not been provided" @@ -66,15 +65,13 @@ - "MAS Instance Id ........................... {{ mas_instance_id }}" - "MAS Config Dir ............................ {{ mas_config_dir }}" - # 4. Create AppConnect project # ----------------------------------------------------------------------------- - name: "Create AppConnect namespace" kubernetes.core.k8s: api_version: v1 kind: Namespace - name: '{{ appconnect_namespace }}' - + name: "{{ appconnect_namespace }}" # 5. Create ibm-entitlement for AppConnect # ----------------------------------------------------------------------------- @@ -83,7 +80,7 @@ entitledAuthStr: "{{appconnect_entitlement_username}}:{{appconnect_entitlement_key}}" entitledAuth: "{{ entitledAuthStr | b64encode }}" content: - - "{\"auths\":{\"{{ appconnect_registry }}\":{\"username\":\"{{ appconnect_entitlement_username }}\",\"password\":\"{{ appconnect_entitlement_key }}\",\"auth\":\"{{entitledAuth}}\"}" + - '{"auths":{"{{ appconnect_registry }}":{"username":"{{ appconnect_entitlement_username }}","password":"{{ appconnect_entitlement_key }}","auth":"{{entitledAuth}}"}' - "}" - "}" kubernetes.core.k8s: @@ -97,7 +94,6 @@ stringData: .dockerconfigjson: "{{ content | join('') | string }}" - # 6. Deploy AppConnect subscription and Operator Group # ----------------------------------------------------------------------------- - name: "Create AppConnect operator group" @@ -112,7 +108,7 @@ wait: yes wait_timeout: 300 wait_condition: - type: 'CatalogSourcesUnhealthy' + type: "CatalogSourcesUnhealthy" status: "False" - name: "Wait until the AppConnect CRD is available" @@ -120,7 +116,6 @@ vars: crd_name: "dashboards.appconnect.ibm.com" - # 7. Wait until AppConnect Operator is ready # ----------------------------------------------------------------------------- - name: "Wait for ibm-appconnect-operator to be ready (60s delay)" @@ -140,14 +135,12 @@ retries: 30 # Approximately 1/2 hour before we give up delay: 60 # 1 minute - # 8. Create AppConnect Dashboard # ----------------------------------------------------------------------------- - name: "Create AppConnect Dashboard" kubernetes.core.k8s: definition: "{{ lookup('template', 'templates/dashboard.yml.j2') }}" - # 9. Wait AppConnect Dashboard to be complete # ----------------------------------------------------------------------------- - name: "Wait for AppConnect Dashboard to be ready (60s delay)" @@ -165,7 +158,6 @@ retries: 45 # approx 45 minutes before we give up delay: 60 # 1 minute - # 10. Retrieve AppConnect dashboard route # ----------------------------------------------------------------------------- - name: "Retrieve AppConnect dashboard Route:" @@ -198,7 +190,6 @@ - appconnect_url is defined - appconnect_url != "" - # 12. AppConnect deployment details # ----------------------------------------------------------------------------- - name: "AppConnect Deployment details" diff --git a/ibm/mas_devops/roles/arcgis/tasks/main.yml b/ibm/mas_devops/roles/arcgis/tasks/main.yml index f5c5e78d31..e13d54aa63 100644 --- a/ibm/mas_devops/roles/arcgis/tasks/main.yml +++ b/ibm/mas_devops/roles/arcgis/tasks/main.yml @@ -62,7 +62,6 @@ catalog_source: "{{ mas_catalog_source }}" register: subscription - # 6. Wait until the IBM Maximo Location Services CRD is available # ----------------------------------------------------------------------------- - name: "Wait until the IBM Maximo Location Services for Esri CRD is available" diff --git a/ibm/mas_devops/roles/aws_bucket_access_point/tasks/main.yml b/ibm/mas_devops/roles/aws_bucket_access_point/tasks/main.yml index 57f6da8303..bd6e22c692 100644 --- a/ibm/mas_devops/roles/aws_bucket_access_point/tasks/main.yml +++ b/ibm/mas_devops/roles/aws_bucket_access_point/tasks/main.yml @@ -79,9 +79,9 @@ # ----------------------------------------------------------------------------- - name: "Generate access point policy for read-only user {{ aws_access_point_name }}" ansible.builtin.template: - src: '{{ role_path }}/files/ap-policy.json.j2' - dest: '{{ role_path }}/files/{{ aws_access_point_name }}-policy.json' - mode: '644' + src: "{{ role_path }}/files/ap-policy.json.j2" + dest: "{{ role_path }}/files/{{ aws_access_point_name }}-policy.json" + mode: "644" - name: "Apply access point policy for read-only access to the bucket" shell: | diff --git a/ibm/mas_devops/roles/aws_documentdb_user/tasks/main.yml b/ibm/mas_devops/roles/aws_documentdb_user/tasks/main.yml index e433676527..00a312f6b9 100644 --- a/ibm/mas_devops/roles/aws_documentdb_user/tasks/main.yml +++ b/ibm/mas_devops/roles/aws_documentdb_user/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Assert if (docdb_host and docdb_port) or docdb_hosts is not empty assert: that: @@ -7,7 +6,6 @@ fail_msg: "Either (docdb_host and docdb_port) or docdb_hosts is required." success_msg: "(docdb_host & docdb_port) or docdb_hosts is present." - - name: Set docdb_hosts when not defined or empty set_fact: docdb_hosts: "{{ docdb_host }}:{{ docdb_port }}" @@ -34,7 +32,6 @@ that: ( mongosh_version.rc == 0 ) fail_msg: "mongosh must be installed (https://www.mongodb.com/docs/mongodb-shell/install/)" - # 2. create docdb user for MAS instance # ----------------------------------------------------------------------------- - name: set fact for document DB host, port and user name diff --git a/ibm/mas_devops/roles/aws_policy/files/policy-template-sample.json b/ibm/mas_devops/roles/aws_policy/files/policy-template-sample.json index f0d3fc27be..f0fb368cf1 100644 --- a/ibm/mas_devops/roles/aws_policy/files/policy-template-sample.json +++ b/ibm/mas_devops/roles/aws_policy/files/policy-template-sample.json @@ -1,13 +1,13 @@ { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:GetObject", "s3:ListBucket"], - "Resource": [ - "arn:aws:s3:us-east-2:435377327995:accesspoint/access-point-c1/object/*", - "arn:aws:s3:us-east-2:435377327995:accesspoint/access-point-c1" - ] - } - ] + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:ListBucket"], + "Resource": [ + "arn:aws:s3:us-east-2:435377327995:accesspoint/access-point-c1/object/*", + "arn:aws:s3:us-east-2:435377327995:accesspoint/access-point-c1" + ] + } + ] } diff --git a/ibm/mas_devops/roles/aws_user_creation/tasks/main.yml b/ibm/mas_devops/roles/aws_user_creation/tasks/main.yml index 66f1816134..ae874669da 100644 --- a/ibm/mas_devops/roles/aws_user_creation/tasks/main.yml +++ b/ibm/mas_devops/roles/aws_user_creation/tasks/main.yml @@ -37,7 +37,6 @@ - name: "Check if IAM user {{ aws_username }} needs an AccessKeyId and SecretAccessKey" when: aws_username_create_access_key_flag == True block: - - name: "Create an IAM user {{ aws_username }} AccessKeyId and SecretAccessKey" shell: | aws iam create-access-key --user-name {{ aws_username }} diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/install.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/install.yml index 5188054b1a..dc5a69bbff 100644 --- a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/install.yml +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/install.yml @@ -25,7 +25,6 @@ - debug: msg: "IBM Certificate Manager installed .................... {{ cpfs_cm_installed | default(false, true) | bool }}" - # 2. Disable IBM Cert Manager OperandRequest # ----------------------------------------------------------------------------- # Check if Operand Deployment Lifecycle Manager is installed @@ -53,7 +52,6 @@ wait: yes wait_timeout: 120 # 2 minutes - # 3. If IBM Certificate Manager is installed, run pre-requisite steps to migrate to Red Hat Certificate Manager # ----------------------------------------------------------------------------- # - Set 'cert_manager_cluster_resource_namespace: ibm-common-services' in CertManager CR @@ -66,7 +64,6 @@ - cpfs_cm_installed is defined - cpfs_cm_installed is true - # 4. Create RHCM Subscription # ----------------------------------------------------------------------------- - name: "install : Create Red Hat Certificate Manager Subscription" @@ -76,7 +73,6 @@ package_channel: stable-v1 register: subscription - # 5. Wait for Subscription to be processed # ----------------------------------------------------------------------------- - name: "install: Wait for Red Hat cert-manager-operator-controller-manager to be ready (60s delay)" @@ -96,7 +92,6 @@ retries: 30 # Approximately 1/2 hour before we give up delay: 60 # 1 minute - # 6. Create the CertManager instance # ----------------------------------------------------------------------------- - name: "install: Wait for CertManager Cluster Custom Resource to be created" @@ -169,7 +164,6 @@ retries: 60 # Approximately 1/2 hour before we give up delay: 60 # 1 minute - # 5. Update IBM CIS Webhook Cluster Role Binding to point to Red Hat Certificate Manager # ----------------------------------------------------------------------------- # Assuming IBM Certificate Manager is being migrated to Red Hat Certificate Manager, diff --git a/ibm/mas_devops/roles/cis/tasks/provider/ibm/provision.yml b/ibm/mas_devops/roles/cis/tasks/provider/ibm/provision.yml index 4bc48ff0a7..838ef22b42 100644 --- a/ibm/mas_devops/roles/cis/tasks/provider/ibm/provision.yml +++ b/ibm/mas_devops/roles/cis/tasks/provider/ibm/provision.yml @@ -164,4 +164,4 @@ ansible.builtin.template: src: cis_output.yml.j2 dest: "{{ mas_config_dir }}/cis-cm.yml" - mode: '664' + mode: "664" diff --git a/ibm/mas_devops/roles/common_services/tasks/actions/upgrade.yml b/ibm/mas_devops/roles/common_services/tasks/actions/upgrade.yml index 513dd06bca..0047dafa42 100644 --- a/ibm/mas_devops/roles/common_services/tasks/actions/upgrade.yml +++ b/ibm/mas_devops/roles/common_services/tasks/actions/upgrade.yml @@ -19,7 +19,6 @@ set_fact: common_services_channel: "{{ common_services_manifest_info.resources[0].status.defaultChannel }}" - # 2. Lookup common services channel # ----------------------------------------------------------------------------- - name: "Get the current subscription channel for Common Services" @@ -35,7 +34,6 @@ set_fact: old_common_services_channel: "{{ common_services_subscription_info.resources[0].spec.channel }}" - # 3. If the current operator channel is the same for the upgrade channel or not defined, no upgrade # ----------------------------------------------------------------------------- - name: "Display if subscription channel is up to date" @@ -51,7 +49,6 @@ debug: msg: "Common Services subscription was not found in the cluster. There's nothing to upgrade" - # 4. Upgrade Subscription otherwise # ----------------------------------------------------------------------------- - name: "Update Common Services subscription" @@ -75,7 +72,6 @@ pause: minutes: 2 - # 5. Lookup the updated OperatorCondition # ----------------------------------------------------------------------------- - name: "Check OperatorCondition after upgrade" diff --git a/ibm/mas_devops/roles/configure_manage_eventstreams/defaults/main.yml b/ibm/mas_devops/roles/configure_manage_eventstreams/defaults/main.yml index 5d079fa4f6..a173d8d9c7 100644 --- a/ibm/mas_devops/roles/configure_manage_eventstreams/defaults/main.yml +++ b/ibm/mas_devops/roles/configure_manage_eventstreams/defaults/main.yml @@ -23,4 +23,4 @@ mas_app_ws_apiversion: apps.mas.ibm.com/v1 mas_app_ws_kind: ManageWorkspace mas_app_cfg_timeout: "{{ lookup('env', 'MAS_APP_CFG_TIMEOUT') | default(480, true)}}" # 8 minutes before we give up and fall back into the retry loop -mas_app_cfg_retries: "{{ lookup('env', 'MAS_APP_CFG_RETRIES') | default(50, true)}}" # 8 mins each loop * 50 loops =~ 400 minutes / 6 2/3 hours (Manage is really slow to set up) +mas_app_cfg_retries: "{{ lookup('env', 'MAS_APP_CFG_RETRIES') | default(50, true)}}" # 8 mins each loop * 50 loops =~ 400 minutes / 6 2/3 hours (Manage is really slow to set up) diff --git a/ibm/mas_devops/roles/configure_manage_eventstreams/tasks/main.yml b/ibm/mas_devops/roles/configure_manage_eventstreams/tasks/main.yml index b840c3a938..baa20c81ae 100644 --- a/ibm/mas_devops/roles/configure_manage_eventstreams/tasks/main.yml +++ b/ibm/mas_devops/roles/configure_manage_eventstreams/tasks/main.yml @@ -197,7 +197,7 @@ ansible.builtin.template: src: configmsgprovider.j2 dest: /tmp/configmsgprovider.sh - mode: '777' + mode: "777" - name: Copy the configmsgprovider script into the db2 pod shell: "oc cp /tmp/configmsgprovider.sh {{ cpd_meta_namespace }}/{{ db2_pod_name }}:/tmp/configmsgprovider.sh" @@ -222,7 +222,7 @@ ansible.builtin.template: src: configcrontask.j2 dest: /tmp/configcrontask.sh - mode: '777' + mode: "777" - name: Copy the configcrontask script into the db2 pod shell: "oc cp /tmp/configcrontask.sh {{ cpd_meta_namespace }}/{{ db2_pod_name }}:/tmp/configcrontask.sh" diff --git a/ibm/mas_devops/roles/cos/tasks/providers/ibm/provision.yml b/ibm/mas_devops/roles/cos/tasks/providers/ibm/provision.yml index b490948dca..722fc36331 100644 --- a/ibm/mas_devops/roles/cos/tasks/providers/ibm/provision.yml +++ b/ibm/mas_devops/roles/cos/tasks/providers/ibm/provision.yml @@ -51,7 +51,6 @@ set_fact: resourceGID: "{{ rg_info.resource.id }}" - # 3. Create a new COS instance # --------------------------------------------------------------------------------------------------------------------- - name: "ibm : Create a cos instance" @@ -88,7 +87,6 @@ set_fact: cos_resource_id: "{{ cos_info.resource.id }}" - # 4. Create authentication credentials for the instance # --------------------------------------------------------------------------------------------------------------------- - name: "ibm : Create a cos service credential for this MAS instance" @@ -139,4 +137,4 @@ ansible.builtin.template: src: ibm/objectstoragecfg.yml.j2 dest: "{{ mas_config_dir }}/cos-ibm-system.yml" - mode: '664' + mode: "664" diff --git a/ibm/mas_devops/roles/cos_bucket/defaults/main.yml b/ibm/mas_devops/roles/cos_bucket/defaults/main.yml index 89e4cb8b9c..eaf93d4800 100644 --- a/ibm/mas_devops/roles/cos_bucket/defaults/main.yml +++ b/ibm/mas_devops/roles/cos_bucket/defaults/main.yml @@ -26,8 +26,19 @@ cos_bucket_name: "{{ lookup('env', 'COS_BUCKET_NAME') | default(cos_bucket_name_ ibmcloud_region: "{{ lookup('env', 'IBMCLOUD_REGION') | default('us-east', True) }}" # var used by SRE team bucket_cross_reg_loc: "{% if ibmcloud_region.split('-')[0]=='jp' %}ap{% elif ibmcloud_region.split('-')[0]=='eu'%}eu{% else %}us{% endif %}" # var used by SRE team -supported_cross_region_location: ['us', 'ap', 'eu'] -supported_region_location: ['au-syd', 'eu-de', 'eu-gb', 'jp-tok', 'us-east', 'us-south', 'ca-tor', 'jp-osa', 'br-sao'] +supported_cross_region_location: ["us", "ap", "eu"] +supported_region_location: + [ + "au-syd", + "eu-de", + "eu-gb", + "jp-tok", + "us-east", + "us-south", + "ca-tor", + "jp-osa", + "br-sao", + ] cos_bucket_region_location_type: "{{ lookup('env', 'COS_BUCKET_REGION_LOCATION_TYPE') | default('cross_region_location', true) }}" # supported: region_location, cross_region_location and single_site_location cos_bucket_region_location: "{{ lookup('env', 'COS_BUCKET_REGION_LOCATION') | default(bucket_cross_reg_loc, true) }}" cos_url_default: "https://s3.{{ cos_bucket_region_location }}.cloud-object-storage.appdomain.cloud" diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/aws/delete.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/aws/delete.yml index 0ceef59902..f4ec1f7409 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/aws/delete.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/aws/delete.yml @@ -22,15 +22,13 @@ # --------------------------------------------------------------------------------------------------------------------- - name: "Delete AWS buckets objects first" when: aws_bucket_force_deletion_flag == True - shell: - aws s3 rm s3://{{ aws_bucket_name }} --recursive + shell: aws s3 rm s3://{{ aws_bucket_name }} --recursive register: aws_bucket_objects_deletion_output failed_when: aws_bucket_objects_deletion_output.rc > 0 # 3. Delete AWS bucket # --------------------------------------------------------------------------------------------------------------------- - name: "Delete AWS buckets" - shell: - aws s3api delete-bucket --bucket {{ aws_bucket_name }} --region {{ aws_bucket_region_location }} + shell: aws s3api delete-bucket --bucket {{ aws_bucket_name }} --region {{ aws_bucket_region_location }} register: aws_bucket_deletion_output failed_when: aws_bucket_deletion_output.rc > 0 diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_cross_region_location_bucket.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_cross_region_location_bucket.yml index 33609f922b..fff611ad13 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_cross_region_location_bucket.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_cross_region_location_bucket.yml @@ -15,29 +15,28 @@ - name: Validate if cos bucket needs to be created when: - - cos_bucket_lookup.resource is defined - - cos_bucket_lookup.resource.bucket_name is not defined # If it fails to find existing bucket, we try to create it anyway. + - cos_bucket_lookup.resource is defined + - cos_bucket_lookup.resource.bucket_name is not defined # If it fails to find existing bucket, we try to create it anyway. block: + - name: "Create cos bucket if it does not exist" + ibm.cloudcollection.ibm_cos_bucket: + resource_instance_id: "{{ cos_resource_id }}" + ibmcloud_api_key: "{{ cos_apikey }}" + bucket_name: "{{ cos_bucket_name }}" + storage_class: "{{ cos_bucket_storage_class }}" + cross_region_location: "{{ cos_bucket_region_location }}" + force_delete: True + object_versioning: + - enable: "{{ cos_bucket_versioning_flag }}" + expire_rule: + - rule_id: "{{ cos_bucket_expire_rule_id }}" + enable: "{{ cos_bucket_expire_rule_enable_flag }}" + days: "{{ cos_bucket_expire_rule_days }}" + register: cos_bucket_create_output - - name: "Create cos bucket if it does not exist" - ibm.cloudcollection.ibm_cos_bucket: - resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ cos_apikey }}" - bucket_name: "{{ cos_bucket_name }}" - storage_class: "{{ cos_bucket_storage_class }}" - cross_region_location: "{{ cos_bucket_region_location }}" - force_delete: True - object_versioning: - - enable: '{{ cos_bucket_versioning_flag }}' - expire_rule: - - rule_id: "{{ cos_bucket_expire_rule_id }}" - enable: '{{ cos_bucket_expire_rule_enable_flag }}' - days: '{{ cos_bucket_expire_rule_days }}' - register: cos_bucket_create_output - - - name: "Assert cos bucket '{{ cos_bucket_name }}' was created" - assert: - that: - - cos_bucket_create_output.rc == 0 - - 'cos_bucket_create_output.stdout_lines | select("equalto", "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.") | list | length' + - name: "Assert cos bucket '{{ cos_bucket_name }}' was created" + assert: + that: + - cos_bucket_create_output.rc == 0 + - 'cos_bucket_create_output.stdout_lines | select("equalto", "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.") | list | length' diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_region_location_bucket.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_region_location_bucket.yml index 79bcd211be..371a07d5b8 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_region_location_bucket.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_region_location_bucket.yml @@ -15,29 +15,28 @@ - name: Validate if cos bucket needs to be created when: - - cos_bucket_lookup.resource is defined - - cos_bucket_lookup.resource.bucket_name is not defined # If it fails to find existing bucket, we try to create it anyway. + - cos_bucket_lookup.resource is defined + - cos_bucket_lookup.resource.bucket_name is not defined # If it fails to find existing bucket, we try to create it anyway. block: + - name: "Create cos bucket if it does not exist" + ibm.cloudcollection.ibm_cos_bucket: + resource_instance_id: "{{ cos_resource_id }}" + ibmcloud_api_key: "{{ cos_apikey }}" + bucket_name: "{{ cos_bucket_name }}" + storage_class: "{{ cos_bucket_storage_class }}" + region_location: "{{ cos_bucket_region_location }}" + force_delete: True + object_versioning: + - enable: "{{ cos_bucket_versioning_flag }}" + expire_rule: + - rule_id: "{{ cos_bucket_expire_rule_id }}" + enable: "{{ cos_bucket_expire_rule_enable_flag }}" + days: "{{ cos_bucket_expire_rule_days }}" + register: cos_bucket_create_output - - name: "Create cos bucket if it does not exist" - ibm.cloudcollection.ibm_cos_bucket: - resource_instance_id: "{{ cos_resource_id }}" - ibmcloud_api_key: "{{ cos_apikey }}" - bucket_name: "{{ cos_bucket_name }}" - storage_class: "{{ cos_bucket_storage_class }}" - region_location: "{{ cos_bucket_region_location }}" - force_delete: True - object_versioning: - - enable: '{{ cos_bucket_versioning_flag }}' - expire_rule: - - rule_id: "{{ cos_bucket_expire_rule_id }}" - enable: '{{ cos_bucket_expire_rule_enable_flag }}' - days: '{{ cos_bucket_expire_rule_days }}' - register: cos_bucket_create_output - - - name: "Assert cos bucket '{{ cos_bucket_name }}' was created" - assert: - that: - - cos_bucket_create_output.rc == 0 - - 'cos_bucket_create_output.stdout_lines | select("equalto", "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.") | list | length' + - name: "Assert cos bucket '{{ cos_bucket_name }}' was created" + assert: + that: + - cos_bucket_create_output.rc == 0 + - 'cos_bucket_create_output.stdout_lines | select("equalto", "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.") | list | length' diff --git a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_service_credentials.yml b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_service_credentials.yml index 00eea0709f..1789701642 100644 --- a/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_service_credentials.yml +++ b/ibm/mas_devops/roles/cos_bucket/tasks/providers/ibm/create_service_credentials.yml @@ -7,7 +7,7 @@ role: "{{ cos_resource_key_iam_role }}" resource_instance_id: "{{ cos_resource_id }}" ibmcloud_api_key: "{{ cos_apikey }}" - parameters: {'HMAC': true } + parameters: { "HMAC": true } - name: "Retrieve IBM COS service credentials {{ cos_service_credentials_name }}" ibm.cloudcollection.ibm_resource_key_info: diff --git a/ibm/mas_devops/roles/cp4d/tasks/authorize-user-cpd-admin.yml b/ibm/mas_devops/roles/cp4d/tasks/authorize-user-cpd-admin.yml index 1ee149e76b..636fef779b 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/authorize-user-cpd-admin.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/authorize-user-cpd-admin.yml @@ -44,7 +44,6 @@ retries: 6 # Just over 1 minute delay: 10 # Every 10 seconds - # Debug - name: "authorize-user-cpd-admin : Debug command output : Assign {{ cpd_admin_login_sa }} service account the admin role in the {{ cpd_operators_namespace }} operators project for the cpd instance" debug: diff --git a/ibm/mas_devops/roles/cp4d/tasks/create-subscriptions.yml b/ibm/mas_devops/roles/cp4d/tasks/create-subscriptions.yml index 64aee3c7be..41f3f2c437 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/create-subscriptions.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/create-subscriptions.yml @@ -25,7 +25,7 @@ - name: "create-subscriptions : Create Cloud Pak for Data platform subscription using channel: {{ cpd_channel }}" kubernetes.core.k8s: apply: yes - template: 'templates/cpd_platform/subscription.yml.j2' + template: "templates/cpd_platform/subscription.yml.j2" - name: "create-subscriptions : Wait for {{ cpd_platform_operator_name }}-manager to be ready (60s delay)" kubernetes.core.k8s_info: diff --git a/ibm/mas_devops/roles/cp4d/tasks/determine-ibmcatalog-tag.yml b/ibm/mas_devops/roles/cp4d/tasks/determine-ibmcatalog-tag.yml index c1cfec8bb2..e660d74b5b 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/determine-ibmcatalog-tag.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/determine-ibmcatalog-tag.yml @@ -11,7 +11,6 @@ # catalog_tag is v9-240625-amd64 # dependency version will be determined from the catalog metadata (get_catalog_info.py) - # 1. Get the IBM Catalog if available # ----------------------------------------------------------------------------- - name: "Lookup ibm-operator-catalog" @@ -34,7 +33,6 @@ set_fact: catalog_tag: "{{ catalog_lookup.resources[0].spec.displayName.split('(')[1].split(')')[0].split(' ')[0] }}" - # 2. Load catalog metadata # ----------------------------------------------------------------------------- - name: "Load catalog metadata to determine default CP4D version" diff --git a/ibm/mas_devops/roles/cp4d/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/cp4d/tasks/determine-storage-classes.yml index deac82d3ef..a558d81825 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/determine-storage-classes.yml @@ -4,7 +4,6 @@ - name: "determine-storage-classes : Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Set default storage classes # ----------------------------------------------------------------------------- - name: "determine-storage-classes : Set default File Storage if not set by user" diff --git a/ibm/mas_devops/roles/cp4d/tasks/entitlement.yml b/ibm/mas_devops/roles/cp4d/tasks/entitlement.yml index dbe7ee14dc..be9c6e6a2d 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/entitlement.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/entitlement.yml @@ -21,8 +21,8 @@ entitledAuth: "{{ entitledAuthStr | b64encode }}" content: - '{"auths":{"cp.icr.io/cp": {"username":"{{ cpd_entitlement_username }}","password":"{{ cpd_entitlement_key }}","auth":"{{ entitledAuth }}"}' - - '}' - - '}' + - "}" + - "}" kubernetes.core.k8s: definition: apiVersion: v1 @@ -45,8 +45,8 @@ entitledAuth: "{{ entitledAuthStr | b64encode }}" content: - '{"auths":{"cp.icr.io/cp": {"username":"{{ cpd_entitlement_username }}","password":"{{ cpd_entitlement_key }}","auth":"{{ entitledAuth }}"}' - - '}' - - '}' + - "}" + - "}" kubernetes.core.k8s: definition: apiVersion: v1 diff --git a/ibm/mas_devops/roles/cp4d/tasks/prereq-check/check-cpfs-version.yml b/ibm/mas_devops/roles/cp4d/tasks/prereq-check/check-cpfs-version.yml index a37f658824..c0e31001ab 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/prereq-check/check-cpfs-version.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/prereq-check/check-cpfs-version.yml @@ -10,7 +10,6 @@ - name: Set ibm-common-services-operator channel if needed when: common_services_channel is not defined or common_services_channel == "" block: - - name: Lookup ibm-common-service-operator packagemanifest kubernetes.core.k8s_info: api_version: packages.operators.coreos.com/v1 diff --git a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-cpfs.yml b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-cpfs.yml index 1016973e4e..fe6c9e72c9 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-cpfs.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-cpfs.yml @@ -65,7 +65,7 @@ - name: "Install common-services-maps configmap" kubernetes.core.k8s: apply: yes - template: 'templates/cpfs/common-service-maps-cm.yml.j2' + template: "templates/cpfs/common-service-maps-cm.yml.j2" # Clean up old ibm-common-service and cpd-platform operators # ----------------------------------------------------------------------------- @@ -83,7 +83,7 @@ - name: "Install IBM Foundational Services Subscription" kubernetes.core.k8s: apply: yes - template: 'templates/cpfs/subscription.yml.j2' + template: "templates/cpfs/subscription.yml.j2" # 2. Patch ZenService lite-cr to set the zen version and increase resource limits # ---------------------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-namespace-scope.yml b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-namespace-scope.yml index 6cae4833ce..286f359890 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-namespace-scope.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/prereqs/install-ibm-namespace-scope.yml @@ -28,7 +28,7 @@ - name: "Install ibm-namespace-scope-operator subscription" kubernetes.core.k8s: apply: yes - template: 'templates/namespacescope/subscription.yml.j2' + template: "templates/namespacescope/subscription.yml.j2" - name: "Wait ibm-namespace-scope-operator subscription installedCSV version at {{ namespace_scope_channel }}" kubernetes.core.k8s_info: @@ -60,7 +60,7 @@ - name: "Install ibm-namespace-scope custom resource" kubernetes.core.k8s: apply: yes - template: 'templates/namespacescope/cr.yml.j2' + template: "templates/namespacescope/cr.yml.j2" - name: "Wait for ibm-namespace-scope instance to be ready" kubernetes.core.k8s_info: diff --git a/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml b/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml index 5df2b778da..36b5165f3f 100644 --- a/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml +++ b/ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml @@ -3,6 +3,7 @@ # ----------------------------------------------------------------------------- - name: "wait-zenmetastore-edb : Wait for Zen Metastore EDB Cluster to be created" kubernetes.core.k8s_info: + api_version: postgresql.k8s.enterprisedb.io/v1 kind: Cluster namespace: "{{ cpd_instance_namespace }}" name: "zen-metastore-edb" @@ -17,12 +18,12 @@ block: - name: "wait-zenmetastore-edb : Fetch the license expiry date" kubernetes.core.k8s_info: + api_version: postgresql.k8s.enterprisedb.io/v1 kind: Cluster namespace: "{{ cpd_instance_namespace }}" name: "zen-metastore-edb" register: zenmetastoreCluster - until: - zenmetastoreCluster.resources[0].spec.instances is defined + until: zenmetastoreCluster.resources[0].spec.instances is defined and zenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus is defined retries: 40 # Give 20 minutes delay: 30 @@ -80,12 +81,12 @@ - name: "wait-zenmetastore-edb : Check and display the license expiry date" kubernetes.core.k8s_info: + api_version: postgresql.k8s.enterprisedb.io/v1 kind: Cluster namespace: "{{ cpd_instance_namespace }}" name: "zen-metastore-edb" register: newzenmetastoreCluster - until: - newzenmetastoreCluster.resources[0].spec.instances is defined + until: newzenmetastoreCluster.resources[0].spec.instances is defined and newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus is defined and "Valid license" in newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus retries: 40 # Give 20 minutes @@ -100,6 +101,7 @@ # ----------------------------------------------------------------------------- - name: "wait-zenmetastore-edb : Wait for ZenMetastore pods to be become ready" kubernetes.core.k8s_info: + api_version: postgresql.k8s.enterprisedb.io/v1 kind: Cluster namespace: "{{ cpd_instance_namespace }}" name: "zen-metastore-edb" diff --git a/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.0.0.yml b/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.0.0.yml index 100b5f77c3..0b0c3cb014 100644 --- a/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.0.0.yml +++ b/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.0.0.yml @@ -18,5 +18,5 @@ catsrc_info: catalog_display_name: ibm-zen-6.0.1 name: cpopen/ibm-zen-operator-catalog registry: icr.io -# tag: 4.8.0 + # tag: 4.8.0 digest: sha256:6b43f09c1335a28e71669a0ecbd6fdee0ec2afe74d62c014b0d1d2d5a04b4ee3 diff --git a/ibm/mas_devops/roles/cp4d_admin_pwd_update/tasks/get-cp4d-bearer-token.yml b/ibm/mas_devops/roles/cp4d_admin_pwd_update/tasks/get-cp4d-bearer-token.yml index 1d0e44bc07..bd4659186d 100644 --- a/ibm/mas_devops/roles/cp4d_admin_pwd_update/tasks/get-cp4d-bearer-token.yml +++ b/ibm/mas_devops/roles/cp4d_admin_pwd_update/tasks/get-cp4d-bearer-token.yml @@ -64,7 +64,7 @@ headers: cache-control: "no-cache" body_format: "json" - body: "{\"username\":\"{{ cp4d_admin_username }}\",\"password\":\"{{ cp4d_admin_password }}\"}" + body: '{"username":"{{ cp4d_admin_username }}","password":"{{ cp4d_admin_password }}"}' status_code: 200 timeout: 30 register: cp4d_authorization diff --git a/ibm/mas_devops/roles/cp4d_admin_pwd_update/tasks/main.yml b/ibm/mas_devops/roles/cp4d_admin_pwd_update/tasks/main.yml index bbf9cba918..73f8be773f 100644 --- a/ibm/mas_devops/roles/cp4d_admin_pwd_update/tasks/main.yml +++ b/ibm/mas_devops/roles/cp4d_admin_pwd_update/tasks/main.yml @@ -10,7 +10,7 @@ # Authenticate to CP4D - get cp4d_host and cp4d_bearer_token - name: Authenticate to CP4D - get cp4d_host and cp4d_bearer_token block: - - include_tasks: 'cp4d-access-control.yml' + - include_tasks: "cp4d-access-control.yml" no_log: true - name: Get initial_admin_password from admin-user-details Secret in CP4D namespace if cp4d_initial_admin_password is not set @@ -71,7 +71,7 @@ headers: cache-control: "no-cache" body_format: "json" - body: "{\"username\":\"{{ cp4d_admin_username }}\",\"password\":\"{{ cp4d_admin_new_password }}\"}" + body: '{"username":"{{ cp4d_admin_username }}","password":"{{ cp4d_admin_new_password }}"}' status_code: 200 timeout: 30 register: cp4d_authorization diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/cp4d_service/tasks/determine-storage-classes.yml index 7d193bdc5d..ec904a0d2f 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/determine-storage-classes.yml @@ -4,7 +4,6 @@ - name: "determine-storage-classes : Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Debug storage class configuration # ----------------------------------------------------------------------------- - name: "determine-storage-classes : Lookup & use default file storage class (if needed)" diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/gencfg/gencfg-wd.yml b/ibm/mas_devops/roles/cp4d_service/tasks/gencfg/gencfg-wd.yml index 51a6f43ee9..526d279b54 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/gencfg/gencfg-wd.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/gencfg/gencfg-wd.yml @@ -10,7 +10,6 @@ - "Discovery Instance name ........ {{ cpd_wd_instance_name }}" - "MAS config directory ........... {{ mas_config_dir }}" - # 2. Get CP4D Beaker token to enable Watson API authentication # ----------------------------------------------------------------------------- - name: "gencfg-wd : Authorize the CP4D (get Bearer token)" @@ -21,7 +20,7 @@ headers: cache-control: "no-cache" body_format: "json" - body: "{\"username\":\"{{ cpd_admin_username }}\",\"password\":\"{{ cpd_admin_password }}\"}" + body: '{"username":"{{ cpd_admin_username }}","password":"{{ cpd_admin_password }}"}' status_code: 200 timeout: 30 register: cpd_authorization @@ -31,7 +30,6 @@ set_fact: cpd_bearer_token: "{{ cpd_authorization.json.token }}" - # 3. Check if there is already a discovery service instance previously created # ----------------------------------------------------------------------------- - name: "gencfg-wd : Get all discovery service instances (query_instance_output)" @@ -66,7 +64,6 @@ msg: - "Discovery Instance Id: ............. {{ (wd_instance_id is defined) | ternary(wd_instance_id,'Not Found') }}" - # 4. Create a Discovery Service instance if none is found # ----------------------------------------------------------------------------- - name: "gencfg-wd : Create a Discovery Service instance if none is found" @@ -97,7 +94,6 @@ - "Discovery Instance Id: ............. {{ wd_instance_id }}" when: wd_instance_id is not defined or wd_instance_id == "" - # 5. Define Watson Discovery API URL # ----------------------------------------------------------------------------- - name: "gencfg-wd : Define Watson Discovery API URL" @@ -105,7 +101,6 @@ set_fact: wd_url: "{{ cpd_admin_url }}/discovery/{{ cpd_instance_namespace }}-wd/instances/{{ wd_instance_id }}/api" - # 6. Write out the config to the local filesystem # ----------------------------------------------------------------------------- - name: "gencfg-wd: Debug Watson Discovery configuration parameters for Assist binding" @@ -124,4 +119,4 @@ ansible.builtin.template: src: wd/wdscfg.yml.j2 dest: "{{ mas_config_dir }}/{{ mas_instance_id }}-wd-config.yml" # WD does not use a WD CR to bind the service - mode: '664' + mode: "664" diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/gencfg/gencfg-wsl.yml b/ibm/mas_devops/roles/cp4d_service/tasks/gencfg/gencfg-wsl.yml index 2273c356d5..150178925d 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/gencfg/gencfg-wsl.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/gencfg/gencfg-wsl.yml @@ -16,7 +16,6 @@ - "CP4D WSL Project Name .......... {{ cpd_wsl_project_name }}" - "MAS config directory ........... {{ mas_config_dir }}" - # 2. Get CP4D Beaker token to enable Watson API authentication # ----------------------------------------------------------------------------- - name: "gencfg-wsl : Authorize the CP4D (get Bearer token)" @@ -27,7 +26,7 @@ headers: cache-control: "no-cache" body_format: "json" - body: "{\"username\":\"{{ cpd_admin_username }}\",\"password\":\"{{ cpd_admin_password }}\"}" + body: '{"username":"{{ cpd_admin_username }}","password":"{{ cpd_admin_password }}"}' status_code: 200 timeout: 30 register: cpd_authorization @@ -37,7 +36,6 @@ set_fact: cpd_bearer_token: "{{ cpd_authorization.json.token }}" - # 3. Look for a project with informed name and retrieve its id # ----------------------------------------------------------------------------- - name: "gencfg-wsl : Look for a project with name {{ cpd_wsl_project_name }} (cpd_wsl_project_name)" @@ -57,7 +55,6 @@ set_fact: wsl_project_id: "{{ lookup_wsl_project_list.json.resources | ibm.mas_devops.getWSLProjectId(cpd_wsl_project_name) }}" - # 4. Communication to User regarding project search # ----------------------------------------------------------------------------- - name: "gencfg-wsl : Debug wsl_project_id" @@ -71,7 +68,6 @@ debug: msg: "Project named {{ cpd_wsl_project_name }} has been found: {{ wsl_project_id }}" - # 5. Create new analytics project if necessary # ----------------------------------------------------------------------------- - name: "gencfg-wsl : Debug data needed to create a new analytics project" @@ -96,7 +92,7 @@ cache-control: "no-cache" Content-Type: "application/json" body_format: "json" - body: "{\"role_name\":\"Administrator\",\"description\":\"Administrator role\",\"permissions\":[\"administrator\",\"can_provision\",\"manage_catalog\",\"create_space\",\"create_project\"]}" + body: '{"role_name":"Administrator","description":"Administrator role","permissions":["administrator","can_provision","manage_catalog","create_space","create_project"]}' status_code: 200 timeout: 30 @@ -119,14 +115,13 @@ - name: "gencfg-wsl : Extract wsl_project_id when it's created via rest api" vars: - regex: '[^/]*$' + regex: "[^/]*$" when: - create_wsl_project_output.status is defined - create_wsl_project_output.status == 201 set_fact: wsl_project_id: "{{ create_wsl_project_output.json.location | regex_search(regex) }}" - # 6. Write out the config to the local filesystem # ----------------------------------------------------------------------------- - name: "Debug final Analytics Project details" @@ -143,11 +138,11 @@ ansible.builtin.template: src: wsl/watsonstudiocfg.yml.j2 dest: "{{ mas_config_dir }}/{{ mas_instance_id }}-watsonstudio-system.yml" - mode: '664' + mode: "664" - name: "gencfg-wsl : Write our wsl_project_id into MAS Configuration Directory" copy: dest: "{{ mas_config_dir }}/{{ cpd_wsl_project_name }}.id" content: | {{ wsl_project_id }} - mode: '664' + mode: "664" diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/prereqs/odf/main.yml b/ibm/mas_devops/roles/cp4d_service/tasks/prereqs/odf/main.yml index 1007e27c88..96e35c75fd 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/prereqs/odf/main.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/prereqs/odf/main.yml @@ -1,5 +1,4 @@ --- - # 1. Disable webhook to allow installing ODF operator - This step is only needed for ROKS classic clusters # ----------------------------------------------------------------------------- - name: Disable webhook to allow installing ODF operator @@ -31,7 +30,7 @@ api_version: packages.operators.coreos.com/v1 kind: PackageManifest name: "{{ storage_operator }}-operator" - namespace: openshift-marketplace # Note: A namespace must be provided when calling packages.operators.coreos.com/v1 + namespace: openshift-marketplace # Note: A namespace must be provided when calling packages.operators.coreos.com/v1 register: storage_operator_manifest - name: Assert that PackageManifest exists diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ca.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ca.yml index d5e305d7ab..4d5ab52937 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ca.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ca.yml @@ -1,5 +1,4 @@ --- - # 1. Wait for CCS # ----------------------------------------------------------------------------- - name: "Wait for CCS" diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml index 9645398c7e..d0b0430b01 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml @@ -117,7 +117,6 @@ when: - cpd_48 - # 6. Wait for CCS CR to be ready # ----------------------------------------------------------------------------- # Note: We can't fail early when we see Failed status, as the operator will diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml index 732b04b342..fa978446bc 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-couchdb.yml @@ -17,7 +17,6 @@ and couchdbStatefulSet.resources[0].status.updatedReplicas is defined and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.updatedReplicas )) - # 2. Wait for couchdb replica pods to become ready # ----------------------------------------------------------------------------- - name: "wait-couchdb: Wait for CouchDB pods to be become ready" @@ -38,8 +37,7 @@ # ----------------------------------------------------------------------------- - set_fact: is_couchdb_ready: true - when: - couchdbStatefulSet.resources[0].status.readyReplicas is defined + when: couchdbStatefulSet.resources[0].status.readyReplicas is defined and couchdbStatefulSet.resources[0].status.replicas == couchdbStatefulSet.resources[0].status.readyReplicas - name: "wait-couchdb: Detecting and restarting pending CouchDB Pods" @@ -102,7 +100,6 @@ msg: - "CouchDB pods are not ready {{ couchdbStatefulSet.resources[0].status }}" - "Pending CouchDB Pods: {{ pending_pod_lookup.resources | map(attribute='metadata.name') }}" - when: - couchdbStatefulSet.resources[0].status.replicas != couchdbStatefulSet.resources[0].status.readyReplicas + when: couchdbStatefulSet.resources[0].status.replicas != couchdbStatefulSet.resources[0].status.readyReplicas when: is_couchdb_ready is not defined diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-elasticsearch.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-elasticsearch.yml index 6e1019f3a5..34fe185cf1 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-elasticsearch.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-elasticsearch.yml @@ -49,7 +49,6 @@ namespace: "{{ cpd_instance_namespace }}" ignore_errors: true # Other roles running at the same time may delete this and then we get a 404 Not Found error - if it dosn't exist that is fine - - name: "wait/ccs : Delete elasticsearch-server-esnodes statefulset so next time it recreates with right imagePullPolicy" kubernetes.core.k8s: state: absent diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-spark.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-spark.yml index a416faf21c..86402b92e9 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-spark.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-spark.yml @@ -1,5 +1,4 @@ --- - # 1. Wait for CP4D service CR to be ready # ----------------------------------------------------------------------------- # We can't stop waiting on Failed status, as it will report failed multiple diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-wml.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-wml.yml index da5fa6f330..5204369172 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-wml.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-wml.yml @@ -1,5 +1,4 @@ --- - # 1. Wait for CCS # ----------------------------------------------------------------------------- - name: "Wait for CCS" diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-wsl.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-wsl.yml index d2b9cf6673..5433f8e72b 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-wsl.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-wsl.yml @@ -1,5 +1,4 @@ --- - # 1. Wait for CCS # ----------------------------------------------------------------------------- - name: "Wait for CCS" @@ -60,8 +59,6 @@ assert: that: cpd_cr_lookup.resources[0].status.wsStatus == "Completed" fail_msg: "Watson Studio install failed (wsStatus)" - - # At this stage the following will be added to the CP4D installation: # # These operators will be installed in ibm-cpd-operators namespace: diff --git a/ibm/mas_devops/roles/cp4d_service/templates/wd/storage/namespace.yaml b/ibm/mas_devops/roles/cp4d_service/templates/wd/storage/namespace.yaml index 2cd6547b22..c378d4acfd 100644 --- a/ibm/mas_devops/roles/cp4d_service/templates/wd/storage/namespace.yaml +++ b/ibm/mas_devops/roles/cp4d_service/templates/wd/storage/namespace.yaml @@ -2,4 +2,4 @@ apiVersion: v1 kind: Namespace metadata: - name: openshift-storage + name: openshift-storage diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml index c2ed35a059..72c6b14e64 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml @@ -8,7 +8,6 @@ - seq: "{{ masbr_job_data_seq }}" phase: "InProgress" - - name: "Backup db2 database" block: # Prepare db2 database backup folder @@ -34,7 +33,6 @@ debug: msg: "Db2 database backup folder ........ {{ db2_backup_folder }}" - # Take a backup of db2 database # https://www.ibm.com/docs/en/db2/11.5?topic=commands-backup-database # ------------------------------------------------------------------------- @@ -55,7 +53,6 @@ debug: msg: "{{ _db2backup_output.stdout_lines }}" - # Extract Db2 keystore master key # https://www.ibm.com/docs/en/db2/11.5?topic=edr-restoring-encrypted-backup-image-different-system-local-keystore # ------------------------------------------------------------------------- @@ -98,7 +95,6 @@ debug: msg: "{{ _copy_keystore_output.stdout_lines }}" - # Create tar.gz archives of database backup files # ------------------------------------------------------------------------- - name: "Create tar.gz archives of database backup files" @@ -115,7 +111,6 @@ debug: msg: "{{ _du_files_output.stdout_lines }}" - # Copy backup files from pod to specified storage location # ------------------------------------------------------------------------- - name: "Copy backup files from pod to specified storage location" @@ -128,7 +123,6 @@ - src_file: "{{ db2_backup_folder }}/{{ masbr_job_name }}.tar.gz" dest_folder: "{{ masbr_job_data_type }}" - # Update database backup status: Completed # ------------------------------------------------------------------------- - name: "Update database backup status: Completed" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index e25acad2e4..a915394c47 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -6,7 +6,6 @@ that: db2_instance_name is defined and db2_instance_name != "" fail_msg: "db2_instance_name is required" - # Set common backup job variables # ----------------------------------------------------------------------------- - name: "Set fact: common backup job variables" @@ -19,7 +18,6 @@ - seq: "1" type: "database" - # Before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" @@ -29,7 +27,6 @@ _job_type: "backup" _component_before_task_path: "{{ role_path }}/tasks/before-backup-restore.yml" - - name: "Run backup tasks" block: # Update backup job status: New @@ -41,7 +38,6 @@ - seq: "1" phase: "New" - # Run backup tasks for each data type # ------------------------------------------------------------------------- - name: "Run backup tasks for each data type" diff --git a/ibm/mas_devops/roles/db2/tasks/before-backup-restore.yml b/ibm/mas_devops/roles/db2/tasks/before-backup-restore.yml index bdb33a79ee..01061b37e0 100644 --- a/ibm/mas_devops/roles/db2/tasks/before-backup-restore.yml +++ b/ibm/mas_devops/roles/db2/tasks/before-backup-restore.yml @@ -37,7 +37,6 @@ that: db2_running is defined and db2_running fail_msg: "Db2 is not running!" - # Get db2 pod name # ----------------------------------------------------------------------------- - name: "Get db2 pod name" @@ -63,7 +62,6 @@ oc exec {{ db2_pod_name }} -c {{ db2_container_name }} -n {{ db2_namespace }} -- su -lc ' exec_in_pod_end: "' {{ db2_jdbc_username }}" - # Set db2 backup/restore variables # ------------------------------------------------------------------------- - name: "Set fact: db2 pod copy file variables" @@ -86,7 +84,6 @@ set_fact: db2_keystore_folder: "/mnt/blumeta0/db2/keystore" - # Output db2 information # ----------------------------------------------------------------------------- - name: "Debug: db2 information" @@ -96,7 +93,6 @@ - "Db2 is running ......................... {{ db2_running }}" - "Db2 pod name ........................... {{ db2_pod_name }}" - # Check if an exiting job is running # ------------------------------------------------------------------------- - name: "Try to find job lock file in pod" @@ -122,7 +118,6 @@ touch {{ masbr_pod_lock_file }} {{ exec_in_pod_end }} - # Check storage usage # ------------------------------------------------------------------------- - name: "Get storage usage of pod temporary folder" diff --git a/ibm/mas_devops/roles/db2/tasks/install/create_ldap_user.yml b/ibm/mas_devops/roles/db2/tasks/install/create_ldap_user.yml index 1f35e721b8..9cafc1a042 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/create_ldap_user.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/create_ldap_user.yml @@ -45,7 +45,6 @@ register: creating_ldap_user_output when: check_ldap_user.rc != 0 - # 4. Test that the newly created user can connect to db2 # ----------------------------------------------------------------------------- - name: "Test that the newly created user can connect to db2" diff --git a/ibm/mas_devops/roles/db2/tasks/install/determine-storage-classes.yml b/ibm/mas_devops/roles/db2/tasks/install/determine-storage-classes.yml index 5f74a94bf0..0aeb1765ba 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/determine-storage-classes.yml @@ -4,7 +4,6 @@ - name: "determine-storage-classes : Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Data Storage (Required) # ----------------------------------------------------------------------------- - name: Default Data Storage for ROKS if not set by user (ReadWriteOnce) @@ -20,7 +19,6 @@ that: db2_data_storage_class is defined and db2_data_storage_class != "" fail_msg: "db2_data_storage_class must be defined" - # 3. Meta Storage (Required) # ----------------------------------------------------------------------------- - name: Default Meta Storage for ROKS if not set by user (ReadWriteMany) @@ -35,7 +33,6 @@ that: db2_meta_storage_class is defined and db2_meta_storage_class != "" fail_msg: "db2_meta_storage_class must be defined" - # 4. Logs Storage (Optional) # ----------------------------------------------------------------------------- - name: Default Logs Storage for ROKS if not set by user (ReadWriteOnce) @@ -45,7 +42,6 @@ set_fact: db2_logs_storage_class: "{{ defaultStorageClasses.rwo }}" - # 5. Backup Storage (Optional) # ----------------------------------------------------------------------------- - name: Default Backup Storage for ROKS if not set by user (ReadWriteMany) @@ -53,7 +49,6 @@ set_fact: db2_backup_storage_class: "{{ defaultStorageClasses.rwx }}" - # 6. Temp Storage (Optional) # ----------------------------------------------------------------------------- - name: Default Temp Storage for ROKS if not set by user (ReadWriteOnce) @@ -63,7 +58,6 @@ set_fact: db2_temp_storage_class: "{{ defaultStorageClasses.rwo }}" - # 7. Debug storage class configuration # ----------------------------------------------------------------------------- - name: "Debug DB2U storage class configuration" diff --git a/ibm/mas_devops/roles/db2/tasks/install/main.yml b/ibm/mas_devops/roles/db2/tasks/install/main.yml index 345d7bf2bd..df0cdeab47 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/main.yml @@ -165,7 +165,6 @@ package_channel: "{{ db2_channel }}" register: subscription - # 7. Get the cluster subdomain to be used for the certificate and route creation # ----------------------------------------------------------------------------- - name: "Get cluster subdomain" diff --git a/ibm/mas_devops/roles/db2/tasks/install/setup_norootsquash.yml b/ibm/mas_devops/roles/db2/tasks/install/setup_norootsquash.yml index bda1565160..1876337a49 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/setup_norootsquash.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/setup_norootsquash.yml @@ -9,13 +9,12 @@ entitledAuthStr: "{{ registry_user }}:{{ ibm_entitlement_key }}" entitledAuth: "{{ entitledAuthStr | b64encode }}" content: - - "{\"auths\":{\"{{ registry }}/cp/cpd\":{\"username\":\"{{ registry_user }}\",\"password\":\"{{ ibm_entitlement_key }}\",\"email\":\"{{ registry_user }}\",\"auth\":\"{{ entitledAuth }}\"}" + - '{"auths":{"{{ registry }}/cp/cpd":{"username":"{{ registry_user }}","password":"{{ ibm_entitlement_key }}","email":"{{ registry_user }}","auth":"{{ entitledAuth }}"}' - "}" - "}" set_fact: new_secret: "{{ content | join('') }}" - # 2. Generate 'ibm-registry' secret # ----------------------------------------------------------------------------- - name: "Generate 'ibm-registry' secret" @@ -31,14 +30,12 @@ .dockerconfigjson: "{{ new_secret | to_json | b64encode }}" register: secretUpdateResult - # 3. Create DaemonSet # ----------------------------------------------------------------------------- - name: Create 'norootsquash' DaemonSet kubernetes.core.k8s: definition: "{{ lookup('template', 'templates/norootsquash_daemonset.yml.j2') }}" - # 4. Wait for 'norootsquash' DaemonSet to be running # ----------------------------------------------------------------------------- - name: "Wait for 'norootsquash' DaemonSet to be running on all nodes" diff --git a/ibm/mas_devops/roles/db2/tasks/install/suite_jdbccfg.yml b/ibm/mas_devops/roles/db2/tasks/install/suite_jdbccfg.yml index 10730c1d40..856d37b34c 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/suite_jdbccfg.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/suite_jdbccfg.yml @@ -5,8 +5,9 @@ - mas_instance_id != "" - mas_config_dir is defined - mas_config_dir != "" - template: src=vars/jdbccfg/{{ mas_config_scope }}.yml.j2 - dest=/tmp/{{ mas_config_scope }}.yml + template: + src: "vars/jdbccfg/{{ mas_config_scope }}.yml.j2" + dest: "/tmp/{{ mas_config_scope }}.yml" - name: Load variables to use in JDBCCfg when: @@ -49,4 +50,4 @@ ansible.builtin.template: src: suite_jdbccfg.yml.j2 dest: "{{ mas_config_dir }}/jdbc-{{ db2_instance_name | lower }}-{{ db2_namespace }}.yml" - mode: '664' + mode: "664" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml b/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml index 20a7f46c72..57fc684401 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml @@ -10,7 +10,6 @@ - src_file: "{{ masbr_job_data_type }}/{{ _job_name }}.tar.gz" dest_folder: "{{ db2_restore_folder }}" - # Extract the tar.gz file # ------------------------------------------------------------------------- - name: "Extract the tar.gz file" @@ -30,7 +29,6 @@ - "Extract output folder .............. {{ db2_restore_folder }}/{{ _job_name }}" - "{{ _extract_output.stdout_lines }}" - # Validate extracted files # ------------------------------------------------------------------------- - name: "Set fact: extracted file names" @@ -39,7 +37,7 @@ - name: "Set fact: Db2 keystore .p12 file" vars: - regex: '.+?(?=.p12)' + regex: ".+?(?=.p12)" when: item is regex(regex) set_fact: db2_keystore_p12_file: "{{ item }}" @@ -54,7 +52,7 @@ - name: "Set fact: Db2 keystore .sth file" vars: - regex: '.+?(?=.sth)' + regex: ".+?(?=.sth)" when: item is regex(regex) set_fact: db2_keystore_sth_file: "{{ item }}" @@ -69,7 +67,7 @@ - name: "Set fact: Db2 keystore .kdb file" vars: - regex: '.+?(?=.kdb)' + regex: ".+?(?=.kdb)" when: item is regex(regex) set_fact: db2_keystore_kdb_file: "{{ item }}" @@ -85,7 +83,7 @@ - name: "Set fact: Db2 backup file timestamp" vars: regex: '\d+\d+\d+\d' - when_regex: '^{{ db2_dbname | upper }}.*' + when_regex: "^{{ db2_dbname | upper }}.*" when: item is regex(when_regex) set_fact: db2_backup_timestamp: "{{ item | regex_search(regex) }}" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/main.yml b/ibm/mas_devops/roles/db2/tasks/restore/main.yml index a6a1b18a25..eb813e356d 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/main.yml @@ -6,7 +6,6 @@ that: db2_instance_name is defined and db2_instance_name != "" fail_msg: "db2_instance_name is required" - # Set common restore variables # ----------------------------------------------------------------------------- - name: "Set fact: common restore variables" @@ -19,7 +18,6 @@ - seq: "1" type: "database" - # Before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" @@ -28,7 +26,6 @@ _job_type: "restore" _component_before_task_path: "{{ role_path }}/tasks/before-backup-restore.yml" - - name: "Run restore tasks" block: # Update restore job status: New @@ -40,7 +37,6 @@ - seq: "1" phase: "New" - # Run restore tasks for each data type # ------------------------------------------------------------------------- - name: "Run restore tasks for each data type" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml index 050690429d..23e1dd324f 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml @@ -8,7 +8,6 @@ - seq: "{{ masbr_job_data_seq }}" phase: "InProgress" - - name: "Restore db2 database" block: # Prepare db2 database restore folder @@ -35,7 +34,6 @@ debug: msg: "Db2 database restore folder ....... {{ db2_restore_folder }}" - # Copy backup file from specified storage location to pod # ------------------------------------------------------------------------- - name: "Copy backup file from specified storage location to pod" @@ -47,7 +45,6 @@ set_fact: masbr_restore_from_timestamp: "{{ db2_backup_timestamp }}" - # This is an incremental backup, we also need to copy the based on full backup file # ------------------------------------------------------------------------- - name: "This is an incremental backup, we also need to copy the based on full backup file" @@ -62,7 +59,6 @@ set_fact: masbr_restore_basedon_timestamp: "{{ db2_backup_timestamp }}" - # Add Db2 keystore master key # https://www.ibm.com/docs/en/db2/11.5?topic=edr-restoring-encrypted-backup-image-different-system-local-keystore # ------------------------------------------------------------------------- @@ -95,7 +91,6 @@ - _add_master_key_output.rc != 0 - ('CTGSK3005W' not in _add_master_key_output.stdout) - # Deactivate Db2 in preparation for restore # https://www.ibm.com/docs/en/db2/11.5?topic=r-restoring-db2-from-online-backup-using-commands # ------------------------------------------------------------------------- @@ -127,7 +122,6 @@ debug: msg: "{{ _pre_restore_output.stdout_lines }}" - - name: "Run Db2 restore commands" block: # Run Db2 full restore command @@ -156,7 +150,6 @@ debug: msg: "{{ _run_full_restore_output.stdout_lines }}" - # Run Db2 incremental restore command # https://www.ibm.com/docs/en/db2/11.5?topic=commands-restore-database # ------------------------------------------------------------------------- @@ -230,7 +223,6 @@ debug: msg: "{{ _run_rollforward_output.stdout_lines }}" - # Active Db2 after successful rollforward # https://www.ibm.com/docs/en/db2/11.5?topic=r-restoring-db2-from-online-backup-using-commands # ------------------------------------------------------------------------- @@ -259,7 +251,6 @@ debug: msg: "{{ _post_restore_output.stdout_lines }}" - # Update database restore status: Completed # ------------------------------------------------------------------------- - name: "Update database restore status: Completed" diff --git a/ibm/mas_devops/roles/dro/tasks/gencfg/main.yml b/ibm/mas_devops/roles/dro/tasks/gencfg/main.yml index 96615e2aa0..a8c3f0ca57 100644 --- a/ibm/mas_devops/roles/dro/tasks/gencfg/main.yml +++ b/ibm/mas_devops/roles/dro/tasks/gencfg/main.yml @@ -12,7 +12,6 @@ - dro_contact.last_name is defined and dro_contact.last_name != "" fail_msg: "One or more required environment variables are not defined [DRO_CONTACT_EMAIL, DRO_CONTACT_FIRSTNAME, DRO_CONTACT_LASTNAME]" - # 1. Lookup the endpoint route # ----------------------------------------------------------------------------- - name: "drocfg : Get ibm-data-reporter route" diff --git a/ibm/mas_devops/roles/dro/tasks/install-dro/determine-storage-classes.yml b/ibm/mas_devops/roles/dro/tasks/install-dro/determine-storage-classes.yml index 2ca02915a6..e8341ff4c5 100644 --- a/ibm/mas_devops/roles/dro/tasks/install-dro/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/dro/tasks/install-dro/determine-storage-classes.yml @@ -4,7 +4,6 @@ - name: "determine-storage-classes : Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Set Storage (Required) # ----------------------------------------------------------------------------- - name: Default Storage for ROKS if not set by user @@ -17,7 +16,6 @@ that: dro_storage_class is defined and dro_storage_class != "" fail_msg: "Unable to auto determine dro_storage_class" - # 3. Debug storage class configuration # ----------------------------------------------------------------------------- - name: "Debug DRO storage class configuration" diff --git a/ibm/mas_devops/roles/dro/tasks/install-dro/main.yml b/ibm/mas_devops/roles/dro/tasks/install-dro/main.yml index 0daddb8e97..df43a7821e 100644 --- a/ibm/mas_devops/roles/dro/tasks/install-dro/main.yml +++ b/ibm/mas_devops/roles/dro/tasks/install-dro/main.yml @@ -1,11 +1,9 @@ --- - # 1. Check for MAS ImageContentSourcePolicy # ----------------------------------------------------------------------------- - name: "Determine whether this is an airgap environment" include_tasks: "{{ role_path }}/../../common_tasks/detect_airgap.yml" - # 2. Display DRO deployment details # ----------------------------------------------------------------------------- - name: "DRO Deployment details" @@ -19,7 +17,6 @@ - "DRO Contact Last Name .............. {{ dro_contact.last_name | default('', True) }}" - "DRO Contact e-mail ................. {{ dro_contact.email | default('', True) }}" - # 3. Check "{{ dro_namespace }}" operator exists # ---------------------------------------------------------------------------- - name: Check if operator group is present in "{{ dro_namespace }}" already @@ -32,7 +29,6 @@ kubernetes.core.k8s: definition: "{{ lookup('template', 'templates/operatorgroup.yml.j2') }}" - # 4. Create a PVC using the Chosen Storage Class # ---------------------------------------------------------------------------- - name: "Debug User Provided DRO Storage Class" @@ -49,7 +45,6 @@ kubernetes.core.k8s: definition: "{{ lookup('template', 'templates/dro-pvc.yml.j2') }}" - # 5. Create Marketplace Pull Secret # ---------------------------------------------------------------------------- - name: Get marketplace secret @@ -115,7 +110,6 @@ fail_msg: "Provide IBM Entitlement Key, Access https://myibm.ibm.com/products-services/containerlibrary using your IBMId to access your entitlement key" when: (rm_sec.resources is defined) and (rm_sec.resources | length == 0) - # 6. Load PodTemplates configuration # ----------------------------------------------------------------------------- - name: "Load podTemplates configuration" @@ -123,7 +117,6 @@ vars: config_files: ["ibm-mas-bascfg.yml"] - # 7. Install ibm-metrics-operator # ----------------------------------------------------------------------------- - name: Get ibm-metrics-operator Subscription @@ -166,7 +159,6 @@ wait_timeout: 60 when: (imo_subscription.resources is defined) and (imo_subscription.resources | length == 0) - # 8. Install ibm-data-reporter-operator # ----------------------------------------------------------------------------- - name: Get ibm-data-operator Subscription @@ -206,7 +198,6 @@ wait_timeout: 60 when: (dro_subscription.resources is defined) and (dro_subscription.resources | length == 0) - # 9. Wait until the marketplaceconfigs CRD is available # ----------------------------------------------------------------------------- - name: "Wait until the marketplaceconfigs CRD is available" @@ -214,14 +205,12 @@ vars: crd_name: marketplaceconfigs.marketplace.redhat.com - # 10. Apply MarketplaceConfig CR # ----------------------------------------------------------------------------- - name: Create MarketplaceConfig CR kubernetes.core.k8s: definition: "{{ lookup('template', 'templates/MarketplaceConfig-cr.yml.j2') }}" - # 11. Create DRO Pull Secret # ----------------------------------------------------------------------------- - name: "Create Secret redhat-marketplace-pull-secret" diff --git a/ibm/mas_devops/roles/eck/defaults/main.yml b/ibm/mas_devops/roles/eck/defaults/main.yml index 4a26b381d9..67c3bb5457 100644 --- a/ibm/mas_devops/roles/eck/defaults/main.yml +++ b/ibm/mas_devops/roles/eck/defaults/main.yml @@ -1,5 +1,4 @@ --- - eck_action: "{{ lookup('env', 'ECK_ACTION') | default('install', true) }}" eck_package: elasticsearch-eck-operator-certified diff --git a/ibm/mas_devops/roles/eck/tasks/install.yml b/ibm/mas_devops/roles/eck/tasks/install.yml index f101daa612..3cae54646e 100644 --- a/ibm/mas_devops/roles/eck/tasks/install.yml +++ b/ibm/mas_devops/roles/eck/tasks/install.yml @@ -22,7 +22,6 @@ eck_source_namespace: "{{ eck_manifest.resources[0].status.catalogSourceNamespace }}" eck_default_channel: "{{ eck_manifest.resources[0].status.defaultChannel }}" - # 2. Subscription (openshift-operators) # ----------------------------------------------------------------------------- - name: "install : Create the subscription" @@ -30,7 +29,6 @@ apply: yes template: "templates/subscription.yml.j2" - # 3. Namespace (ECK) # ----------------------------------------------------------------------------- - name: "install : Create the namespace" @@ -38,7 +36,6 @@ apply: yes template: "templates/elastic/namespace.yml.j2" - # 4. LetsEncrypt Issuer # ----------------------------------------------------------------------------- - name: "install : Create the letsencrypt issuer" @@ -51,7 +48,6 @@ wait: yes wait_timeout: 60 - # 5. Elasticsearch # ----------------------------------------------------------------------------- - name: "install: Wait until the Elasticsearch CRD is available" @@ -82,7 +78,6 @@ apply: yes template: "templates/elastic/elasticsearch.yml.j2" - # 6. Kibana # ----------------------------------------------------------------------------- - name: "install : Create the kibana certificate" @@ -102,7 +97,6 @@ apply: yes template: "templates/elastic/kibana.yml.j2" - # 7. Logstash # ----------------------------------------------------------------------------- - name: "install : Create ServiceAccount" @@ -155,7 +149,6 @@ retries: 30 # Approximately 30 minutes before we give up delay: 60 # 1 minute - # 8. Filebeat # ----------------------------------------------------------------------------- - name: "install : Create ServiceAccount" @@ -175,7 +168,6 @@ apply: yes template: "templates/elastic/filebeat.yml.j2" - # 9. Routes # ----------------------------------------------------------------------------- - name: "install : Create the elasticsearch route" diff --git a/ibm/mas_devops/roles/entitlement_key_rotation/tasks/rotate-ibm-entitlement.yml b/ibm/mas_devops/roles/entitlement_key_rotation/tasks/rotate-ibm-entitlement.yml index 33bba6ef39..f3a343675a 100644 --- a/ibm/mas_devops/roles/entitlement_key_rotation/tasks/rotate-ibm-entitlement.yml +++ b/ibm/mas_devops/roles/entitlement_key_rotation/tasks/rotate-ibm-entitlement.yml @@ -6,8 +6,8 @@ kind: Secret type: kubernetes.io/dockerconfigjson metadata: - name: 'ibm-entitlement' - namespace: '{{ item }}' + name: "ibm-entitlement" + namespace: "{{ item }}" data: .dockerconfigjson: "{{ lookup('template', 'templates/ibm-entitlement-with-artifactory.json.j2') | to_nice_json | b64encode }}" diff --git a/ibm/mas_devops/roles/entitlement_key_rotation/tasks/rotate-wiotp-docker-local.yml b/ibm/mas_devops/roles/entitlement_key_rotation/tasks/rotate-wiotp-docker-local.yml index 705c13a171..7491dff3d6 100644 --- a/ibm/mas_devops/roles/entitlement_key_rotation/tasks/rotate-wiotp-docker-local.yml +++ b/ibm/mas_devops/roles/entitlement_key_rotation/tasks/rotate-wiotp-docker-local.yml @@ -6,8 +6,8 @@ kind: Secret type: kubernetes.io/dockerconfigjson metadata: - name: 'wiotp-docker-local' - namespace: 'openshift-marketplace' + name: "wiotp-docker-local" + namespace: "openshift-marketplace" data: .dockerconfigjson: "{{ lookup('template', 'templates/ibm-entitlement-with-artifactory.json.j2') | to_nice_json | b64encode }}" diff --git a/ibm/mas_devops/roles/gencfg_watsonstudio/tasks/main.yml b/ibm/mas_devops/roles/gencfg_watsonstudio/tasks/main.yml index 276a34f9c1..ed55838736 100644 --- a/ibm/mas_devops/roles/gencfg_watsonstudio/tasks/main.yml +++ b/ibm/mas_devops/roles/gencfg_watsonstudio/tasks/main.yml @@ -1,5 +1,4 @@ --- - # 1. Check for undefined properties that do not have a default # ----------------------------------------------------------------------------- - name: "Fail if mas_instance_id is not provided" @@ -33,7 +32,6 @@ that: cpd_admin_url is defined and cpd_admin_url != "" fail_msg: "cpd_admin_url property is required" - # 2. Provide debug information # ----------------------------------------------------------------------------- - name: "Configure namespace" @@ -69,4 +67,4 @@ ansible.builtin.template: src: wscfg.yml.j2 dest: "{{ mas_config_dir }}/watsonstudiocfg.yml" - mode: '664' + mode: "664" diff --git a/ibm/mas_devops/roles/grafana/tasks/install/determine-storage-classes.yml b/ibm/mas_devops/roles/grafana/tasks/install/determine-storage-classes.yml index d429a9e209..febe7f9793 100644 --- a/ibm/mas_devops/roles/grafana/tasks/install/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/grafana/tasks/install/determine-storage-classes.yml @@ -4,7 +4,6 @@ - name: "determine-storage-classes : Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Grafana Instance Storage (Required) # ----------------------------------------------------------------------------- - name: "install : determine-storage-classes : Default Grafana Instance Storage if not set by user" @@ -17,7 +16,6 @@ that: grafana_instance_storage_class is defined and grafana_instance_storage_class != "" fail_msg: "grafana_instance_storage_class must be defined" - # 3. Debug storage class configuration # ----------------------------------------------------------------------------- - name: "install : determine-storage-classes : Debug grafana storage class configuration" diff --git a/ibm/mas_devops/roles/grafana/tasks/install/main.yml b/ibm/mas_devops/roles/grafana/tasks/install/main.yml index fbdf94f041..76492677a0 100644 --- a/ibm/mas_devops/roles/grafana/tasks/install/main.yml +++ b/ibm/mas_devops/roles/grafana/tasks/install/main.yml @@ -1,11 +1,9 @@ --- - # 1. Load default storage classes (if not provided by the user) # ----------------------------------------------------------------------------- - name: "install : Determine storage classes" include_tasks: tasks/install/determine-storage-classes.yml - # 2. Ensure User Workload Monitoring is Enabled # ----------------------------------------------------------------------------- - name: "install : Get User Workload Monitoring Token" @@ -72,7 +70,6 @@ value: "true" register: subscription - # 5. Configure Grafana Operator so it can scan all namespaces for dashboards # ------------------------------------------------------------------------------------- # See https://github.com/grafana-operator/grafana-operator/blob/master/documentation/multi_namespace_support.md @@ -126,8 +123,7 @@ - name: "install : Fail if we didn't get the Thanos route" fail: msg: "The Thanos route `thanos-querier` or host within route not found in openshift-monitoring" - when: - thanos_route is not defined or + when: thanos_route is not defined or thanos_route.resources[0].spec is not defined or thanos_route.resources[0].spec.host is not defined diff --git a/ibm/mas_devops/roles/grafana/tasks/uninstall/uninstall.yml b/ibm/mas_devops/roles/grafana/tasks/uninstall/uninstall.yml index e74c3d5860..ed19753a09 100644 --- a/ibm/mas_devops/roles/grafana/tasks/uninstall/uninstall.yml +++ b/ibm/mas_devops/roles/grafana/tasks/uninstall/uninstall.yml @@ -1,5 +1,4 @@ --- - # 1. Delete Grafana Datasource # ------------------------------------------------------------------------------------- - name: "uninstall : Delete Grafana v5 Datasource" @@ -22,7 +21,6 @@ ignore_errors: true # in case the api_version doesn't exist when: grafana_version == 4 - # 2. Delete Grafana Instance # ------------------------------------------------------------------------------------- - name: "uninstall : Delete Grafana Instance" @@ -33,7 +31,6 @@ wait_timeout: 300 ignore_errors: true # in case the api_version doesn't exist - # 3. Delete Grafana's ClusterRole and ClusterRoleBinding # ------------------------------------------------------------------------------------- - name: "uninstall : Delete Grafana's ClusterRole and ClusterRoleBinding" @@ -41,7 +38,6 @@ state: absent template: "templates/grafana/v{{grafana_version}}/grafana-rbac.yml.j2" - # 4. Delete Grafana Namespace # ------------------------------------------------------------------------------------- - name: "uninstall : Delete Grafana namespace" diff --git a/ibm/mas_devops/roles/grafana/tasks/uninstall_grafanav4/main.yml b/ibm/mas_devops/roles/grafana/tasks/uninstall_grafanav4/main.yml index 4b76b56448..24b98a5bc7 100644 --- a/ibm/mas_devops/roles/grafana/tasks/uninstall_grafanav4/main.yml +++ b/ibm/mas_devops/roles/grafana/tasks/uninstall_grafanav4/main.yml @@ -1,5 +1,4 @@ --- - # 1. Uninstall Grafana V4 # ------------------------------------------------------------------------------------- - name: "uninstall_grafanav4 : Uninstall Grafana Operator v4" diff --git a/ibm/mas_devops/roles/grafana/tasks/update/determine-storage-config.yml b/ibm/mas_devops/roles/grafana/tasks/update/determine-storage-config.yml index e8178d4502..18f98c083e 100644 --- a/ibm/mas_devops/roles/grafana/tasks/update/determine-storage-config.yml +++ b/ibm/mas_devops/roles/grafana/tasks/update/determine-storage-config.yml @@ -17,7 +17,6 @@ grafana_instance_storage_class: "{{ grafana_cr_result.resources[0].spec.persistentVolumeClaim.spec.storageClassName }}" grafana_instance_storage_size: "{{ grafana_cr_result.resources[0].spec.persistentVolumeClaim.spec.resources.requests.storage }}" - # 2. Set Grafana Instance Storage Class and Size from existing V4 Operator # ----------------------------------------------------------------------------- - name: "update : determine-storage-config : Get V4 Grafana Instance" @@ -37,7 +36,6 @@ grafana_instance_storage_class: "{{ grafana_cr_result.resources[0].spec.dataStorage.class }}" grafana_instance_storage_size: "{{ grafana_cr_result.resources[0].spec.dataStorage.size }}" - # 3. Set Default Grafana Instance Storage Class if not already determined # ----------------------------------------------------------------------------- - name: "update : determine-storage-config : Load default storage class information" @@ -48,7 +46,6 @@ set_fact: grafana_instance_storage_class: "{{ defaultStorageClasses.rwo }}" - # 4. Fail if Grafana Storage Class has not been set # ----------------------------------------------------------------------------- - name: "update : determine-storage-config : Assert that Grafana Instance storage class has been defined" @@ -56,7 +53,6 @@ that: grafana_instance_storage_class is defined and grafana_instance_storage_class != "" fail_msg: "grafana_instance_storage_class cannot be determined" - # 5. Debug storage class configuration # ----------------------------------------------------------------------------- - name: "update : determine-storage-config : Debug grafana storage class configuration" diff --git a/ibm/mas_devops/roles/grafana/tasks/update/main.yml b/ibm/mas_devops/roles/grafana/tasks/update/main.yml index c1b9de99be..878ebc135e 100644 --- a/ibm/mas_devops/roles/grafana/tasks/update/main.yml +++ b/ibm/mas_devops/roles/grafana/tasks/update/main.yml @@ -5,7 +5,6 @@ include_tasks: tasks/update/determine-storage-config.yml when: grafana_instance_storage_class is not defined or grafana_instance_storage_class == "" - # 2. Install and configure Grafana v5 # ----------------------------------------------------------------------------- - name: "update : Install Grafana Operator V5" @@ -50,7 +49,6 @@ ignore_errors: true when: v4Dashboards.resources is defined - # 4. Display the registered grafana dashboards # ----------------------------------------------------------------------------- - name: "update: Fetch list of registered dashboards" @@ -88,7 +86,6 @@ loop_var: v4_dashboard label: "{{ v4_dashboard.metadata.name }}.{{ v4_dashboard.metadata.namespace }}" - # 5. Uninstall Grafana v4 # ----------------------------------------------------------------------------- - name: "update: Uninstall Grafana Operator v4" diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/install/development-catalog.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/install/development-catalog.yml index e145e3b25f..7cc6d7ac64 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/install/development-catalog.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/install/development-catalog.yml @@ -11,8 +11,8 @@ - '{"auths":{"docker-na-public.artifactory.swg-devops.com/wiotp-docker-local": {"username":"{{artifactory_username}}","password":"{{artifactory_token}}","auth":"{{artifactoryAuth}}"}' - ',"docker-na-proxy-svl.artifactory.swg-devops.com/wiotp-docker-local": {"username":"{{artifactory_username}}","password":"{{artifactory_token}}","auth":"{{artifactoryAuth}}"}' - ',"docker-na-proxy-rtp.artifactory.swg-devops.com/wiotp-docker-local": {"username":"{{artifactory_username}}","password":"{{artifactory_token}}","auth":"{{artifactoryAuth}}"}' - - '}' - - '}' + - "}" + - "}" kubernetes.core.k8s: definition: apiVersion: v1 @@ -26,7 +26,6 @@ .dockerconfigjson: "{{ content | join('') | string }}" register: result - # 2. Create the service accounts in openshift-marketplace so that it is able to pull from Artifactory # --------------------------------------------------------------------------------------------------------------------- # In OCP 4.12+ each catalog source uses its own ServiceAccount @@ -40,7 +39,7 @@ metadata: labels: # This label is required in OCP 4.15+ otherwise the catalog source doesn't become READY - olm.managed: 'true' + olm.managed: "true" name: "{{ item }}" namespace: openshift-marketplace imagePullSecrets: @@ -49,7 +48,6 @@ - default - ibm-operator-catalog - # 3. Install the catalog # --------------------------------------------------------------------------------------------------------------------- - name: "Create development catalog" diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/install/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/install/main.yml index 6023105083..ca65375a3a 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/install/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/install/main.yml @@ -1,11 +1,9 @@ --- - # 1. Check for MAS ImageContentSourcePolicy & load CASE bundle info # ----------------------------------------------------------------------------- - name: "Determine whether this is an airgap environment" include_tasks: "{{ role_path }}/../../common_tasks/detect_airgap.yml" - # 2. Install IBM catalog # ----------------------------------------------------------------------------- @@ -34,7 +32,6 @@ kubernetes.core.k8s: template: templates/offline-catalog.yml.j2 - # 3. Install development (pre-release) catalog # ----------------------------------------------------------------------------- # non-airgap catalog diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/uninstall/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/uninstall/main.yml index b89a40faa1..953b2a2b9d 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/uninstall/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/uninstall/main.yml @@ -1,5 +1,4 @@ --- - # 1. Delete the ibm-operator-catalog CatalogSource # ----------------------------------------------------------------------------- - name: "Remove the ibm-operator-catalog CatalogSource" diff --git a/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/main.yml b/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/main.yml index b929d80018..b4ef90fc10 100644 --- a/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/main.yml +++ b/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/main.yml @@ -1,5 +1,4 @@ --- - # 1. Check required properties # ----------------------------------------------------------------------------- - name: "Fail if ibmcloud_apikey is not provided" diff --git a/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/service_key_create.yml b/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/service_key_create.yml index fd96d237fd..60f279d17c 100644 --- a/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/service_key_create.yml +++ b/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/service_key_create.yml @@ -1,5 +1,4 @@ --- - # 1. Check if resource key with the requested name already exists # ----------------------------------------------------------------------------- - name: "Check if resource key with the requested name already exists" diff --git a/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/service_key_delete.yml b/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/service_key_delete.yml index 458fce8f6c..8be3bcfd2a 100644 --- a/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/service_key_delete.yml +++ b/ibm/mas_devops/roles/ibmcloud_resource_key/tasks/service_key_delete.yml @@ -1,5 +1,4 @@ --- - # 1. Check if requested resource key to delete exists # ----------------------------------------------------------------------------- - name: "Check if requested resource key to delete exists" diff --git a/ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml index 773a766863..b320b94daa 100644 --- a/ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml @@ -4,7 +4,6 @@ - name: "determine-storage-classes : Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Set Storage (Required) # ----------------------------------------------------------------------------- - name: Default Kafka Storage for ROKS if not set by user diff --git a/ibm/mas_devops/roles/kafka/tasks/provider/aws/install.yml b/ibm/mas_devops/roles/kafka/tasks/provider/aws/install.yml index 57b9ccefbe..c3cbd77176 100644 --- a/ibm/mas_devops/roles/kafka/tasks/provider/aws/install.yml +++ b/ibm/mas_devops/roles/kafka/tasks/provider/aws/install.yml @@ -4,7 +4,6 @@ - name: "Pre-validation mandatory parameter" include_tasks: tasks/provider/aws/utils/validate.yml - # 2. Generate Kafka username / password if missing # ----------------------------------------------------------------------------- - name: "Generate kafka password" @@ -33,13 +32,11 @@ - "aws_msk_secret ......................... {{ aws_msk_secret}}" - "aws_msk_kms_key ........................ {{ aws_msk_kms_key}}" - # 3. Generate AWS MSK Key # ----------------------------------------------------------------------------- - name: "Pre-validation mandatory parameter" include_tasks: tasks/provider/aws/utils/create-msk-key.yml - # 4. Create three subnets in VPC # ----------------------------------------------------------------------------- - name: "Initialize Facts for provisioning" diff --git a/ibm/mas_devops/roles/kafka/tasks/provider/ibm/install.yml b/ibm/mas_devops/roles/kafka/tasks/provider/ibm/install.yml index 3fc800fde1..340e229d8b 100644 --- a/ibm/mas_devops/roles/kafka/tasks/provider/ibm/install.yml +++ b/ibm/mas_devops/roles/kafka/tasks/provider/ibm/install.yml @@ -102,7 +102,7 @@ partitions: 3 resource_instance_id: "{{ es_exist_info.resource.id }}" config: - # we must enclose properties having . in double quotes then single quote or vice-versa + # we must enclose properties having . in double quotes then single quote or vice-versa '"cleanup.policy"': "delete" '"retention.ms"': "{{ eventstreams_retention }}" '"retention.bytes"': 1073741824 diff --git a/ibm/mas_devops/roles/kafka/tasks/provider/redhat/run-kafka-instances-upgrade.yml b/ibm/mas_devops/roles/kafka/tasks/provider/redhat/run-kafka-instances-upgrade.yml index d45674ce4a..e725423854 100644 --- a/ibm/mas_devops/roles/kafka/tasks/provider/redhat/run-kafka-instances-upgrade.yml +++ b/ibm/mas_devops/roles/kafka/tasks/provider/redhat/run-kafka-instances-upgrade.yml @@ -19,7 +19,6 @@ - "Kafka Cluster Versions ................... {{ kafka_cluster_list.resources | map(attribute='spec.kafka.version') }}" when: kafka_cluster_list.resources is defined - - name: "Get and set variables to store the lists" set_fact: kafkaCluster_names: "{{ kafka_cluster_list.resources | map(attribute='metadata.name') }}" diff --git a/ibm/mas_devops/roles/kafka/tasks/provider/redhat/upgrade.yml b/ibm/mas_devops/roles/kafka/tasks/provider/redhat/upgrade.yml index 67f66f6b96..f53fed3ae4 100644 --- a/ibm/mas_devops/roles/kafka/tasks/provider/redhat/upgrade.yml +++ b/ibm/mas_devops/roles/kafka/tasks/provider/redhat/upgrade.yml @@ -18,7 +18,6 @@ when: - not found_kafka_namespace - # If Kafka namespace exists, then we prepare for the upgrade - include_tasks: tasks/provider/redhat/prepare-kafka-upgrade.yml when: found_kafka_namespace diff --git a/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka-exporter.json b/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka-exporter.json index 33e46fed13..d7b28cb7d3 100644 --- a/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka-exporter.json +++ b/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka-exporter.json @@ -1,1721 +1,1664 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Kafka topics and consumer groups", - "editable": true, - "gnetId": 7589, - "graphTooltip": 0, - "iteration": 1649706719723, - "links": [], - "panels": [ + "annotations": { + "list": [ { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 0 - }, - "id": 27, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(kafka_topic_partitions{topic=~\"$topic\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Topics", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Kafka topics and consumer groups", + "editable": true, + "gnetId": 7589, + "graphTooltip": 0, + "iteration": 1649706719723, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 0 - }, - "id": 28, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_topic_partitions{topic=~\"$topic\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Partitions", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 0 - }, - "id": 29, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_topic_partition_replicas{topic=~\"$topic\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Replicas", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 0 }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 0 - }, - "id": 30, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_topic_partition_in_sync_replica{topic=~\"$topic\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "In Sync Replicas", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgb(255, 255, 255)", - "#F2495C", - "#d44a3a" - ], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 0 - }, - "id": 31, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_topic_partition_under_replicated_partition{topic=~\"$topic\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1,2", - "timeFrom": null, - "timeShift": null, - "title": "Under Replicated Partitions", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "tableColumn": "", + "targets": [ + { + "expr": "count(kafka_topic_partitions{topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Topics", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgb(255, 255, 255)", - "#F2495C", - "#d44a3a" - ], - "datasource": null, - "description": "Number of partitions which are at their minimum in sync replica count (| ISR | == | min.insync.replicas |).", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 0 - }, - "id": 33, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_cluster_partition_atminisr{topic=~\"$topic\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1,2", - "timeFrom": null, - "timeShift": null, - "title": "Partitions at minimum ISR", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgb(255, 255, 255)", - "#F2495C", - "#d44a3a" - ], - "datasource": null, - "description": "Number of partitions which are under their minimum in sync replica count (| ISR | < | min.insync.replicas |).", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 0 - }, - "id": 34, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_cluster_partition_underminisr{topic=~\"$topic\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1,2", - "timeFrom": null, - "timeShift": null, - "title": "Partitions under minimum ISR", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 0 }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "#F2495C", - "#d44a3a" - ], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 0 - }, - "id": 32, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "tableColumn": "", - "targets": [ - { - "expr": "count(kafka_topic_partition_leader_is_preferred{topic=~\"$topic\"}<1)", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1,2", - "timeFrom": null, - "timeShift": null, - "title": "Partitions not on preferred node", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 0, - "y": 4 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": 480, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kafka_topic_partition_current_offset{topic=~\"$topic\"}[1m])) by (topic)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Messages in per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_topic_partitions{topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Partitions", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 6, - "y": 4 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": 480, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 29, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_topic_partition_replicas{topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Replicas", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 30, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(kafka_consumergroup_current_offset{consumergroup=~\"$consumergroup\",topic=~\"$topic\"}[1m])/60) by (consumergroup, topic)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{consumergroup}} (topic: {{topic}})", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Messages consumed per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_topic_partition_in_sync_replica{topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "In Sync Replicas", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["rgb(255, 255, 255)", "#F2495C", "#d44a3a"], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 0 + }, + "id": 31, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_topic_partition_under_replicated_partition{topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Under Replicated Partitions", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["rgb(255, 255, 255)", "#F2495C", "#d44a3a"], + "datasource": null, + "description": "Number of partitions which are at their minimum in sync replica count (| ISR | == | min.insync.replicas |).", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 33, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_cluster_partition_atminisr{topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Partitions at minimum ISR", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["rgb(255, 255, 255)", "#F2495C", "#d44a3a"], + "datasource": null, + "description": "Number of partitions which are under their minimum in sync replica count (| ISR | < | min.insync.replicas |).", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 34, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 12, - "y": 4 + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_cluster_partition_underminisr{topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Partitions under minimum ISR", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#299c46", "#F2495C", "#d44a3a"], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": 480, - "sortDesc": true, - "total": false, - "values": true + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(kafka_topic_partition_leader_is_preferred{topic=~\"$topic\"}<1)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Partitions not on preferred node", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 480, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kafka_topic_partition_current_offset{topic=~\"$topic\"}[1m])) by (topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages in per second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 4 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 480, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(kafka_consumergroup_current_offset{consumergroup=~\"$consumergroup\",topic=~\"$topic\"}[1m])/60) by (consumergroup, topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}} (topic: {{topic}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages consumed per second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kafka_consumergroup_lag{consumergroup=~\"$consumergroup\",topic=~\"$topic\"}) by (consumergroup, topic) ", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{consumergroup}} (topic: {{topic}})", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Lag by Consumer Group", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 4 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 480, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_consumergroup_lag{consumergroup=~\"$consumergroup\",topic=~\"$topic\"}) by (consumergroup, topic) ", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}} (topic: {{topic}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Lag by Consumer Group", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 4 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 480, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_consumergroup_members{consumergroup=~\"$consumergroup\"}) by (consumergroup) ", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Consumer per Consumer Group", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 14 + }, + "hideTimeOverride": true, + "id": 24, + "links": [], + "pageSize": 10, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 18, - "y": 4 + { + "alias": "Offset", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "none" }, - "hiddenSeries": false, - "id": 35, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": 480, - "sortDesc": true, - "total": false, - "values": true + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kafka_consumergroup_current_offset{consumergroup=~\"$consumergroup\",topic=~\"$topic\"}) by (consumergroup,partition,topic)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Consumer Group Offsets", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 14 + }, + "hideTimeOverride": true, + "id": 25, + "links": [], + "pageSize": 10, + "scroll": true, + "showHeader": true, + "sort": { + "col": 4, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true + { + "alias": "Lag", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "none" }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kafka_consumergroup_members{consumergroup=~\"$consumergroup\"}) by (consumergroup) ", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{consumergroup}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Consumer per Consumer Group", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kafka_consumergroup_lag{consumergroup=~\"$consumergroup\",topic=~\"$topic\"}) by (consumergroup,partition,topic)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Consumer Group Lag", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 24 + }, + "hideTimeOverride": true, + "id": 20, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "alias": "Partitions", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "short" }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kafka_topic_partitions{topic=~\"$topic\"}) by (topic)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" } + ], + "timeFrom": null, + "timeShift": null, + "title": "Number of Partitions", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "columns": [], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fontSize": "100%", - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 14 + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 24 + }, + "hideTimeOverride": true, + "id": 22, + "links": [], + "pageSize": 10, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" }, - "hideTimeOverride": true, - "id": 24, - "links": [], - "pageSize": 10, - "scroll": true, - "showHeader": true, - "sort": { - "col": 3, - "desc": true + { + "alias": "Offset", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "none" }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "Offset", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "mappingType": 1, - "pattern": "Value", - "thresholds": [], - "type": "number", - "unit": "none" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(kafka_consumergroup_current_offset{consumergroup=~\"$consumergroup\",topic=~\"$topic\"}) by (consumergroup,partition,topic)", - "format": "table", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Consumer Group Offsets", - "transform": "table", - "type": "table-old" + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kafka_topic_partition_current_offset{topic=~\"$topic\"}) by (partition,topic)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latest Offsets", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "columns": [], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 24 + }, + "hideTimeOverride": true, + "id": 23, + "links": [], + "pageSize": 10, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" }, - "fontSize": "100%", - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 14 + { + "alias": "Offset", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "none" }, - "hideTimeOverride": true, - "id": 25, - "links": [], - "pageSize": 10, - "scroll": true, - "showHeader": true, - "sort": { - "col": 4, - "desc": true + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kafka_topic_partition_oldest_offset{topic=~\"$topic\"}) by (partition,topic)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Oldest Offsets", + "transform": "table", + "type": "table-old" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": ["Kafka", "Kafka Exporter"], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "kafka", + "value": "kafka" }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "Lag", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "mappingType": 1, - "pattern": "Value", - "thresholds": [], - "type": "number", - "unit": "none" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(kafka_consumergroup_lag{consumergroup=~\"$consumergroup\",topic=~\"$topic\"}) by (consumergroup,partition,topic)", - "format": "table", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Consumer Group Lag", - "transform": "table", - "type": "table-old" + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "kubernetes_namespace", + "options": [], + "query": "query_result(kafka_exporter_build_info)", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "columns": [], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fontSize": "100%", - "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 24 + "allValue": null, + "current": { + "selected": false, + "text": "maskafka", + "value": "maskafka" }, - "hideTimeOverride": true, - "id": 20, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "Partitions", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "mappingType": 1, - "pattern": "Value", - "thresholds": [], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(kafka_topic_partitions{topic=~\"$topic\"}) by (topic)", - "format": "table", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Number of Partitions", - "transform": "table", - "type": "table-old" + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Cluster Name", + "multi": false, + "name": "strimzi_cluster_name", + "options": [], + "query": "query_result(kafka_exporter_build_info{namespace=\"$kubernetes_namespace\"})", + "refresh": 1, + "regex": "/.*strimzi_io_cluster=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "columns": [], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fontSize": "100%", - "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 24 + "allValue": null, + "current": { + "selected": false, + "text": ["kafka-cqin", "kafka-errqueue"], + "value": ["kafka-cqin", "kafka-errqueue"] }, - "hideTimeOverride": true, - "id": 22, - "links": [], - "pageSize": 10, - "scroll": true, - "showHeader": true, - "sort": { - "col": 3, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "Offset", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "mappingType": 1, - "pattern": "Value", - "thresholds": [], - "type": "number", - "unit": "none" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(kafka_topic_partition_current_offset{topic=~\"$topic\"}) by (partition,topic)", - "format": "table", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Latest Offsets", - "transform": "table", - "type": "table-old" + "datasource": "prometheus", + "definition": "label_values(kafka_consumergroup_current_offset{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}, consumergroup)", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Consumer Group", + "multi": true, + "name": "consumergroup", + "options": [], + "query": "label_values(kafka_consumergroup_current_offset{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}, consumergroup)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "topic", + "type": "query", + "useTags": false }, { - "columns": [], - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] + "allValue": null, + "current": { + "selected": false, + "text": ["kafka-cqin", "kafka-errqueue"], + "value": ["kafka-cqin", "kafka-errqueue"] }, - "fontSize": "100%", - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 24 - }, - "hideTimeOverride": true, - "id": 23, - "links": [], - "pageSize": 10, - "scroll": true, - "showHeader": true, - "sort": { - "col": 3, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "Offset", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "mappingType": 1, - "pattern": "Value", - "thresholds": [], - "type": "number", - "unit": "none" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(kafka_topic_partition_oldest_offset{topic=~\"$topic\"}) by (partition,topic)", - "format": "table", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Oldest Offsets", - "transform": "table", - "type": "table-old" + "datasource": "prometheus", + "definition": "label_values(kafka_topic_partition_current_offset{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}, topic)", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Topic", + "multi": true, + "name": "topic", + "options": [], + "query": "label_values(kafka_topic_partition_current_offset{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}, topic)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "topic", + "type": "query", + "useTags": false } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "Kafka", - "Kafka Exporter" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "kafka", - "value": "kafka" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "kubernetes_namespace", - "options": [], - "query": "query_result(kafka_exporter_build_info)", - "refresh": 1, - "regex": "/.*namespace=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "maskafka", - "value": "maskafka" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Cluster Name", - "multi": false, - "name": "strimzi_cluster_name", - "options": [], - "query": "query_result(kafka_exporter_build_info{namespace=\"$kubernetes_namespace\"})", - "refresh": 1, - "regex": "/.*strimzi_io_cluster=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": [ - "kafka-cqin", - "kafka-errqueue" - ], - "value": [ - "kafka-cqin", - "kafka-errqueue" - ] - }, - "datasource": "prometheus", - "definition": "label_values(kafka_consumergroup_current_offset{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}, consumergroup)", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Consumer Group", - "multi": true, - "name": "consumergroup", - "options": [], - "query": "label_values(kafka_consumergroup_current_offset{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}, consumergroup)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "topic", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": [ - "kafka-cqin", - "kafka-errqueue" - ], - "value": [ - "kafka-cqin", - "kafka-errqueue" - ] - }, - "datasource": "prometheus", - "definition": "label_values(kafka_topic_partition_current_offset{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}, topic)", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Topic", - "multi": true, - "name": "topic", - "options": [], - "query": "label_values(kafka_topic_partition_current_offset{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}, topic)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "topic", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Kafka Exported JMX metrics", - "version": 1885 -} \ No newline at end of file + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Kafka Exported JMX metrics", + "version": 1885 +} diff --git a/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka-zookeeper.json b/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka-zookeeper.json index 13fca8d708..1d1c6715b0 100644 --- a/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka-zookeeper.json +++ b/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka-zookeeper.json @@ -1,1687 +1,1658 @@ { - "annotations": { - "list": [ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 60, + "iteration": 1631716755895, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 26, + "panels": [], + "title": "Zookeeper", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], + "datasource": "prometheus", + "description": "Quorum Size of Zookeeper ensemble", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 52, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "limit": 100, - "name": "Annotations & Alerts", - "showIn": 0, - "type": "dashboard" + "expr": "max(zookeeper_quorumsize{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" } - ] + ], + "thresholds": "2,3", + "title": "Quorum Size", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 60, - "iteration": 1631716755895, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 26, - "panels": [], - "title": "Zookeeper", - "type": "row" + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "datasource": "prometheus", + "description": "Number of Alive Connections", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "prometheus", - "description": "Quorum Size of Zookeeper ensemble", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 52, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(zookeeper_quorumsize{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "2,3", - "title": "Quorum Size", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "prometheus", - "description": "Number of Alive Connections", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 54, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(zookeeper_numaliveconnections{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "60,120", - "title": "Alive Connections", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Number of queued requests in the server. This goes up when the server receives more requests than it can process", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 16, - "x": 8, - "y": 1 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "zookeeper_outstandingrequests{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Outstanding Requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "id": 54, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "name": "range to text", + "value": 2 } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 64, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(zookeeper_inmemorydatatree_nodecount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "500,800", - "title": "Number of ZNodes", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "tableColumn": "", + "targets": [ + { + "expr": "sum(zookeeper_numaliveconnections{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "60,120", + "title": "Alive Connections", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Number of queued requests in the server. This goes up when the server receives more requests than it can process", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "prometheus", - "description": "Number of Watchers", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 66, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(zookeeper_inmemorydatatree_watchcount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "100,200", - "title": "Number of Watchers", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 1 }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "ZooKeeper Pods Memory Usage", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 87, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_usage_bytes{namespace=\"$kubernetes_namespace\",container=\"zookeeper\",pod=~\"$strimzi_cluster_name-$zk_node\"}) by (pod)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zookeeper_outstandingrequests{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outstanding Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Aggregated ZooKeeper Pods CPU Usage", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 9 - }, - "hiddenSeries": false, - "id": 85, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$kubernetes_namespace\",container=\"zookeeper\",pod=~\"$strimzi_cluster_name-$zk_node\"}[5m])) by (pod)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Kafka Broker Pods Disk Usage", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 89, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\"data(-[0-9]+)?-$strimzi_cluster_name-zookeeper-[0-9]+\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{persistentvolumeclaim}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Available Disk Space", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 64, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "name": "range to text", + "value": 2 } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Open File Descriptors", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 9 - }, - "hiddenSeries": false, - "id": 96, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{namespace=\"$kubernetes_namespace\",pod=~\"$strimzi_cluster_name-zookeeper-[0-9+]\",container=\"zookeeper\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "tableColumn": "", + "targets": [ + { + "expr": "avg(zookeeper_inmemorydatatree_nodecount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" } + ], + "thresholds": "500,800", + "title": "Number of ZNodes", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "datasource": "prometheus", + "description": "Number of Watchers", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 91, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}) by (kubernetes_pod_name)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JVM Memory Used", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 66, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 16 - }, - "hiddenSeries": false, - "id": 93, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(jvm_gc_collection_seconds_sum{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}[5m])) by (kubernetes_pod_name)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JVM GC Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "tableColumn": "", + "targets": [ + { + "expr": "sum(zookeeper_inmemorydatatree_watchcount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",strimzi_io_kind=\"Kafka\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "100,200", + "title": "Number of Watchers", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "ZooKeeper Pods Memory Usage", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 95, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(jvm_gc_collection_seconds_count{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}[5m])) by (kubernetes_pod_name)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JVM GC Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 87, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{namespace=\"$kubernetes_namespace\",container=\"zookeeper\",pod=~\"$strimzi_cluster_name-$zk_node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "JVM Thread Count", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 16 - }, - "hiddenSeries": false, - "id": 97, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "jvm_threads_current{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-zookeeper-[0-9+]\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JVM Thread Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Aggregated ZooKeeper Pods CPU Usage", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 9 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$kubernetes_namespace\",container=\"zookeeper\",pod=~\"$strimzi_cluster_name-$zk_node\"}[5m])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Amount of time (in ms) it takes for the server to respond to a client request", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "zookeeper_minrequestlatency{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Request Latency - Minimum", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "ms", - "label": "Request Latency (ms)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Kafka Broker Pods Disk Usage", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Amount of time (in ms) it takes for the server to respond to a client request", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 23 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "zookeeper_avgrequestlatency{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Request Latency - Average", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": "Request Latency (ms)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 89, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\"data(-[0-9]+)?-$strimzi_cluster_name-zookeeper-[0-9]+\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{persistentvolumeclaim}}", + "refId": "A" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Available Disk Space", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Amount of time (in ms) it takes for the server to respond to a client request", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 23 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Open File Descriptors", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 9 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{namespace=\"$kubernetes_namespace\",pod=~\"$strimzi_cluster_name-zookeeper-[0-9+]\",container=\"zookeeper\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Open File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "zookeeper_maxrequestlatency{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Request Latency - Maximum", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}) by (kubernetes_pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM Memory Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 16 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(jvm_gc_collection_seconds_sum{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}[5m])) by (kubernetes_pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM GC Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "ms", - "label": "Request Latency (ms)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null } - ], - "refresh": "5s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "Kafka", - "Zookeeper" - ], - "templating": { - "list": [ + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 95, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(jvm_gc_collection_seconds_count{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}[5m])) by (kubernetes_pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM GC Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "allValue": null, - "current": { - "selected": false, - "text": "kafka", - "value": "kafka" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "kubernetes_namespace", - "options": [], - "query": "query_result(zookeeper_inmemorydatatree_nodecount)", - "refresh": 1, - "regex": "/.*namespace=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "allValue": null, - "current": { - "selected": false, - "text": "maskafka", - "value": "maskafka" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Cluster Name", - "multi": false, - "name": "strimzi_cluster_name", - "options": [], - "query": "query_result(zookeeper_inmemorydatatree_nodecount{namespace=\"$kubernetes_namespace\"})", - "refresh": 1, - "regex": "/.*strimzi_io_cluster=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "JVM Thread Count", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 16 + }, + "hiddenSeries": false, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "jvm_threads_current{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-zookeeper-[0-9+]\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM Thread Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Node", - "multi": false, - "name": "zk_node", - "options": [], - "query": "query_result(zookeeper_inmemorydatatree_nodecount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "refresh": 1, - "regex": "/.*statefulset_kubernetes_io_pod_name=\"$strimzi_cluster_name-([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, - "time": { - "from": "now-1h", - "to": "now" + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Amount of time (in ms) it takes for the server to respond to a client request", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zookeeper_minrequestlatency{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency - Minimum", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Request Latency (ms)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Amount of time (in ms) it takes for the server to respond to a client request", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zookeeper_avgrequestlatency{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency - Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Request Latency (ms)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "yaxis": { + "align": false, + "alignLevel": null + } }, - "timezone": "", - "title": "ZooKeeper metrics", - "version": 1 -} \ No newline at end of file + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Amount of time (in ms) it takes for the server to respond to a client request", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zookeeper_maxrequestlatency{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency - Maximum", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Request Latency (ms)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": ["Kafka", "Zookeeper"], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "kafka", + "value": "kafka" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "kubernetes_namespace", + "options": [], + "query": "query_result(zookeeper_inmemorydatatree_nodecount)", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "maskafka", + "value": "maskafka" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Cluster Name", + "multi": false, + "name": "strimzi_cluster_name", + "options": [], + "query": "query_result(zookeeper_inmemorydatatree_nodecount{namespace=\"$kubernetes_namespace\"})", + "refresh": 1, + "regex": "/.*strimzi_io_cluster=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Node", + "multi": false, + "name": "zk_node", + "options": [], + "query": "query_result(zookeeper_inmemorydatatree_nodecount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "refresh": 1, + "regex": "/.*statefulset_kubernetes_io_pod_name=\"$strimzi_cluster_name-([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "", + "title": "ZooKeeper metrics", + "version": 1 +} diff --git a/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka.json b/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka.json index a920a9dcbe..99d8f91d76 100644 --- a/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka.json +++ b/ibm/mas_devops/roles/kafka/templates/redhat/grafana-json/kafka.json @@ -1,3098 +1,3038 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 62, + "iteration": 1631716609909, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], + "datasource": "prometheus", + "description": "Number of Brokers Online", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 46, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "repeatDirection": "h", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(kafka_server_replicamanager_leadercount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,2", + "title": "Brokers Online", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#299c46", "#e5ac0e", "#bf1b00"], + "datasource": "prometheus", + "description": "Number of active controllers in the cluster.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 36, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_controller_kafkacontroller_activecontrollercount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "2", + "title": "Active Controllers", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "datasource": "prometheus", + "description": "Unclean leader election rate", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 38, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(kafka_controller_controllerstats_uncleanleaderelections_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}[5m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "2", + "title": "Unclean Leader Election Rate", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], + "datasource": "prometheus", + "description": "Replicas that are online", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 40, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_server_replicamanager_partitioncount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0,0", + "title": "Online Replicas", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#508642", "rgba(237, 129, 40, 0.89)", "#bf1b00"], + "datasource": "prometheus", + "description": "Number of under-replicated partitions (| ISR | < | all replicas |).", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 0 + }, + "id": 30, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "100%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "1,5", + "title": "Under Replicated Partitions", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#508642", "#ef843c", "#bf1b00"], + "datasource": "prometheus", + "description": "Number of partitions which are at their minimum in sync replica count (| ISR | == | min.insync.replicas |).", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 102, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "100%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_cluster_partition_atminisr{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "1,5", + "title": "Partitions at minimum ISR", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#508642", "#ef843c", "#bf1b00"], + "datasource": "prometheus", + "description": "Number of partitions which are under their minimum in sync replica count (| ISR | < | min.insync.replicas |).", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 103, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "100%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_cluster_partition_underminisr{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "1,1", + "title": "Partitions under minimum ISR", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#508642", "#ef843c", "#bf1b00"], + "datasource": "prometheus", + "description": "Number of partitions that don’t have an active leader and are hence not writable or readable.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,1", + "title": "Offline Partitions Count", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 28, + "panels": [], + "title": "Kafka", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Kafka Broker Pods Memory Usage", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 82, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{namespace=\"$kubernetes_namespace\",pod=~\"$strimzi_cluster_name-$kafka_broker\",container=\"kafka\"}) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Aggregated Kafka Broker Pods CPU Usage", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 5 + }, + "hiddenSeries": false, + "id": 81, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$kubernetes_namespace\",pod=~\"$strimzi_cluster_name-$kafka_broker\",container=\"kafka\"}[5m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Kafka Broker Pods Disk Usage", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 83, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\"data(-[0-9]+)?-$strimzi_cluster_name-kafka-[0-9]+\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{persistentvolumeclaim}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Available Disk Space", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Open File Descriptors", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 5 + }, + "hiddenSeries": false, + "id": 107, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{namespace=\"$kubernetes_namespace\",pod=~\"$strimzi_cluster_name-$kafka_broker\",container=\"kafka\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Open File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}) by (kubernetes_pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM Memory Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 12 + }, + "hiddenSeries": false, + "id": 95, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(jvm_gc_collection_seconds_sum{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}[5m])) by (kubernetes_pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM GC Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(jvm_gc_collection_seconds_count{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}[5m])) by (kubernetes_pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM GC Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "JVM Thread Count", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 12 + }, + "hiddenSeries": false, + "id": 108, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "jvm_threads_current{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM Thread Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], + "datasource": "prometheus", + "description": "Total Incoming Byte Rate", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 98, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "0", + "to": "null" + } + ], + "repeatDirection": "h", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,2", + "title": "Total Incoming Byte Rate", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 62, - "iteration": 1631716609909, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "prometheus", - "description": "Number of Brokers Online", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 0 - }, - "id": 46, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": null, - "repeatDirection": "h", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(kafka_server_replicamanager_leadercount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "0,2", - "title": "Brokers Online", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], + "datasource": "prometheus", + "description": "Total Outgoing Byte Rate", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "#e5ac0e", - "#bf1b00" - ], - "datasource": "prometheus", - "description": "Number of active controllers in the cluster.", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 0 - }, - "id": 36, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_controller_kafkacontroller_activecontrollercount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "2", - "title": "Active Controllers", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "prometheus", - "description": "Unclean leader election rate", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 0 - }, - "id": 38, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(kafka_controller_controllerstats_uncleanleaderelections_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"}[5m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "2", - "title": "Unclean Leader Election Rate", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 19 }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "prometheus", - "description": "Replicas that are online", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 0 - }, - "id": 40, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_server_replicamanager_partitioncount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0,0", - "title": "Online Replicas", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "id": 99, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeatDirection": "h", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#508642", - "rgba(237, 129, 40, 0.89)", - "#bf1b00" - ], - "datasource": "prometheus", - "description": "Number of under-replicated partitions (| ISR | < | all replicas |).", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 0 - }, - "id": 30, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "100%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "1,5", - "title": "Under Replicated Partitions", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,2", + "title": "Total Outgoing Byte Rate", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], + "datasource": "prometheus", + "description": "Incoming Messages Rate", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#508642", - "#ef843c", - "#bf1b00" - ], - "datasource": "prometheus", - "description": "Number of partitions which are at their minimum in sync replica count (| ISR | == | min.insync.replicas |).", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 0 - }, - "id": 102, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "100%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_cluster_partition_atminisr{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "1,5", - "title": "Partitions at minimum ISR", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "format": "wps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#508642", - "#ef843c", - "#bf1b00" - ], - "datasource": "prometheus", - "description": "Number of partitions which are under their minimum in sync replica count (| ISR | < | min.insync.replicas |).", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 0 - }, - "id": 103, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "100%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_cluster_partition_underminisr{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "1,1", - "title": "Partitions under minimum ISR", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 19 }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#508642", - "#ef843c", - "#bf1b00" - ], - "datasource": "prometheus", - "description": "Number of partitions that don’t have an active leader and are hence not writable or readable.", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 0 - }, - "id": 32, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1,1", - "title": "Offline Partitions Count", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "id": 100, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeatDirection": "h", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 28, - "panels": [], - "title": "Kafka", - "type": "row" + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_messagesin_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,2", + "title": "Incoming Messages Rate", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], + "datasource": "prometheus", + "description": "Total Produce Request Rate", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Kafka Broker Pods Memory Usage", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 82, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_usage_bytes{namespace=\"$kubernetes_namespace\",pod=~\"$strimzi_cluster_name-$kafka_broker\",container=\"kafka\"}) by (pod)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "format": "reqps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 19 + }, + "id": 101, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" } + ], + "repeatDirection": "h", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Aggregated Kafka Broker Pods CPU Usage", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 5 - }, - "hiddenSeries": false, - "id": 81, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$kubernetes_namespace\",pod=~\"$strimzi_cluster_name-$kafka_broker\",container=\"kafka\"}[5m])) by (pod)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_totalproducerequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,2", + "title": "Total Produce Request Rate", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Byte Rate", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Kafka Broker Pods Disk Usage", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 5 - }, - "hiddenSeries": false, - "id": 83, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\"data(-[0-9]+)?-$strimzi_cluster_name-kafka-[0-9]+\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{persistentvolumeclaim}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Available Disk Space", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Total Incoming Byte Rate", + "refId": "A" + }, + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Total Outgoing Byte Rate", + "refId": "B" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Byte Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Open File Descriptors", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 5 - }, - "hiddenSeries": false, - "id": 107, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{namespace=\"$kubernetes_namespace\",pod=~\"$strimzi_cluster_name-$kafka_broker\",container=\"kafka\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 93, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}) by (kubernetes_pod_name)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JVM Memory Used", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_messagesin_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Total Incoming Messages Rate", + "refId": "D" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages In Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 12 - }, - "hiddenSeries": false, - "id": 95, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(jvm_gc_collection_seconds_sum{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}[5m])) by (kubernetes_pod_name)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JVM GC Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Produce Request Rate.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 12 - }, - "hiddenSeries": false, - "id": 97, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(jvm_gc_collection_seconds_count{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}[5m])) by (kubernetes_pod_name)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JVM GC Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 50, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_totalproducerequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Total Produce Request Rate", + "refId": "A" + }, + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_failedproducerequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Failed Produce Request Rate", + "refId": "B" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Produce Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "JVM Thread Count", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 12 - }, - "hiddenSeries": false, - "id": 108, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "jvm_threads_current{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JVM Thread Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Fetch Request Rate", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "prometheus", - "description": "Total Incoming Byte Rate", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "Bps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 19 - }, - "id": 98, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "0", - "to": "null" - } - ], - "repeatDirection": "h", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "0,2", - "title": "Total Incoming Byte Rate", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "prometheus", - "description": "Total Outgoing Byte Rate", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "Bps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 19 - }, - "id": 99, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeatDirection": "h", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "0,2", - "title": "Total Outgoing Byte Rate", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "hiddenSeries": false, + "id": 56, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "prometheus", - "description": "Incoming Messages Rate", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "wps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 19 - }, - "id": 100, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeatDirection": "h", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_messagesin_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "0,2", - "title": "Incoming Messages Rate", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "prometheus", - "description": "Total Produce Request Rate", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "reqps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 19 - }, - "id": 101, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeatDirection": "h", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_totalproducerequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "0,2", - "title": "Total Produce Request Rate", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(kafka_server_brokertopicmetrics_totalfetchrequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Fetch Request Rate", + "refId": "A" + }, + { + "expr": " sum(irate(kafka_server_brokertopicmetrics_failedfetchrequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed Fetch Request Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Fetch Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Byte Rate", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 44, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Total Incoming Byte Rate", - "refId": "A" - }, - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Total Outgoing Byte Rate", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Byte Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Network Processor Avg Idle Percent", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "hiddenSeries": false, - "id": 58, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_messagesin_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Total Incoming Messages Rate", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Messages In Per Second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_socketserver_networkprocessoravgidle_percent{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}*100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Processor Avg Idle Percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Produce Request Rate.", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 50, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_totalproducerequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Total Produce Request Rate", - "refId": "A" - }, - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_failedproducerequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Failed Produce Request Rate", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Produce Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Request Handler Avg Idle Percent", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Fetch Request Rate", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "hiddenSeries": false, - "id": 56, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(kafka_server_brokertopicmetrics_totalfetchrequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Fetch Request Rate", - "refId": "A" - }, - { - "expr": " sum(irate(kafka_server_brokertopicmetrics_failedfetchrequests_total{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Failed Fetch Request Rate", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Fetch Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_kafkarequesthandlerpool_requesthandleravgidle_percent{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}*100", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Handler Avg Idle Percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "description": "Network Processor Avg Idle Percent", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "hiddenSeries": false, - "id": 60, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "kafka_network_socketserver_networkprocessoravgidle_percent{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}*100", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Network Processor Avg Idle Percent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Disk writes", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 104, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(kafka_server_kafkaserver_linux_disk_write_bytes{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Writes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Disk reads", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 47 + }, + "hiddenSeries": false, + "id": 105, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(kafka_server_kafkaserver_linux_disk_read_bytes{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Reads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Disk reads", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 47 + }, + "hiddenSeries": false, + "id": 106, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_server_socket_server_metrics_connection_count{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}) by (kubernetes_pod_name, listener)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{listener}}-{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connection Count per Listener", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 55 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.3.10", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_log_log_size{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",topic=~\"$kafka_topic\",partition=~\"$kafka_partition\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{topic}}:{{partition}}", + "refId": "A" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Log Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": ["Kafka"], + "templating": { + "list": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "allValue": null, + "current": { + "selected": false, + "text": "kafka", + "value": "kafka" + }, "datasource": "prometheus", - "description": "Request Handler Avg Idle Percent", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "hiddenSeries": false, - "id": 62, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "kafka_server_kafkarequesthandlerpool_requesthandleravgidle_percent{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}*100", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Request Handler Avg Idle Percent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "kubernetes_namespace", + "options": [], + "query": "query_result(kafka_server_replicamanager_leadercount)", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "allValue": null, + "current": { + "selected": false, + "text": "maskafka", + "value": "maskafka" + }, "datasource": "prometheus", - "description": "Disk writes", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 47 - }, - "hiddenSeries": false, - "id": 104, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(kafka_server_kafkaserver_linux_disk_write_bytes{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk Writes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Cluster Name", + "multi": false, + "name": "strimzi_cluster_name", + "options": [], + "query": "query_result(kafka_server_replicamanager_leadercount{namespace=\"$kubernetes_namespace\"})", + "refresh": 1, + "regex": "/.*strimzi_io_cluster=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, "datasource": "prometheus", - "description": "Disk reads", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 47 - }, - "hiddenSeries": false, - "id": 105, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(kafka_server_kafkaserver_linux_disk_read_bytes{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk Reads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Broker", + "multi": false, + "name": "kafka_broker", + "options": [], + "query": "query_result(kafka_server_replicamanager_leadercount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", + "refresh": 1, + "regex": "/.*statefulset_kubernetes_io_pod_name=\"$strimzi_cluster_name-([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, "datasource": "prometheus", - "description": "Disk reads", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 47 - }, - "hiddenSeries": false, - "id": 106, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kafka_server_socket_server_metrics_connection_count{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"}) by (kubernetes_pod_name, listener)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{listener}}-{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Connection Count per Listener", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Topic", + "multi": false, + "name": "kafka_topic", + "options": [], + "query": "query_result(kafka_cluster_partition_replicascount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"})", + "refresh": 1, + "regex": "/.*topic=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 55 - }, - "hiddenSeries": false, - "id": 91, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.10", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "kafka_log_log_size{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",topic=~\"$kafka_topic\",partition=~\"$kafka_partition\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{topic}}:{{partition}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Log Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Partition", + "multi": true, + "name": "kafka_partition", + "options": [], + "query": "query_result(kafka_log_log_size{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",topic=~\"$kafka_topic\"})", + "refresh": 1, + "regex": "/.*partition=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" ], - "refresh": "5s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "Kafka" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "kafka", - "value": "kafka" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "kubernetes_namespace", - "options": [], - "query": "query_result(kafka_server_replicamanager_leadercount)", - "refresh": 1, - "regex": "/.*namespace=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "maskafka", - "value": "maskafka" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Cluster Name", - "multi": false, - "name": "strimzi_cluster_name", - "options": [], - "query": "query_result(kafka_server_replicamanager_leadercount{namespace=\"$kubernetes_namespace\"})", - "refresh": 1, - "regex": "/.*strimzi_io_cluster=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Broker", - "multi": false, - "name": "kafka_broker", - "options": [], - "query": "query_result(kafka_server_replicamanager_leadercount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\"})", - "refresh": 1, - "regex": "/.*statefulset_kubernetes_io_pod_name=\"$strimzi_cluster_name-([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Topic", - "multi": false, - "name": "kafka_topic", - "options": [], - "query": "query_result(kafka_cluster_partition_replicascount{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\"})", - "refresh": 1, - "regex": "/.*topic=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Partition", - "multi": true, - "name": "kafka_partition", - "options": [], - "query": "query_result(kafka_log_log_size{namespace=\"$kubernetes_namespace\",strimzi_io_cluster=\"$strimzi_cluster_name\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",topic=~\"$kafka_topic\"})", - "refresh": 1, - "regex": "/.*partition=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kafka metrics", - "version": 2 -} \ No newline at end of file + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "", + "title": "Kafka metrics", + "version": 2 +} diff --git a/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml index a8661d67f2..a9fb5fc456 100644 --- a/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/kmodels/tasks/determine-storage-classes.yml @@ -4,7 +4,6 @@ - name: "determine-storage-classes : Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Set Primary Storage (Required) # ----------------------------------------------------------------------------- - name: "determine-storage-classes : Default Primary Storage if not set by user" diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 index d03dfbce27..3a42e984ce 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 @@ -16,7 +16,7 @@ data: PIPELINE_DEFAULT_STEP_TAG: {{ mas_aibroker_pipeline_steps_tag }} CONNECTOR_DEFAULT_TAG: {{ mas_aibroker_connector_tag }} {% if mas_aibroker_saas %} - DEFAULT_TENANTS: provision-tenant + DEFAULT_TENANTS: {{ mas_aibroker_provision_tenant }} {% else %} DEFAULT_TENANTS: {{ tenantNamespace }} {% endif %} diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 index b8893fd874..5243829ebc 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-watcher/configmap.yml.j2 @@ -7,6 +7,6 @@ metadata: labels: app: km-watcher data: - WATCHER_SENDER_DELAY: "5" PIPELINES: mas-{{ mas_instance_id }}-aibroker TENANTS_BUCKET: {{ mas_aibroker_storage_tenants_bucket }} + EXCLUDE_TENANTS: {{ mas_aibroker_provision_tenant }} diff --git a/ibm/mas_devops/roles/kmodels/vars/main.yml b/ibm/mas_devops/roles/kmodels/vars/main.yml index 43f34dece6..7a4f767474 100644 --- a/ibm/mas_devops/roles/kmodels/vars/main.yml +++ b/ibm/mas_devops/roles/kmodels/vars/main.yml @@ -29,5 +29,5 @@ mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | defaul tenantNamespace: "aibroker-{{ mas_aibroker_tenant_name }}" # Saas -# mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) }}" mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" +mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" diff --git a/ibm/mas_devops/roles/mirror_case_prepare/tasks/main.yml b/ibm/mas_devops/roles/mirror_case_prepare/tasks/main.yml index 44c85b3ca4..a56951c0c7 100644 --- a/ibm/mas_devops/roles/mirror_case_prepare/tasks/main.yml +++ b/ibm/mas_devops/roles/mirror_case_prepare/tasks/main.yml @@ -1,5 +1,4 @@ --- - # 1. Check for undefined properties that do not have a default # ----------------------------------------------------------------------------- - name: "{{ case_name }} : Fail if required properties are not provided" @@ -123,7 +122,6 @@ src: ~/.ibm-pak/data/mirror/{{ case_name }}/{{ build_version }}/images-mapping-from-filesystem.txt dest: "{{ mirror_working_dir }}/manifests/from-filesystem/{{ case_name }}_{{ _manifest_version }}.txt" - # 9. IBM SLS 3.5.0 Bad Digest Hack # ----------------------------------------------------------------------------- # The SLS CASE bundle for 3.5.0 has the wrong image digest in it ... not really sure why TBH diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/defaults/main.yml b/ibm/mas_devops/roles/mirror_extras_prepare/defaults/main.yml index cf1af97fe2..92e8daadfe 100644 --- a/ibm/mas_devops/roles/mirror_extras_prepare/defaults/main.yml +++ b/ibm/mas_devops/roles/mirror_extras_prepare/defaults/main.yml @@ -8,7 +8,6 @@ registry_public_url: "{{ registry_public_host }}{% if registry_public_port != '' registry_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') | default('', true) }}" registry_public_url_with_path: "{{ registry_public_url }}{% if registry_prefix | length > 0 %}/{{ registry_prefix }}{% endif %}" - # Extras config # ----------------------------------------------------------------------------- extras_name: "{{ lookup('env', 'EXTRAS_NAME') }}" @@ -21,7 +20,6 @@ artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" artifactory_image_repo: "{{ lookup('env', 'ARTIFACTORY_IMAGE_REPO') | default('docker-na-public.artifactory.swg-devops.com/wiotp-docker-local', True) }}" - # Configure working directory # ----------------------------------------------------------------------------- mirror_working_dir: "{{ lookup('env', 'MIRROR_WORKING_DIR') }}" diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/tasks/catalog.yml b/ibm/mas_devops/roles/mirror_extras_prepare/tasks/catalog.yml index 9a50ba3e43..7c1f077979 100644 --- a/ibm/mas_devops/roles/mirror_extras_prepare/tasks/catalog.yml +++ b/ibm/mas_devops/roles/mirror_extras_prepare/tasks/catalog.yml @@ -1,5 +1,4 @@ --- - # 0. Get catalog digest if catalog is a dev catalog # ----------------------------------------------------------------------------- - name: "catalog : Fetch catalog digest for dev catalog" @@ -20,7 +19,6 @@ set_fact: catalog_digest: "{{ catalog_lookup.stdout }}" - # 1. Check for additional required properties # ----------------------------------------------------------------------------- - name: "catalog : Fail if required properties are not provided" @@ -34,7 +32,6 @@ include_vars: file: "catalog.yml" - # 2. Debug # ----------------------------------------------------------------------------- - name: "catalog : Airgap setup configuration" @@ -46,7 +43,6 @@ - "Registry Public Port ................... {{ registry_public_port }}" - "Registry Prefix ........................ {{ registry_prefix}}" - # 3. Generate mirror-manifest # ----------------------------------------------------------------------------- - name: "catalog : Ensure manifests/to-filesystem directory exists" diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/tasks/extras.yml b/ibm/mas_devops/roles/mirror_extras_prepare/tasks/extras.yml index e56eeeda2c..f4a4d81218 100644 --- a/ibm/mas_devops/roles/mirror_extras_prepare/tasks/extras.yml +++ b/ibm/mas_devops/roles/mirror_extras_prepare/tasks/extras.yml @@ -1,5 +1,4 @@ --- - # 1. Check for additional required properties # ----------------------------------------------------------------------------- - name: "{{ extras_name }} : Fail if required properties are not provided" @@ -8,12 +7,10 @@ - extras_version is defined and extras_version != "" fail_msg: "One or more required properties are missing" - - name: "{{ extras_name }} : Load variables" include_vars: file: "{{ extras_name }}_{{ extras_version }}.yml" - # 2. Debug # ----------------------------------------------------------------------------- - name: "{{ extras_name }} : Airgap setup configuration" @@ -25,7 +22,6 @@ - "Registry Public Port ................... {{ registry_public_port }}" - "Registry Prefix ........................ {{ registry_prefix}}" - # 3. Generate mirror-manifest # ----------------------------------------------------------------------------- - name: "{{ extras_name }} : Ensure manifests/to-filesystem directory exists" diff --git a/ibm/mas_devops/roles/mirror_extras_prepare/tasks/main.yml b/ibm/mas_devops/roles/mirror_extras_prepare/tasks/main.yml index 5c0bf39386..2c686c2e11 100644 --- a/ibm/mas_devops/roles/mirror_extras_prepare/tasks/main.yml +++ b/ibm/mas_devops/roles/mirror_extras_prepare/tasks/main.yml @@ -1,5 +1,4 @@ --- - # 1. Check for undefined properties that do not have a default # ----------------------------------------------------------------------------- - name: "extras : Fail if required properties are not provided" diff --git a/ibm/mas_devops/roles/mirror_images/tasks/main.yml b/ibm/mas_devops/roles/mirror_images/tasks/main.yml index 8cae2a7de7..1936866a4d 100644 --- a/ibm/mas_devops/roles/mirror_images/tasks/main.yml +++ b/ibm/mas_devops/roles/mirror_images/tasks/main.yml @@ -20,7 +20,6 @@ - "Mode ................................... {{ mirror_mode }}" - "Auth File .............................. {{ auth_file }}" - # 2. Set up auth secret # ----------------------------------------------------------------------------- - name: "{{ manifest_name }} : Create working directory" @@ -34,7 +33,6 @@ dest: "{{ auth_file }}" no_log: true - # 3. Modify the manifest # ----------------------------------------------------------------------------- - name: Validate that mirror_single_arch is correctly set @@ -67,7 +65,7 @@ ansible.builtin.lineinfile: path: "{{ mirror_working_dir }}/manifests/{{ mirror_mode }}/{{ manifest_name }}_{{ manifest_version }}.txt" state: absent - regexp: '-{{ item }}$' + regexp: "-{{ item }}$" backup: true with_items: "{{ exclude_arch }}" @@ -78,10 +76,9 @@ ansible.builtin.lineinfile: path: "{{ mirror_working_dir }}/manifests/{{ mirror_mode }}/{{ manifest_name }}_{{ manifest_version }}.txt" state: absent - regexp: '11.5.8.0' + regexp: "11.5.8.0" backup: true - # 4A. Execute the mirror (to filesystem) # ----------------------------------------------------------------------------- - name: "{{ manifest_name }} : Mirror Images to local filesystem" @@ -112,7 +109,6 @@ that: ocimagemirror_result.rc == 0 fail_msg: "Image mirroring failed (see debug information above)." - # 4B. Execute the mirror (from filesystem) # ----------------------------------------------------------------------------- - name: "{{ manifest_name }} : Mirror Images from local filesystem" @@ -143,7 +139,6 @@ that: ocimagemirror_result.rc == 0 fail_msg: "Image mirroring failed (see debug information above)." - # 4C. Execute the mirror (direct) # ----------------------------------------------------------------------------- - name: "Mirror Images directly" diff --git a/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml b/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml index 169c7f6e4a..f28ef5a862 100644 --- a/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml +++ b/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml @@ -7,7 +7,6 @@ mirror_working_dir: "{{ lookup('env', 'MIRROR_WORKING_DIR') }}" mirror_redhat_platform: "{{ lookup('env', 'MIRROR_REDHAT_PLATFORM') | default('False', True) | bool }}" mirror_redhat_operators: "{{ lookup('env', 'MIRROR_REDHAT_OPERATORS') | default('False', True) | bool }}" - # Target Registry # ----------------------------------------------------------------------------- registry_public_host: "{{ lookup('env', 'REGISTRY_PUBLIC_HOST') }}" @@ -21,14 +20,12 @@ registry_username: "{{ lookup('env', 'REGISTRY_USERNAME') }}" registry_password: "{{ lookup('env', 'REGISTRY_PASSWORD') }}" registry_auth: "{{ registry_username }}:{{ registry_password }}" - # Release Config # ----------------------------------------------------------------------------- ocp_release: "{{ lookup('env', 'OCP_RELEASE') }}" ocp_min_version: "{{ lookup('env', 'OCP_MIN_VERSION') }}" ocp_max_version: "{{ lookup('env', 'OCP_MAX_VERSION') }}" - # Authentication # ----------------------------------------------------------------------------- # https://console.redhat.com/openshift/install/pull-secret diff --git a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml index 53d30669da..4daab84c6e 100644 --- a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml +++ b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml @@ -9,14 +9,13 @@ - mirror_working_dir is defined and mirror_working_dir != "" fail_msg: "One or more required properties are missing" - # 2. Perform Mirroring # ----------------------------------------------------------------------------- - name: "Generate ImageSetConfiguration" ansible.builtin.template: src: imagesetconfiguration.yml.j2 dest: "{{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml" - mode: '664' + mode: "664" - name: "Debug Information" debug: diff --git a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml index cfbad8bfa0..b44b5fa2a4 100644 --- a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml +++ b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml @@ -8,7 +8,6 @@ - mirror_working_dir is defined and mirror_working_dir != "" fail_msg: "One or more required properties are missing" - # 2. Perform Mirroring # ----------------------------------------------------------------------------- - name: "Debug Information" diff --git a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/to-filesystem.yml b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/to-filesystem.yml index 6d0d69ea0a..c6c387272a 100644 --- a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/to-filesystem.yml +++ b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/to-filesystem.yml @@ -8,14 +8,13 @@ - mirror_working_dir is defined and mirror_working_dir != "" fail_msg: "One or more required properties are missing" - # 2. Perform Mirroring # ----------------------------------------------------------------------------- - name: "Generate ImageSetConfiguration" ansible.builtin.template: src: imagesetconfiguration.yml.j2 dest: "{{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml" - mode: '664' + mode: "664" - name: "Debug Information" debug: diff --git a/ibm/mas_devops/roles/mirror_ocp/tasks/main.yml b/ibm/mas_devops/roles/mirror_ocp/tasks/main.yml index ece89bc356..33378c96bb 100644 --- a/ibm/mas_devops/roles/mirror_ocp/tasks/main.yml +++ b/ibm/mas_devops/roles/mirror_ocp/tasks/main.yml @@ -1,5 +1,4 @@ --- - # 1. Check for undefined properties that do not have a default # ----------------------------------------------------------------------------- - name: "Fail if required properties are not provided" @@ -10,7 +9,6 @@ - redhat_pullsecret is defined and redhat_pullsecret != "" fail_msg: "One or more required properties are missing" - # 2. Check for required software # ----------------------------------------------------------------------------- - name: "Test if oc is installed" @@ -33,7 +31,6 @@ that: _oc_mirror_help['rc'] == 0 fail_msg: "oc mirror plugin must be installed" - # 3. Debug # ----------------------------------------------------------------------------- - name: "Airgap setup configuration" @@ -51,11 +48,10 @@ - "Target Registry ........................ {{ registry_public_url | default('', True) }}" - # 4. Generate new docker config # ----------------------------------------------------------------------------- - name: Generate Docker config - no_log: true # Output contains credentials for all docker registries + no_log: true # Output contains credentials for all docker registries command: > jq ".auths[\"{{ registry_public_url }}\"]={\"auth\":\"{{ registry_auth | b64encode }}\"}" "{{ redhat_pullsecret }}" register: new_pull_secret @@ -65,7 +61,6 @@ content: "{{ new_pull_secret.stdout }}" dest: "{{ mirror_working_dir }}/config.json" - # 5. Mirror Release # ----------------------------------------------------------------------------- - name: Run Mirroring diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index abc032395e..9cfaa3ecfa 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -10,7 +10,6 @@ mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('install', true) } # Backup mongodb databases for specified MAS application. Backup all mongodb databases if not specify this value. mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" - # mongodb community vars # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mongodb/tasks/determine-ibmcatalog-tag.yml b/ibm/mas_devops/roles/mongodb/tasks/determine-ibmcatalog-tag.yml index ebcc2dd797..11d0b9146c 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/determine-ibmcatalog-tag.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/determine-ibmcatalog-tag.yml @@ -34,7 +34,6 @@ debug: msg: "Last Catalog Version ................... {{ last_catalog_tag | default('') }}" - # 1. Get the IBM Catalog if available # ----------------------------------------------------------------------------- - name: "Lookup ibm-operator-catalog" @@ -62,7 +61,6 @@ mas_catalog_version: "{{ catalog_tag }}" register: catalog_metadata - # 2. Fallback to the most recent catalog tag # ----------------------------------------------------------------------------- - name: "Default to the newest catalog metadata" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/aws/destroy-data.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/aws/destroy-data.yml index 72ce1afd25..971ca8e33a 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/aws/destroy-data.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/aws/destroy-data.yml @@ -31,4 +31,4 @@ - name: "wipe_mongo : dump output" debug: - msg: '{{ wipe_mongo_output }}' + msg: "{{ wipe_mongo_output }}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/aws/docdb_secret_rotate.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/aws/docdb_secret_rotate.yml index 33b3824661..908d5c2ed0 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/aws/docdb_secret_rotate.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/aws/docdb_secret_rotate.yml @@ -34,7 +34,6 @@ - "DocumentDB Port ........................ {{ docdb_port }}" - "DocumentDB Instance Username ........... {{ docdb_instance_username }}" - # 2. Check for required software # ----------------------------------------------------------------------------- - name: "Test if mongosh is installed" @@ -47,7 +46,6 @@ that: ( mongosh_version.rc == 0 ) fail_msg: "mongosh must be installed (https://www.mongodb.com/docs/mongodb-shell/install/)" - # 3. Get the secret of documentdb from vault # ----------------------------------------------------------------------------- - name: "Download Amazon DocumentDB Certificate Authority (CA) certificate" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index d2504bc3d5..17250ab324 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -8,7 +8,6 @@ - seq: "{{ masbr_job_data_seq }}" phase: "InProgress" - - name: "Backup mongodb databases" block: # Create mongodb role and user for backing up databases @@ -16,7 +15,6 @@ - name: "Create mongodb role and user for backing up databases" include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/create-role-user.yml" - # Prepare mongodb database backup folder # ------------------------------------------------------------------------- - name: "Set fact: mongodb database backup folder" @@ -37,7 +35,6 @@ mkdir -p {{ mongodb_backup_folder }} {{ exec_in_pod_end }} - # Set database name filter # ------------------------------------------------------------------------- - name: "Set fact: default database name filter" @@ -59,7 +56,8 @@ - name: "Set fact: always backup databases for core" set_fact: - mongodb_db_app_filters: ["mas_{{ mas_instance_id }}_(core|catalog|adoptionusage)"] + mongodb_db_app_filters: + ["mas_{{ mas_instance_id }}_(core|catalog|adoptionusage)"] - name: "Set fact: append database name filters for ({{ mas_app_id }})" set_fact: @@ -75,7 +73,6 @@ debug: msg: "{{ mongodb_db_filter }}" - # Take a full backup of mongodb databases # ------------------------------------------------------------------------- - name: "Take a full backup of mongodb databases" @@ -139,7 +136,6 @@ debug: msg: "{{ _mongodump_output | json_query('results[*].stdout_lines') }}" - # Take an incremental backup of mongodb databases # ------------------------------------------------------------------------- - name: "Take an incremental backup of mongodb databases" @@ -199,7 +195,6 @@ debug: msg: "{{ _mongodump_output.stdout_lines }}" - # Create tar.gz archives of database backup files # ------------------------------------------------------------------------- - name: "Create tar.gz archives of database backup files" @@ -216,7 +211,6 @@ debug: msg: "{{ _du_files_output.stdout_lines }}" - # Copy backup files to specified storage location # ------------------------------------------------------------------------- - name: "Copy backup files from pod to specified storage location" @@ -248,7 +242,6 @@ - src_file: "{{ masbr_job_data_type }}/{{ masbr_job_name }}.tar.gz" dest_folder: "{{ masbr_job_data_type }}" - # Create query file used by the subsequent incremental backups # ------------------------------------------------------------------------- - name: "Create query file used by the subsequent incremental backups" @@ -278,7 +271,6 @@ - src_file: "{{ masbr_job_data_type }}/query.json" dest_folder: "{{ masbr_job_data_type }}" - # Update database backup status: Completed # ------------------------------------------------------------------------- - name: "Update database backup status: Completed" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml index 8911d64430..ed74e87464 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml @@ -29,7 +29,6 @@ mongodb_pod_temp_folder: "{{ masbr_pod_temp_folder }}/{{ masbr_job_name }}" mongodb_pvc_temp_folder: "{{ masbr_cf_pvc_mount_path }}/{{ masbr_job_name }}" - # Get mongodb admin password # ----------------------------------------------------------------------------- - name: "Get mongodb admin password" @@ -49,7 +48,6 @@ - _mongodb_password_output.resources[0].data.password is defined no_log: true - # Get mongodb ca file location # ----------------------------------------------------------------------------- - name: "Get mongodb ca file" @@ -67,7 +65,6 @@ - _mongodb_ca_file_output.rc == 0 - '"No such file or directory" not in _mongodb_ca_file_output.stdout' - # Get mongodb primary host # ----------------------------------------------------------------------------- - name: "Get mongodb server information" @@ -86,7 +83,6 @@ set_fact: mongodb_primary_host: "{{ _mongodb_info_output.stdout_lines[-1] | from_json | json_query('primary') }}" - # Output mongodb information # ----------------------------------------------------------------------------- - name: "Debug: mongodb information" @@ -98,7 +94,6 @@ - "MongoDB primary host ....................... {{ mongodb_primary_host }}" - "MongoDB ca file ............................ {{ mongodb_ca_file }}" - # Check if an exiting job is running # ------------------------------------------------------------------------- - name: "Try to find job lock file in pod" @@ -125,7 +120,6 @@ {{ exec_in_pod_end }} register: _create_restore_lock_output - # Check storage usage # ------------------------------------------------------------------------- - name: "Get storage usage of pod temporary folder" @@ -154,7 +148,6 @@ debug: msg: "{{ _df_pvc_output.stdout_lines }}" - # Workarounds # ------------------------------------------------------------------------- - name: "Set fact: how to copy backup files to specified storage location" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml index c6f7379ffa..4a9d97c804 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml @@ -4,7 +4,6 @@ mongodb_role: sysadmin mongodb_user: sysadmin - # Create mongodb role # ----------------------------------------------------------------------------- - name: "Get mongodb role '{{ mongodb_role }}'" @@ -38,7 +37,6 @@ debug: msg: "Create mongodb role result ........ {{ _mongodb_create_role_output.stdout_lines }}" - # Create mongodb user # ----------------------------------------------------------------------------- - name: "Get mongodb user '{{ mongodb_user }}'" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml index 19bf1c1b3c..19f45bf41d 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml @@ -5,7 +5,6 @@ - "Source cluster ................... {{ masbr_restore_from_yaml.source.domain }}" - "Target cluster ................... {{ masbr_cluster_domain }}" - # Because we will not restore the OauthClient collection, disable below patches for now. - name: "Update domain in 'mas_{{ mas_instance_id }}_core.OauthClient' collection" when: masbr_restore_to_diff_domain and false @@ -36,7 +35,6 @@ debug: msg: "Suite version ..................... {{ mas_core_version }}" - - name: "This fix only for MAS 8.x" when: mas_core_version is match("^8.") block: @@ -68,7 +66,6 @@ retries: 30 delay: 10 - # Restart entitymgr-ws pod # --------------------------------------------------------------------- - name: "Restart entitymgr-ws pod" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml index 5651e725c2..581bc827e5 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml @@ -34,7 +34,6 @@ {{ masbr_local_job_folder }}/{{ masbr_job_data_type }}/{{ _job_name }}.tar.gz {{ mongodb_namespace }}/{{ mongodb_pod_name }}:{{ mongodb_restore_folder }} - # Extract the tar.gz file # ------------------------------------------------------------------------- - name: "Extract the tar.gz file" @@ -54,7 +53,6 @@ - "Extract output folder .............. {{ mongodb_restore_folder }}/{{ _job_name }}" - "{{ _extract_output.stdout_lines }}" - # Restore mongodb databases # ------------------------------------------------------------------------- - name: "Set fact: mongodb ns instance" @@ -106,7 +104,6 @@ debug: msg: "{{ _mongorestore_output.stdout_lines }}" - # Save restored database names # ------------------------------------------------------------------------- - name: "Get restored database names" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 46941268bd..cfbdde90d9 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -8,7 +8,6 @@ - seq: "{{ masbr_job_data_seq }}" phase: "InProgress" - - name: "Restore mongodb databases" block: # Create mongodb role and user for backing up databases @@ -16,7 +15,6 @@ - name: "Create mongodb role and user for backing up databases" include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/create-role-user.yml" - # Prepare mongodb database restore folders # ------------------------------------------------------------------------- - name: "Set fact: mongodb database restore variables" @@ -41,7 +39,6 @@ debug: msg: "Database restore folder ........... {{ mongodb_restore_folder }}" - # This is an incremental backup, need to restore based on full backup first # ------------------------------------------------------------------------- - name: "Restore based on full backup" @@ -53,7 +50,6 @@ _job_type: "full" _job_name: "{{ masbr_restore_basedon }}" - # Restore databases from the specified Full or Incremental backup # ------------------------------------------------------------------------- - name: "Restore databases from the specified {{ 'Incremental' if masbr_restore_from_incr else 'Full' }} backup" @@ -62,13 +58,11 @@ _job_type: "{{ 'incr' if masbr_restore_from_incr else 'full' }}" _job_name: "{{ masbr_restore_from }}" - # Do some post restoration tasks # ------------------------------------------------------------------------- - name: "Do some post restoration tasks " include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database-patch.yml" - # Update database restore status: Completed # ------------------------------------------------------------------------- - name: "Update database restore status: Completed" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index fd50a9696e..605d26764e 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -6,13 +6,11 @@ that: mas_instance_id is defined and mas_instance_id != "" fail_msg: "mas_instance_id is required" - # Get mongodb information # ------------------------------------------------------------------------- - name: "Get mongodb information" include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/get-mongo-info.yml" - # Set common backup job variables # ----------------------------------------------------------------------------- - name: "Set fact: common backup job variables" @@ -28,7 +26,6 @@ - seq: "1" type: "database" - # Before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" @@ -38,7 +35,6 @@ _job_type: "backup" _component_before_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/before-backup-restore.yml" - - name: "Perform backup" block: # Update backup job status: New diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/determine-storage-classes.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/determine-storage-classes.yml index 54d6c7103e..4975a6b7fd 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/determine-storage-classes.yml @@ -7,7 +7,6 @@ - name: "Load default storage class information" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" - # 2. Set Storage (Required) # ----------------------------------------------------------------------------- - name: Default Storage if not set by user @@ -20,7 +19,6 @@ that: mongodb_storage_class is defined and mongodb_storage_class != "" fail_msg: "mongodb_storage_class must be defined" - # 3. Debug storage class configuration # ----------------------------------------------------------------------------- - name: "Debug MongoDb storage class configuration" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml index 3a11c92d7c..43737389c2 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml @@ -58,7 +58,6 @@ target_mongodb_readinessprobe_image: "{{ mongodb_readinessprobe_image_registry }}/{{ mongodb_readinessprobe_image_name }}@{{ mongodb_readinessprobe_image_digest }}" target_mongodb_image: "{{ mongodb_image_registry }}/{{ mongodb_image_name }}@{{ mongodb_image_digest }}" - # 2. Debug Properties # ----------------------------------------------------------------------------- - name: "community : install : Debug properties" @@ -100,7 +99,6 @@ - existing_mongo_operator.resources[0] is defined - existing_mongo_operator.resources[0].spec.containers[0].image != target_mongodb_operator_image - # 3. Install the CRD # ----------------------------------------------------------------------------- - name: "community : install : Install MongoDBCommunity CRD" @@ -108,7 +106,6 @@ apply: yes definition: "{{ lookup('template', 'templates/community/{{ mongodb_ce_version }}/crd.yml.j2') }}" - # 4. Create namespace & install RBAC # ----------------------------------------------------------------------------- - name: "community : install : Create namespace & install RBAC" @@ -116,7 +113,6 @@ apply: yes definition: "{{ lookup('template', 'templates/community/{{ mongodb_ce_version }}/rbac.yml.j2') }}" - # 5. Configure anyuid permissions in the MongoDb namespace # ----------------------------------------------------------------------------- - name: "community : install : Configure anyuid permissions in the MongoDb namespace" @@ -129,7 +125,6 @@ shell: | oc adm policy add-scc-to-user anyuid system:serviceaccount:{{ mongodb_namespace }}:mongodb-database - # 6. Install the operator # ----------------------------------------------------------------------------- - name: "community : install : Install the MongoDb Operator" @@ -141,7 +136,6 @@ status: "True" type: Available - # 7. Configure TLS using cert manager # ----------------------------------------------------------------------------- - name: "Create a issuer in '{{ mongodb_namespace }}' namespace" @@ -210,7 +204,6 @@ apply: yes definition: "{{ lookup('template', 'templates/community/tls.yml') }}" - # 8. Create MongoDb admin user credentials secret # ----------------------------------------------------------------------------- - name: "community : install : Check for existing user password secret" @@ -229,7 +222,6 @@ apply: yes definition: "{{ lookup('template', 'templates/community/admin-password.yml') }}" - # 9. Create MongoDb metrics endpoint secret # ----------------------------------------------------------------------------- - name: "community : install : Check for existing metrics endpoint secret" @@ -251,7 +243,6 @@ apply: yes definition: "{{ lookup('template', 'templates/community/metrics-endpoint-secret.yml.j2') }}" - # 10. Deploy the cluster # ----------------------------------------------------------------------------- - name: "community : install : Install new MongoDB cluster CR" @@ -297,7 +288,6 @@ - not controlled_upgrade - mongodb_ce_version is version('0.7.0', '==') - # 11. Wait for the cluster to be ready # status.version on the mongo cr is not available with older mongo operators # ----------------------------------------------------------------------------- @@ -388,7 +378,6 @@ - (mongodb_ce_version is version('0.7.8', '>=') and mongodb_cr.resources[0].status.version is defined) or mongodb_ce_version is version('0.7.0', '==') - (mongodb_ce_version is version('0.7.8', '>=') and mongodb_cr.resources[0].status.version == expected_mongodb_version) or mongodb_ce_version is version('0.7.0', '==') - # 12. Create MongoDb service monitor # ----------------------------------------------------------------------------- - name: "community : install : Check for existing service monitor" @@ -410,7 +399,6 @@ apply: yes definition: "{{ lookup('template', 'templates/community/servicemonitor.yml.j2') }}" - # 13. Create MongoDb Grafana dashboard # ----------------------------------------------------------------------------- - name: Get cluster info @@ -477,7 +465,6 @@ apply: yes definition: "{{ lookup('template', 'templates/community/dashboards/mongodb-overview-v5.yml.j2') }}" - # 14. Create MAS MongoCfg # ----------------------------------------------------------------------------- # Lookup the mongo pods diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml index c4794d140b..ccc4bda8ed 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml @@ -4,7 +4,6 @@ - name: "community : install : Lookup existing mongo install" include_tasks: tasks/providers/community/check-mongo-exists.yml - # 2. Load default storage classes (if not provided by the user and not an update) # ----------------------------------------------------------------------------- - name: Use chosen (or default) storage class @@ -41,7 +40,6 @@ mongodb_storage_capacity_data: "{{ existing_mongodb_storage_capacity_data }}" mongodb_storage_capacity_logs: "{{ existing_mongodb_storage_capacity_logs }}" - # 3. Perform upgrade from 4.2 to 4.4 if necessary # ----------------------------------------------------------------------------- # We will ALWAYS upgrade from 4.2 to 4.4, no matter what. v4.4 is the minimum version everyone should be running at @@ -53,13 +51,11 @@ - existing_mongo_minor_version is defined - existing_mongo_minor_version == '4.2' - # 4. Check for existing MongoDb install (again) # ----------------------------------------------------------------------------- - name: "community : install : Lookup existing mongo install (again)" include_tasks: tasks/providers/community/check-mongo-exists.yml - # 5. Install the selected MongoDb version # ----------------------------------------------------------------------------- - name: "community : install : Install/upgrade to chosen mongo version" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml index 38800a9eff..5cb5b4be8f 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml @@ -10,13 +10,11 @@ that: mas_instance_id is defined and mas_instance_id != "" fail_msg: "mas_instance_id is required" - # Get mongodb information # ------------------------------------------------------------------------- - name: "Get mongodb information" include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/get-mongo-info.yml" - # Set common restore job variables # ----------------------------------------------------------------------------- - name: "Set fact: common restore job variables" @@ -31,7 +29,6 @@ - seq: "1" type: "database" - # Before run tasks # ------------------------------------------------------------------------- - name: "Before run tasks" @@ -40,7 +37,6 @@ _job_type: "restore" _component_before_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/before-backup-restore.yml" - - name: "Perform restore" block: # Update restore job status: New diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml index 1a00eab5a0..e71fcd4082 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml @@ -1,5 +1,4 @@ --- - # 1. Debug Properties # ----------------------------------------------------------------------------- - name: "community : uninstall : Debug properties" @@ -7,7 +6,6 @@ msg: - "MongoDb namespace ............ {{ mongodb_namespace }}" - # 2. Delete MongoDBCommunity (mas-mongo-ce) # ----------------------------------------------------------------------------- - name: "community : uninstall : Delete the MongoDBCommunity CR" @@ -18,7 +16,6 @@ namespace: "{{ mongodb_namespace }}" name: mas-mongo-ce - # 3. Wait for StatefulSet to shut down # ----------------------------------------------------------------------------- - name: "community : uninstall : Wait for all sts to wind down" @@ -33,7 +30,6 @@ retries: 30 delay: 10 # seconds - # 3. Verify that MongoDBCommunity CR is gone # ----------------------------------------------------------------------------- - name: "community : uninstall : Look for the MongoDBCommunity CR after deletion" @@ -50,7 +46,6 @@ - verify_mongo_delete.resources is defined - verify_mongo_delete.resources | length == 0 - # 4. Delete Namespace # ----------------------------------------------------------------------------- - name: "community : uninstall : Delete Mongo Namespace" @@ -60,7 +55,6 @@ kind: Namespace name: "{{ mongodb_namespace }}" - # 5. Verify Namespace is gone # ----------------------------------------------------------------------------- - name: "community : uninstall : Wait for namespace to be deleted" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/validate-upgrade.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/validate-upgrade.yml index c46b492e38..75bfd90eef 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/validate-upgrade.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/validate-upgrade.yml @@ -33,7 +33,6 @@ fail_msg: "Mongo upgrade is not allowed due to either missing featureCompatibilityVersion field or wrong version in featureCompatibilityVersion field in MongoDBCommunity CR" when: mongodb_v5_upgrade or mongodb_v6_upgrade or mongodb_v7_upgrade - # Only allow Mongo upgrades to be next compatible minor version - name: "Assert Mongo upgrade compatibility: {{ target_mongodb_version }} must be in {{ mongo_compatible_target_version[existing_mongo_version] }}" when: existing_mongo_minor_version != '4.2' # will skip this assertion if mongo v4.2 as we'll assist in the auto upgrade to v4.4 and v5 in a single run diff --git a/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml b/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml index e326d1bfd0..9637715c32 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml @@ -13,9 +13,9 @@ spec: algorithm: ECDSA size: 256 dnsNames: - - "*.mas-mongo-ce-svc.{{mongodb_namespace}}.svc.cluster.local" - - "127.0.0.1" - - "localhost" + - "*.mas-mongo-ce-svc.{{mongodb_namespace}}.svc.cluster.local" + - "127.0.0.1" + - "localhost" issuerRef: name: mongo-issuer kind: Issuer diff --git a/ibm/mas_devops/roles/mongodb/templates/community/dashboards/json/mongodb-overview-grafana.json b/ibm/mas_devops/roles/mongodb/templates/community/dashboards/json/mongodb-overview-grafana.json index a94e40f4be..81c9aca778 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/dashboards/json/mongodb-overview-grafana.json +++ b/ibm/mas_devops/roles/mongodb/templates/community/dashboards/json/mongodb-overview-grafana.json @@ -1093,9 +1093,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1161,9 +1159,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1331,9 +1327,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -1488,11 +1482,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -1643,11 +1633,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -1820,11 +1806,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -1986,11 +1968,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2158,11 +2136,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2333,11 +2307,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2510,11 +2480,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2782,11 +2748,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2960,11 +2922,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3110,11 +3068,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3260,11 +3214,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3411,11 +3361,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3577,11 +3523,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3774,11 +3716,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3924,11 +3862,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4074,11 +4008,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4243,11 +4173,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4393,11 +4319,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4543,11 +4465,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4693,11 +4611,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4843,11 +4757,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4993,11 +4903,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5157,11 +5063,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5306,11 +5208,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5455,11 +5353,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5623,11 +5517,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -5798,12 +5688,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean", - "last" - ], + "calcs": ["min", "max", "mean", "last"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5948,11 +5833,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6118,11 +5999,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6323,11 +6200,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6474,11 +6347,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6624,11 +6493,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6784,11 +6649,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6938,11 +6799,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -7099,11 +6956,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -7251,11 +7104,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -7410,11 +7259,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -7585,11 +7430,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -7779,11 +7620,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -7971,11 +7808,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -8164,11 +7997,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -8324,11 +8153,7 @@ "options": { "alertThreshold": true, "legend": { - "calcs": [ - "min", - "max", - "mean" - ], + "calcs": ["min", "max", "mean"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -8505,4 +8330,4 @@ "uid": "bdz3fm9wei0aoc", "version": 2, "weekStart": "" -} \ No newline at end of file +} diff --git a/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml b/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml index bff24d6ebc..3c6ca1e53f 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml @@ -7,9 +7,9 @@ spec: duration: 8760h # 365d renewBefore: 360h # 15d dnsNames: - - "*.mas-mongo-ce-svc.{{mongodb_namespace}}.svc.cluster.local" - - "127.0.0.1" - - "localhost" + - "*.mas-mongo-ce-svc.{{mongodb_namespace}}.svc.cluster.local" + - "127.0.0.1" + - "localhost" issuerRef: name: mongo-server-cert-issuer diff --git a/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml b/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml index 598db8a3be..87ed944d6d 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml +++ b/ibm/mas_devops/roles/nvidia_gpu/tasks/main.yml @@ -3,7 +3,6 @@ # ----------------------------------------------------------------------------- - include_tasks: tasks/nfd_setup.yml - # 1. Look up OCP version in Cluster # ----------------------------------------------------------------------------- - name: "Look up cluster ocp version" @@ -25,7 +24,7 @@ api_version: packages.operators.coreos.com/v1 kind: PackageManifest name: gpu-operator-certified - namespace: openshift-marketplace # Note: A namespace must be provided when calling packages.operators.coreos.com/v1 + namespace: openshift-marketplace # Note: A namespace must be provided when calling packages.operators.coreos.com/v1 register: gpu_manifest - name: Assert that PackageManifest exists @@ -42,7 +41,6 @@ gpu_source_namespace: "{{ gpu_manifest.resources[0].status.catalogSourceNamespace }}" gpu_default_channel: "{{ gpu_manifest.resources[0].status.defaultChannel }}" - # 3. Provide Debug information # ----------------------------------------------------------------------------- - name: "Debug information" @@ -70,66 +68,66 @@ # ----------------------------------------------------------------------------- - name: Install GPU operator if not already installed block: - # 4.1 Create NVIDIA GPU project - # ----------------------------------------------------------------------------- - - name: "Create gpu namespace" - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: '{{ gpu_namespace }}' - - # 4.2 Create NVIDIA GPU operator group and subscription - # ----------------------------------------------------------------------------- - - name: "Create gpu operator group" - kubernetes.core.k8s: - definition: "{{ lookup('template', 'templates/gpu-operatorgroup.yml.j2') }}" - wait: yes - wait_timeout: 60 #subsequent tasks will fail if the CRD isn't fully created - - - name: "Create gpu subscription" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/gpu-subscription.yml.j2') }}" - wait: yes - wait_timeout: 300 - wait_condition: - type: 'CatalogSourcesUnhealthy' - status: "False" - - - name: "Wait until ClusterPolicy CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: clusterpolicies.nvidia.com - - # 4.3 Create GPU Cluster Policy - # ----------------------------------------------------------------------------- - - name: "Create Cluster Policy instance using latest driver" - when: gpu_driver_version is not defined - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/clusterpolicy-v2.yml.j2') }}" - - - name: "Create Cluster Policy instance using custom driver" - when: gpu_driver_version is defined - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/clusterpolicy-customversion.yml.j2') }}" - - # 4.4. Wait for Cluster Policy to be ready - # ----------------------------------------------------------------------------- - - name: "Wait for Cluster Policy instance to be ready (60s delay)" - kubernetes.core.k8s_info: - api_version: nvidia.com/v1 - name: "gpu-cluster-policy" - kind: ClusterPolicy - register: gpu_cr_result - until: - - gpu_cr_result.resources is defined and gpu_cr_result.resources | length == 1 - - gpu_cr_result.resources[0].status is defined - - gpu_cr_result.resources[0].status.state is defined - - gpu_cr_result.resources[0].status.state == 'ready' - retries: 30 # approx 30 minutes before we give up - delay: 60 # 1 minute + # 4.1 Create NVIDIA GPU project + # ----------------------------------------------------------------------------- + - name: "Create gpu namespace" + kubernetes.core.k8s: + api_version: v1 + kind: Namespace + name: "{{ gpu_namespace }}" + + # 4.2 Create NVIDIA GPU operator group and subscription + # ----------------------------------------------------------------------------- + - name: "Create gpu operator group" + kubernetes.core.k8s: + definition: "{{ lookup('template', 'templates/gpu-operatorgroup.yml.j2') }}" + wait: yes + wait_timeout: 60 #subsequent tasks will fail if the CRD isn't fully created + + - name: "Create gpu subscription" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/gpu-subscription.yml.j2') }}" + wait: yes + wait_timeout: 300 + wait_condition: + type: "CatalogSourcesUnhealthy" + status: "False" + + - name: "Wait until ClusterPolicy CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: clusterpolicies.nvidia.com + + # 4.3 Create GPU Cluster Policy + # ----------------------------------------------------------------------------- + - name: "Create Cluster Policy instance using latest driver" + when: gpu_driver_version is not defined + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/clusterpolicy-v2.yml.j2') }}" + + - name: "Create Cluster Policy instance using custom driver" + when: gpu_driver_version is defined + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/clusterpolicy-customversion.yml.j2') }}" + + # 4.4. Wait for Cluster Policy to be ready + # ----------------------------------------------------------------------------- + - name: "Wait for Cluster Policy instance to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: nvidia.com/v1 + name: "gpu-cluster-policy" + kind: ClusterPolicy + register: gpu_cr_result + until: + - gpu_cr_result.resources is defined and gpu_cr_result.resources | length == 1 + - gpu_cr_result.resources[0].status is defined + - gpu_cr_result.resources[0].status.state is defined + - gpu_cr_result.resources[0].status.state == 'ready' + retries: 30 # approx 30 minutes before we give up + delay: 60 # 1 minute when: - gpu_clusterpolicy_result.resources | length == 0 - gpu_clusterpolicy_result.resources[0].status is not defined diff --git a/ibm/mas_devops/roles/nvidia_gpu/tasks/nfd_setup.yml b/ibm/mas_devops/roles/nvidia_gpu/tasks/nfd_setup.yml index 9bb13a913b..273b465ecd 100644 --- a/ibm/mas_devops/roles/nvidia_gpu/tasks/nfd_setup.yml +++ b/ibm/mas_devops/roles/nvidia_gpu/tasks/nfd_setup.yml @@ -1,5 +1,4 @@ --- - # 1. Lookup the packagemanifest for gpu-operator-certified # ----------------------------------------------------------------------------- - name: Get nfd package manifest @@ -7,7 +6,7 @@ api_version: packages.operators.coreos.com/v1 kind: PackageManifest name: nfd - namespace: openshift-marketplace # Note: A namespace must be provided when calling packages.operators.coreos.com/v1 + namespace: openshift-marketplace # Note: A namespace must be provided when calling packages.operators.coreos.com/v1 register: nfd_manifest - name: Assert that PackageManifest exists @@ -24,7 +23,6 @@ nfd_source_namespace: "{{ nfd_manifest.resources[0].status.catalogSourceNamespace }}" nfd_default_channel: "{{ nfd_manifest.resources[0].status.defaultChannel }}" - # 2. Provide Debug information # ----------------------------------------------------------------------------- - name: "Debug information" @@ -33,7 +31,6 @@ - "NFD Namespace ...................... {{ nfd_namespace }}" - "NFD Channel ...................... {{ nfd_channel }}" - # 3. Check if NFD operator is already installed # ----------------------------------------------------------------------------- - name: Check if NFD operator is already installed @@ -53,63 +50,63 @@ # ----------------------------------------------------------------------------- - name: Install NFD operator if not already installed block: - # 4.1 Create NFD project - # ----------------------------------------------------------------------------- - - name: "Create nfd namespace" - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: '{{ nfd_namespace }}' + # 4.1 Create NFD project + # ----------------------------------------------------------------------------- + - name: "Create nfd namespace" + kubernetes.core.k8s: + api_version: v1 + kind: Namespace + name: "{{ nfd_namespace }}" - # 4.2 Create NFD operator group and subscription - # ----------------------------------------------------------------------------- - - name: "Create nfd operator group" - kubernetes.core.k8s: - definition: "{{ lookup('template', 'templates/nfd-operatorgroup.yml.j2') }}" - wait: yes - wait_timeout: 60 #subsequent tasks will fail if the CRD isn't fully created + # 4.2 Create NFD operator group and subscription + # ----------------------------------------------------------------------------- + - name: "Create nfd operator group" + kubernetes.core.k8s: + definition: "{{ lookup('template', 'templates/nfd-operatorgroup.yml.j2') }}" + wait: yes + wait_timeout: 60 #subsequent tasks will fail if the CRD isn't fully created - - name: "Create nfd subscription" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/nfd-subscription.yml.j2') }}" - wait: yes - wait_timeout: 300 - wait_condition: - type: 'CatalogSourcesUnhealthy' - status: "False" + - name: "Create nfd subscription" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/nfd-subscription.yml.j2') }}" + wait: yes + wait_timeout: 300 + wait_condition: + type: "CatalogSourcesUnhealthy" + status: "False" - - name: "Wait until NodeFeatureDiscoveries CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: nodefeaturediscoveries.nfd.openshift.io + - name: "Wait until NodeFeatureDiscoveries CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: nodefeaturediscoveries.nfd.openshift.io - # 4.3 Create NFD instance - # ----------------------------------------------------------------------------- - - name: "Create NodeFeatureDiscovery instance" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/nfd-instance.yml.j2') }}" + # 4.3 Create NFD instance + # ----------------------------------------------------------------------------- + - name: "Create NodeFeatureDiscovery instance" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/nfd-instance.yml.j2') }}" - # 4.4. Make sure NFD daemonsets have been created and all pods are ready - # ----------------------------------------------------------------------------- - # Depending on the version of NFD there may also be a nfd-master DaemonSet, but because - # newer versions use a combined worker-master model we will only wait for the nfd-worker - # DaemonSet so that this will work regardless of the version of OCP/NFD that is being used. + # 4.4. Make sure NFD daemonsets have been created and all pods are ready + # ----------------------------------------------------------------------------- + # Depending on the version of NFD there may also be a nfd-master DaemonSet, but because + # newer versions use a combined worker-master model we will only wait for the nfd-worker + # DaemonSet so that this will work regardless of the version of OCP/NFD that is being used. - - name: "Wait for 'nfd-worker' DaemonSet pods to be ready" - kubernetes.core.k8s_info: - api_version: apps/v1 - name: nfd-worker - namespace: "{{nfd_namespace}}" - kind: DaemonSet - register: nfd_worker_daemonset - until: - - nfd_worker_daemonset.resources is defined - - nfd_worker_daemonset.resources | length > 0 - - nfd_worker_daemonset.resources[0].status.numberReady > 0 - - nfd_worker_daemonset.resources[0].status.numberReady == nfd_worker_daemonset.resources[0].status.desiredNumberScheduled - retries: 30 # approx 30 minutes before we give up - delay: 60 # 1 minute + - name: "Wait for 'nfd-worker' DaemonSet pods to be ready" + kubernetes.core.k8s_info: + api_version: apps/v1 + name: nfd-worker + namespace: "{{nfd_namespace}}" + kind: DaemonSet + register: nfd_worker_daemonset + until: + - nfd_worker_daemonset.resources is defined + - nfd_worker_daemonset.resources | length > 0 + - nfd_worker_daemonset.resources[0].status.numberReady > 0 + - nfd_worker_daemonset.resources[0].status.numberReady == nfd_worker_daemonset.resources[0].status.desiredNumberScheduled + retries: 30 # approx 30 minutes before we give up + delay: 60 # 1 minute when: - nfd_worker_daemonset_result.resources | length == 0 diff --git a/ibm/mas_devops/roles/ocp_provision/tasks/main.yml b/ibm/mas_devops/roles/ocp_provision/tasks/main.yml index 47aaadb8fd..124df87876 100644 --- a/ibm/mas_devops/roles/ocp_provision/tasks/main.yml +++ b/ibm/mas_devops/roles/ocp_provision/tasks/main.yml @@ -30,11 +30,11 @@ vars: rotate_ocp_version: Monday: 4.16 - Tuesday: 4.14 - Wednesday: 4.13 - Thursday: 4.12 + Tuesday: 4.15 + Wednesday: 4.14 + Thursday: 4.16 Friday: 4.15 - Saturday: 4.16 + Saturday: 4.14 Sunday: 4.16 - name: "Set default OCP version" diff --git a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml index 21badf8ebf..003dc5e8b3 100644 --- a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml @@ -1,6 +1,6 @@ --- -# # Install Operator & create entitlement openshift-odh -# # ----------------------------------------------------------------------------- +# Install Operator & create entitlement openshift-odh +# ----------------------------------------------------------------------------- # - name: "Install Openshift odh Operator" # ibm.mas_devops.apply_subscription: # namespace: "{{ openshift_namespace }}" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-pv.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-pv.yml index da792acc54..c73928580a 100644 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-pv.yml +++ b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-pv.yml @@ -36,7 +36,7 @@ phase: "Completed" -- name: "Restoe pv data" +- name: "Restore pv data" when: mas_app_pv_list | length > 0 block: # Copy pv data from specified storage location diff --git a/ibm/mas_devops/roles/suite_app_uninstall/tasks/main.yml b/ibm/mas_devops/roles/suite_app_uninstall/tasks/main.yml index ff519b35bd..3000039fb5 100644 --- a/ibm/mas_devops/roles/suite_app_uninstall/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_app_uninstall/tasks/main.yml @@ -52,7 +52,7 @@ - app_pre_uninstall_tasks_absolute_file is file include_tasks: "{{ app_pre_uninstall_tasks_relative_file }}" -# 4. Delete Application Workspaces +# 5. Delete Application Workspaces # ----------------------------------------------------------------------------- - name: "Get all {{ mas_app_id }} Workspaces" when: @@ -78,7 +78,39 @@ wait_timeout: 600 # 10 minutes loop: "{{ app_workspace_lookup.resources }}" -# 4. Delete Application Custom Resource +# 6. Check and wait for Workspace deletion +# ----------------------------------------------------------------------------- +- name: Wait for {{ mas_app_id }} workspaces to be deleted + when: + - mas_app_ws_apiversion is defined + - mas_app_ws_kind is defined + kubernetes.core.k8s_info: + kind: "{{ mas_app_ws_kind }}" + api_version: "{{ mas_app_ws_apiversion }}" + namespace: "{{ mas_app_namespace }}" + register: workspace_status + until: workspace_status.resources == [] + retries: 120 # 20 minutes + delay: 10 + +# 6.1. Check and wait for ManageDeployment deletion +# This is a temporary workaround for now since the ManageWorkspace finalizer doesn't wait for ManageDeployment deletion to complete +# ----------------------------------------------------------------------------- +- name: Wait for ManageDeployment to be deleted + when: + - mas_app_id == "manage" + - mas_app_ws_apiversion is defined + - mas_app_ws_kind is defined + kubernetes.core.k8s_info: + kind: "ManageDeployment" + api_version: "apps.mas.ibm.com/v1" + namespace: "{{ mas_app_namespace }}" + register: deployment_destroy + until: deployment_destroy.resources == [] + retries: 50 # 30 minutes + delay: 30 + +# 7. Delete Application Custom Resource # ----------------------------------------------------------------------------- - name: "Get the {{ mas_app_id }} Custom Resource" when: @@ -105,7 +137,7 @@ wait_timeout: 600 # 10 minutes loop: "{{ app_cr_lookup.resources }}" -# 4. Delete Application Operator Subscription +# 8. Delete Application Operator Subscription # ----------------------------------------------------------------------------- - name: "Delete the {{ mas_app_id }} Operator Subscription" when: @@ -120,7 +152,7 @@ wait: true wait_timeout: 600 # 10 minutes -# 4. Delete Application Namespace +# 9. Delete Application Namespace # ----------------------------------------------------------------------------- - name: "Delete {{ mas_app_id }} namespace" kubernetes.core.k8s: @@ -131,7 +163,7 @@ wait: true wait_timeout: 600 # 10 minutes -# 4. Run Application Specific Post-Uninstall Tasks if there are any +# 10. Run Application Specific Post-Uninstall Tasks if there are any # ----------------------------------------------------------------------------- - name: Run application specific post-uninstall tasks vars: diff --git a/prettierrc.yaml b/prettierrc.yaml new file mode 100644 index 0000000000..eeb7a737e6 --- /dev/null +++ b/prettierrc.yaml @@ -0,0 +1,3 @@ +printWidth: 160 +tabWidth: 2 +bracketSameLine: false From fcccaf5c66bc72984458e8a3c2717a8078ac6a0e Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 20 Jan 2025 10:02:45 +0000 Subject: [PATCH 083/169] [patch] update playbook --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 6c29f2912f..7a4ce733cf 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -72,7 +72,7 @@ - ibm.mas_devops.dro - ibm.mas_devops.db2 # - ibm.mas_devops.minio - # - ibm.mas_devops.mariadb + # - ibm.mas_devops.mariadb # - ibm.mas_devops.odh # - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker From f8e51e2b87b7efc4167bfc9e75f9fa3903b7ea45 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 20 Jan 2025 10:10:01 +0000 Subject: [PATCH 084/169] Squashed commit of the following: commit d256f798038391af6827dbfd20b953d2e27fa906 Author: David Parker Date: Sat Jan 18 13:42:33 2025 +0000 [doc] Improve local install information commit 9c33df9b7da2167dd9154d87b09472d020ad8375 Author: rene-oromtz <157750458+rene-oromtz@users.noreply.github.com> Date: Mon Jan 13 10:26:02 2025 -0600 [patch] Add missing api_version to k8s_info lookup (#1616) commit 0417879bd9fe9912ff71a148b5c53ae0aadab3da Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Mon Jan 13 21:52:43 2025 +0530 [patch] Adds some deletion waits in suite_app_uninstall (#1615) Co-authored-by: Harsh Tamakuwala commit 9a0893d48a2fa727325f02722799517058e67441 Author: David Parker Date: Mon Jan 13 16:05:34 2025 +0000 [patch] Run prettier on yaml files (#1621) commit d50047368037b203bd672bac0d7b838fcffa087f Author: Tremaine Hart Date: Fri Jan 10 15:32:13 2025 -0600 [patch] [MASCORE-5063] Problems with Restoring PV data - DT420492 (#1618) commit 0fa5980090b7ea4887bfc59289b26b02ded49f22 Author: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Date: Thu Jan 9 02:28:38 2025 +0530 [minor] Support January Catalog Update (#1595) Co-authored-by: Parveen Kumar commit 5653ebdafa56e9cdf040a0ff82fb2e0dd3ed6d01 Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Mon Jan 6 16:24:12 2025 +0530 [patch] Fixes manage namespace uninstall stuck in terminating state (#1611) Co-authored-by: Harsh Tamakuwala commit 83d83650cb75eaccaf6460c9b0fe6fba03fe1b8f Author: leo-miran <105313348+leo-miran@users.noreply.github.com> Date: Thu Dec 19 17:09:45 2024 -0300 [patch] Remove OCP 4.12 & 4.13 from rotation (#1609) commit 13770e9343341a5963d2918f1814006a6f326964 Author: sekharcvalluri <54737838+sekharcvalluri@users.noreply.github.com> Date: Thu Dec 19 19:19:48 2024 +0530 [patch] Fix cos_bucket picking incorrect resourcegroup (#1593) Co-authored-by: Harsh Tamakuwala Co-authored-by: leo-miran <105313348+leo-miran@users.noreply.github.com> commit 0a3b97bce5c3ed58d10995c773305eeb376067f2 Author: David Parker Date: Thu Dec 19 13:47:57 2024 +0000 [major] Replace ICSP with IDMS (#1594) commit b08cda4327ab30d03326dbc5c7b35bee86bf16ba Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Thu Dec 19 18:49:47 2024 +0530 [patch] Update supported kafka versions (#1608) Co-authored-by: Harsh Tamakuwala commit cf057173082d9a94137cf3aa3c63d5bb9aa7b3bd Author: David Parker Date: Thu Dec 19 11:39:55 2024 +0000 [patch] Lock mas-devops less than v2 (#1607) commit 6ff397e628583040bc4e7aad56ae5558d7796db2 Author: David Parker Date: Tue Dec 17 22:35:41 2024 +0000 [patch] Enable HMAC by default for IBM COS (#1606) commit 49b2099ff65f2322fb8c670311dc5ea7030ae2ea Author: Rawa <61942902+rawa-resul@users.noreply.github.com> Date: Thu Dec 12 16:28:49 2024 +0000 [patch] Fix subscription lookup in upgrade (#1589) Co-authored-by: Rawa Resul commit 79c5991d608369f36eed2c50ea2edb69225b5ac7 Author: leo-miran <105313348+leo-miran@users.noreply.github.com> Date: Thu Dec 12 13:26:06 2024 -0300 [patch] fix default booleans for suite_dns (#1590) commit 1e91ba5f6361f484e67c1ab6181f399eebd4269c Author: Andrew Whitfield Date: Thu Dec 12 07:34:31 2024 +0000 [patch] Increase rosa deprovision wait time (#1592) commit 91bd67beb1637d11677e8d835594eb26dbd07da3 Author: Ian Boden <82514609+IanBoden@users.noreply.github.com> Date: Mon Dec 9 10:32:54 2024 +0000 [patch] reinstate zenmetastore and couchdb workarounds for cpd 4.8 (#1580) commit aa05d8a589d8334149fcd53c66d5fcea01977d07 Author: Rawa <61942902+rawa-resul@users.noreply.github.com> Date: Fri Dec 6 16:32:24 2024 +0000 [patch] Fix label_selectors pattern (#1586) Co-authored-by: Rawa Resul commit e0ba0decf64579d7b021e16e68d866f9704c6a5e Author: Parveen Kumar Date: Fri Dec 6 19:23:46 2024 +0530 [minor] Support December catalog (#1562) Co-authored-by: Anil Prajapati Co-authored-by: Jon Levell Co-authored-by: Jon Levell Co-authored-by: Ian Boden <82514609+IanBoden@users.noreply.github.com> Co-authored-by: Ian Boden Co-authored-by: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Co-authored-by: Terence Quinn Co-authored-by: David Parker Co-authored-by: Rawa Resul Co-authored-by: Rawa <61942902+rawa-resul@users.noreply.github.com> commit 9206656d7384f91469b03841606d73c06682bd18 Author: David Parker Date: Tue Dec 3 14:36:31 2024 +0000 [major] Use new OLM functions in mas-devops (#1573) commit 3ccdbef982c83989942445ef077d4117d81fec48 Author: Harsh Tamakuwala <31183552+harsh42774@users.noreply.github.com> Date: Mon Dec 2 18:30:04 2024 +0530 [patch] Fixes ibmcloud_api_key issue in COS deprovision for ibmcloud (#1581) Co-authored-by: Harsh Tamakuwala commit 1283c03b65c60cfc0095969abc878264db98724f Author: Terence Quinn Date: Fri Nov 29 03:52:51 2024 -0600 [patch] create unique certificate for each db2 instance (#1579) commit e04138afb8c7ffdfe4be159323d98d404426bb88 Author: David Parker Date: Tue Nov 26 21:40:39 2024 +0000 [patch] get_default_storage_classes should always set failed=False (#1578) commit 7014f49fc63410d44808a7082e8a1ce922e93879 Author: Sanjay Prabhakar Date: Tue Nov 26 16:32:42 2024 +0000 [patch] Add estimates for cpd in readme (#1577) commit 9933bc23fb272ba7fe169ded7a54de71e952b55b Author: David Parker Date: Tue Nov 26 13:46:05 2024 +0000 [patch] Use correct variable for mongo extras (#1576) commit bb720a394502505f5c91849992fc276e0a4cc1c4 Author: David Parker Date: Tue Nov 26 13:32:30 2024 +0000 [patch] Set cpd_product_version from catalog metadata (#1575) commit ba23e83ae87fe889f6bf2fd6bd26da16d1034881 Author: Terence Quinn Date: Mon Nov 25 15:03:03 2024 -0600 [patch] Fix missing catalog_digest (#1574) Co-authored-by: David Parker commit 0f60c6150f2f4541673c22b8312113fe24ce67ac Author: Sanjay Prabhakar Date: Fri Nov 22 17:33:20 2024 +0000 [patch] Add back restore.yml.j2 (#1572) Co-authored-by: Sanjay Prabhakar commit 739c4d03781bca7c983d92e103e62e6542b72fa7 Author: David Parker Date: Fri Nov 22 16:02:24 2024 +0000 [major] Overhaul backup/restore capability (#1557) Co-authored-by: Sanjay Prabhakar commit d5ef4102841b56a9fa7045be01989a3056fed5ef Author: David Parker Date: Fri Nov 22 12:39:48 2024 +0000 [doc] Fix formatting commit 61272c972bc692df31bdd64104234257b1e41ef2 Author: Sanjay Prabhakar Date: Fri Nov 22 12:27:00 2024 +0000 [patch] fix restore for manage role (#1571) Co-authored-by: Sanjay Prabhakar commit 0a13abdc64bf2e458dcb42c9560965a5c107da9f Author: Sanjay Prabhakar Date: Thu Nov 21 10:40:05 2024 +0000 [patch] Db2 backup/restore: fix copy local to pod script (#1569) Co-authored-by: Sanjay Prabhakar commit f4745094e7a119e8fbf5eb2a341e856baaf757e3 Author: David Parker Date: Wed Nov 20 23:43:25 2024 +0000 [major] Move catalog & storage class metadata to python package (#1561) commit df4fe4c55d7a1f1d78723f1452b8a6c3f4dda3a9 Author: Lokesh <110647904+lokesh-sreedhara@users.noreply.github.com> Date: Wed Nov 20 08:49:28 2024 -0500 [patch] remove MIRRORLOGPATH folder creation (#1568) commit 5668f331e38307250c4bdb22e5e5dd2e1f687607 Author: Lokesh <110647904+lokesh-sreedhara@users.noreply.github.com> Date: Wed Nov 20 07:49:43 2024 -0500 [patch] fix conflicting DB config (#1567) commit 5c595c11aacfdb9a3044c257ba9042a08005f311 Author: Sanjay Prabhakar Date: Tue Nov 19 17:45:03 2024 +0000 [patch] can skip db2 in br_manage playbook (#1566) Co-authored-by: Sanjay Prabhakar commit 8c8f9933ce35e16862e3f328ee5fe79349a634ea Author: stonepd Date: Tue Nov 19 16:39:18 2024 +0000 [patch] update CP4D README, remove outdated fix code (#1565) Co-authored-by: Sanjay Prabhakar commit 5eb2cd4aa2ef8f9fcc12626e1f85b577b9219ef3 Author: Sanjay Prabhakar Date: Tue Nov 19 14:42:36 2024 +0000 [patch] fix restore for core and fix tar command in restore tasks (#1564) Co-authored-by: Sanjay Prabhakar commit e81d3a3e0ddd89ef0f7ed1289ff8a7f311599b33 Author: dclain Date: Sat Nov 16 15:39:54 2024 -0600 [patch] Address several issues w/ Nvidia playbook (#1558) commit 1792e98b77b32185c486da66a31c4c89fb11bfcf Author: Tremaine Hart Date: Thu Nov 14 13:19:26 2024 -0600 [patch] Fixes and updates for IBMCloud COS support (#1555) Co-authored-by: David Parker commit 495e26d52efbda11ccb94bbab990a00652e9cc0e Author: Daniel Istrate <16302386+istrate@users.noreply.github.com> Date: Thu Nov 14 13:02:17 2024 +0200 [doc] Document storage class access mode (#1554) Co-authored-by: Daniel Istrate commit 08c22d74515c90a46f19894a630cd2c8155bccef Author: Anil Prajapati <169060963+anilprajapatiibm@users.noreply.github.com> Date: Tue Nov 12 02:51:17 2024 +0530 [minor] Support November catalog (#1508) Co-authored-by: Parveen Kumar Co-authored-by: Jon Levell Co-authored-by: Jon Levell Co-authored-by: Ian Boden <82514609+IanBoden@users.noreply.github.com> Co-authored-by: Ian Boden commit 35017863e41ff139d567671a507800c49b4f1bc5 Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Thu Nov 7 09:53:24 2024 -0800 [patch] MAXMIS-897: check if attribute 'deployimagestitching' is defined (#1551) Co-authored-by: Brendan More O'Ferrall commit 00c50f15b3ba266f4ec42d0fd44bd81f95f9543c Author: caemar <44871769+caemar@users.noreply.github.com> Date: Thu Nov 7 11:47:38 2024 +0100 [minor] Add Kafka Cluster CA certificate validityDays (#1543) commit ee3a2272b4a3418339eb2c4df77598d56f1f0179 Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Wed Nov 6 11:57:00 2024 -0800 [patch] MAXMIS-886: disable stitching deploy when civil is not enabled2 (#1548) Co-authored-by: Brendan More O'Ferrall commit 3ee251228c962a156692391bceb063c22ebac459 Author: Tremaine Hart Date: Wed Nov 6 11:41:34 2024 -0600 [patch] [MASCORE-4344] Grafana role overwriting critical k8s configuration resource (#1549) commit 482b3868b31468f220da901d5d23458eb47d8afd Author: bmoferrall <44264093+bmoferrall@users.noreply.github.com> Date: Tue Nov 5 21:51:50 2024 -0800 [minor] #MAXMIS-748: add role to auto configure image stitching (#1530) Co-authored-by: Brendan More O'Ferrall Co-authored-by: Alexandre Quinteiro Co-authored-by: unnati-solanki-git <131964221+unnati-solanki-git@users.noreply.github.com> commit dd13898e98f238b724da63f476d59312c494f139 Author: Tremaine Hart Date: Tue Nov 5 14:45:45 2024 -0600 [patch] [MASCORE-4344] Grafana role overwriting critical k8s configuration resource (#1547) commit ade2a0ea54989b2b2ce55a47890f5d249d13484a Author: Andrew Whitfield Date: Tue Nov 5 17:18:24 2024 +0000 [patch] wait for olm to update operatorcondition (#1545) commit c267990a26a599d405fc78fe248181ab13ad258a Author: Sanjay Prabhakar Date: Tue Nov 5 16:33:07 2024 +0000 [patch] prevent mongo upgrade when featureCompatibilityVersion is not defined or set incorrectly (#1544) Co-authored-by: Sanjay Prabhakar commit f4c04d569cbad2268ad763b344fd297399c5e19a Author: Ian Boden <82514609+IanBoden@users.noreply.github.com> Date: Tue Nov 5 11:21:08 2024 +0000 [patch] Fix gpu-certified-operator default channel (#1542) commit 733b595cebbd00232c415a1d62d456266936ed58 Author: David Parker Date: Mon Nov 4 15:49:12 2024 +0000 [patch] Fix oc exec commands (#1541) commit 224a9a8707befe26612523397a2564a1f31b4071 Author: David Parker Date: Thu Oct 31 18:10:39 2024 +0000 [patch] Reduce useless logging (#1540) --- .github/workflows/ansible.yml | 2 ++ docs/index.md | 43 +++++++++-------------------------- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 1b3cfc5ba2..5a68766eb5 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -7,7 +7,9 @@ on: - '**' jobs: ansible-build: + name: Build Ansible Collection runs-on: ubuntu-latest + if: ${{ !contains(github.event.head_commit.message, '[doc]') }} steps: - name: Install Python v3.11 uses: actions/setup-python@v5 diff --git a/docs/index.md b/docs/index.md index d38113e392..685be25e7e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,48 +36,27 @@ docker run -ti --rm --pull always quay.io/ibmmas/cli Local Install ------------------------------------------------------------------------------- -### Install Python & Ansible -[Python 3.9](https://www.python.org/downloads/) is recommended as it is the most widely used version of Python within our development team, but any in-support 3.x version of Python should work fine. +Install the collection direct from [Ansible Galaxy](https://galaxy.ansible.com/ibm/mas_devops), you must also install the [mas-devops](https://pypi.org/project/mas-devops) Python package. [Python 3.11](https://www.python.org/downloads/) is recommended as it is the most widely used version of Python within our development team, but any in-support version of Python should work. -```bash -python3 --version -python3 -m pip install ansible junit_xml pymongo xmljson jmespath kubernetes==12.0.1 openshift==0.12.1 -ansible --version -ansible-playbook --version +``` +ansible-galaxy collection install ibm.mas_devops +python3 -m pip install mas-devops ``` -We recommend using the latest version of ansible-core at all times (at time of writing this was v2.12.3) and the collection has a minimum supported version of ansible-core v2.10.3 which is enforced by the `ibm.mas_devops.ansible_version_check` role. - -### Install OpenShift CLI -If you do not already have the `oc` command line tool, you can download it as below: +Optionally, you can also pin the version of the collection that you install, allowing you to control exactly what version of the collection is in use in your automation: ``` -wget -q https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/4.14.7/openshift-client-linux-4.14.7.tar.gz -tar -zxf openshift-client-linux.tar.gz -mv oc kubectl /usr/local/bin/ -rm -rf openshift-client-linux.tar.gz -oc version +ansible-galaxy collection install ibm.mas_devops:18.10.4 +python3 -m pip install mas-devops ``` -### Install IBM Cloud CLI -If you are using this collection to manage an OpenShift cluster in IBM Cloud RedHat OpenShift Kubernetes Service (ROKS), then you must also install the IBM Cloud CLI: - -```bash -curl -sL https://raw.githubusercontent.com/IBM-Cloud/ibm-cloud-developer-tools/master/linux-installer/idt-installer | bash -ibmcloud version` -``` +The ansible collection makes use of many dependencies, you can find install scripts showing how we install these dependencies in our own container image in the [ibm-mas/cli-base](https://github.com/ibm-mas/cli-base/tree/stable/image/cli-base/install) repository, the dependencies you need will be determined by the roles that you intend to use, refer to the roles documentation for dependency infomation. -### Install the Ansible Collection -Install the collection direct from [Ansible Galaxy](https://galaxy.ansible.com/ibm/mas_devops) +!!! tip + Many systems contain more than one installation of Python, when you install the **mas-devops** package you must install it to the Python that Ansible is configured to use. You can check the version being used by Ansible by reviewing the output of `ansible --version`. -``` -ansible-galaxy collection install ibm.mas_devops -``` + If you see the error message `ERROR! Unexpected Exception, this is probably a bug: No module named 'mas'` it almost certainly means that you have not installed the mas-devops package, or have added it to the wrong instance of Python. -Optionally, you can also pin the version of the collection that you install, allowing you to control exactly what version of the collection is in use in your automation: -``` -ansible-galaxy collection install ibm.mas_devops:18.10.4 -``` Ansible Automation Platform ------------------------------------------------------------------------------- From 79aec2d75e963393456f8aa7f3b787fcb141a273 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 20 Jan 2025 11:53:10 +0000 Subject: [PATCH 085/169] [patch] update aibroker playbook --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 7a4ce733cf..412c9d083e 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -21,7 +21,7 @@ # DB2 # ----------------------------------------------------------------------------- - # db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" + db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" pre_tasks: # For the full set of supported environment variables refer to the playbook documentation @@ -76,3 +76,4 @@ # - ibm.mas_devops.odh # - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker + # - ibm.mas_devops.aibroker_app_config From 3fd2b3ee01ad695dce45f451a83cf4742b810b8f Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 21 Jan 2025 09:28:34 +0000 Subject: [PATCH 086/169] [patch] add new role for aibroker configuration --- .../roles/aibroker_app_config/README.md | 102 +++++++ .../aibroker_app_config/defaults/main.yml | 27 ++ .../aibroker_app_config/files/access_wx.py | 29 ++ .../files/alm_sample_sls_use.py | 267 ++++++++++++++++++ .../files/create_apikey.sh | 27 ++ .../files/create_bucket.py | 82 ++++++ .../files/create_dro_token.sh | 40 +++ .../files/create_entitlement.py | 103 +++++++ .../aibroker_app_config/files/create_s3.sh | 122 ++++++++ .../files/create_sls_secret.sh | 47 +++ .../files/create_tenant.py | 72 +++++ .../files/create_tenant_entitlement.py | 113 ++++++++ .../aibroker_app_config/files/create_wx.sh | 70 +++++ .../aibroker_app_config/files/delete_s3.py | 49 ++++ .../files/generate_api_key.py | 19 ++ .../roles/aibroker_app_config/meta/main.yml | 21 ++ .../tasks/config_sls/main.yml | 20 ++ .../roles/aibroker_app_config/tasks/main.yml | 6 + .../roles/aibroker_app_config/vars/main.yml | 50 ++++ 19 files changed, 1266 insertions(+) create mode 100644 ibm/mas_devops/roles/aibroker_app_config/README.md create mode 100644 ibm/mas_devops/roles/aibroker_app_config/defaults/main.yml create mode 100755 ibm/mas_devops/roles/aibroker_app_config/files/access_wx.py create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/alm_sample_sls_use.py create mode 100755 ibm/mas_devops/roles/aibroker_app_config/files/create_apikey.sh create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/create_bucket.py create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/create_dro_token.sh create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/create_entitlement.py create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/create_s3.sh create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/create_sls_secret.sh create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/create_tenant.py create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/create_tenant_entitlement.py create mode 100755 ibm/mas_devops/roles/aibroker_app_config/files/create_wx.sh create mode 100644 ibm/mas_devops/roles/aibroker_app_config/files/delete_s3.py create mode 100755 ibm/mas_devops/roles/aibroker_app_config/files/generate_api_key.py create mode 100644 ibm/mas_devops/roles/aibroker_app_config/meta/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_app_config/vars/main.yml diff --git a/ibm/mas_devops/roles/aibroker_app_config/README.md b/ibm/mas_devops/roles/aibroker_app_config/README.md new file mode 100644 index 0000000000..782f4a18d8 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/README.md @@ -0,0 +1,102 @@ +# AI Broker +===== + +This role provides support to install and configure AI Broker: + +* Install AI Broker api application +* Create, delete AI Broker tenant +* Create, delete AI Broker API Key +* Create, delete AWS S3 API Key +* Create, delete WatsonX AI API Key + +Role Variables +-------------- + +### tenant_action + +Action to be performed by AI Broker role. Valid values are `install` or `remove` . + +* Environment Variable: `TENANT_ACTION` +* Default Value: `install` + +### tenantName + +The tenant name for AI Broker role. + +* Environment Variable: `MAS_AIBROKER_TENANT_NAME` +* Default Value: `user` + +### app_domain + +The application domain for AI Broker role. Valid values is domain string `apps.domain` + +* Environment Variable: `APP_DOMAIN` +* Default Value: `` + +### mas_aibroker_s3_action + +Action to be performed by AI Broker role. Valid values are `install` or `remove` + +* Environment Variable: `MAS_AIBROKER_S3_ACTION` +* Default Value: `install` + +### mas_aibroker_storage_host + +The storge host for AI Broker role. + +* Environment Variable: `MAS_AIBROKER_STORAGE_HOST` +* Default Value: `` + +### mas_aibroker_storage_accesskey + +The storage accesskey for AI Broker role. + +* Environment Variable: `MAS_AIBROKER_STORAGE_ACCESSKEY` +* Default Value: `` + +### mas_aibroker_storage_secretkey + +The storage secretkey for AI Broker role. + +* Environment Variable: `MAS_AIBROKER_STORAGE_SECRETKEY` +* Default Value: `` + +### mas_aibroker_storage_region + +The storage region for AI Broker role. + +* Environment Variable: `MAS_AIBROKER_STORAGE_REGION` +* Default Value: `` + +### mas_aibroker_watsonx_action + +Action to be performed by AI Broker role. Valid values are `install` or `remove` + +* Environment Variable: `MAS_AIBROKER_WATSONX_ACTION` +* Default Value: `install` + +### mas_aibroker_watsonxai_apikey + +The watsonxai apikey for AI Broker role. + +* Environment Variable: `MAS_AIBROKER_WATSONXAI_APIKEY` +* Default Value: `` + +### mas_aibroker_watsonxai_url + +The watsonxai url for AI Broker role. + +* Environment Variable: `MAS_AIBROKER_WATSONXAI_URL` +* Default Value: `` + +### mas_aibroker_watsonxai_project_id + +The watsonxai project id for AI Broker role. + +* Environment Variable: `MAS_AIBROKER_WATSONXAI_PROJECT_ID` +* Default Value: `` + +License +------- + +EPL-2.0 diff --git a/ibm/mas_devops/roles/aibroker_app_config/defaults/main.yml b/ibm/mas_devops/roles/aibroker_app_config/defaults/main.yml new file mode 100644 index 0000000000..0b78b5f25c --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/defaults/main.yml @@ -0,0 +1,27 @@ +--- +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" +aibroker_namespace: 'mas-{{ mas_instance_id }}-aibroker' +mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" + +# Custom Labels +# ----------------------------------------------------------------------------- +custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string | ibm.mas_devops.string2dict() }}" + +# Source container registry +# ----------------------------------------------------------------------------- +mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" + +# MAS Entitlement +# ----------------------------------------------------------------------------- +mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}" +ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" +mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" + +# Development Registry Entitlement +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + +# MAS Annotation block +# ----------------------------------------------------------------------------- +mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/access_wx.py b/ibm/mas_devops/roles/aibroker_app_config/files/access_wx.py new file mode 100755 index 0000000000..ecf56af78b --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/access_wx.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + + +from ibm_watson_machine_learning.foundation_models import Model +from ibm_watson_machine_learning.foundation_models.utils.enums import ModelTypes +from ibm_watson_machine_learning.metanames import GenTextParamsMetaNames as GenParams +import sys + +GenParams().get_example_values() + +generate_params = {GenParams.MAX_NEW_TOKENS: 25} +try: + model = Model( + model_id=ModelTypes.FLAN_UL2, + params=generate_params, + credentials={"apikey": sys.argv[1], "url": sys.argv[2]}, + project_id=sys.argv[3], + ) + print("You enter the valid watsonx.ai credential.") + sys.exit(0) +except Exception as e: + print("You enter invalid watsonx.ai credential.") + print(e) + sys.exit(1) diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker_app_config/files/alm_sample_sls_use.py new file mode 100644 index 0000000000..666dc461ef --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/alm_sample_sls_use.py @@ -0,0 +1,267 @@ +# ----------------------------------------------------------- +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +# ----------------------------------------------------------- + +# This python script shows a simple model through which Sustainability offerings can interact in the SaaS pattern with a common ALM AppPoint pool. We'll +# see registration (provisioning), usage and deprovisioning. For clarity and simplicity, we do not perform validation of the certificate presented by the SLS service. Of course, you will want to +# do this in your offering. + +import os, base64, time, urllib3, requests, json, sys , secrets , string , random +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +slsUrl = sys.argv[1] +slsRegistrationKey = sys.argv[2] + +caCertificateForSSL = sys.argv[3] +# print('slsUrl='+slsUrl) +# print(slsRegistrationKey) +# print("caCertificateForSSL") +# print(caCertificateForSSL) +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Configuration section - you need to complete steps 1-5 +# - +# -------------------------------------------------------------------------------------------------------------------------------- + + +def generate_api_key(): + alphabet = string.ascii_letters + string.digits + temp= ''.join(secrets.choice(alphabet) for i in range(12)) + return "88"+ temp.lower() + + +# 1: Provide a name for your offering + +offeringName = "aibroker" + +# 2: Provide am identifier for your instance. This should be pseudounique, so chances of collisions are low. For example, based on the cluster +# identity, or a pseudo unique UUID created at initial installation time + +instanceIdentifier = generate_api_key() #"xxxx" +#print("id") +print(instanceIdentifier) + +# 3: Provide the SLS instance URL (SRE will give you this) + +#slsUrl = "https://sls.ibm-sls.ibm-sls.apps.xxx.cp.fyre.ibm.com" + +# 4: Provide the SLS registration key (SRE will give you this) + +#slsRegistrationKey = "xxxx" + +# 5: Decide if you want the demonstration to tear down your client afterwards (as you would do in a deprovision). If not, the client +# will remain registered for you to experiment with. + +# set it to be False Kewei +#deprovision=True +deprovision=False + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: preparation +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# Prepare the SLS clientId to use, and prepare for storing client certificates that are generated at registration time. +slsClientId=offeringName+"-"+instanceIdentifier +certs_dir=f"{os.getcwd()}/certs" +# print("xxxxxxx") +# print(certs_dir) +clientCaCrtPath=certs_dir+"/"+slsClientId+"-ca.crt" +clientTlsCrtPath=certs_dir+"/"+slsClientId+"-tls.crt" +clientTlsKeyPath=certs_dir+"/"+slsClientId+"-tls.key" + +#print ("Using clientId %s"%slsClientId) +#print ("Client certificates for interacting with SLS will be stored in "+certs_dir+"/"+slsClientId) + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: registration (provision) or reuse +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# Let's check to see if client certificates already exist - if so, we can immediately move on to working with AppPoints. If not, +# we'll need to register ourselves with SLS to obtain the client certificates. + +if os.path.isfile(clientCaCrtPath): + # A client certificate appears to exist already. So let's use it. + #print ("Discovered client certificates for SLS client "+slsClientId+". Let's use them.") + pass +else: + #print ("This appears to be a new SLS client. Let's attempt to register and obtain some client certificates.") + pass + + # Make initial registration request + # print("will use certificate for client provisioning ") + data = {'clientId': slsClientId, 'description': 'ALM sample client registration', 'role': 'manage'} # TODO: Rawa - Swagger docs incorrectly state 'id' rather than 'clientId' + headers = {'X-Registration-Key': slsRegistrationKey} # TODO: Rawa - Swagger docs don't document the need for this critical header + + #print ("Making initial registration request for client "+slsClientId) + response = requests.post(slsUrl+"/api/registrations",verify=False,json=data, headers=headers ) + response.raise_for_status() + + registrationId = response.json()['registrationId'] + #print("Completing registration for registration request "+registrationId) + + # + # Poll registration status until it's complete + # + provisioningWaiting = True + while provisioningWaiting: + headers = {'X-Registration-Key': slsRegistrationKey} + response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=False,headers=headers ) + response.raise_for_status() + status = response.json()['state'] + # print("Waiting for client provisioning to be completed. Current status: %s" %status) + if status=="AWAITING_CONFIRMATION": + # We are now ready to proceed; client certs are available + # print ("Extracting client certificates") + # print(clientTlsCrtPath) + clientTlskey= base64.b64decode(response.json()['certs']['tls.key']) + clientTlscrt= base64.b64decode(response.json()['certs']['tls.crt']) + clientCacrt= base64.b64decode(response.json()['certs']['ca.crt']) + with open(f'{clientTlsCrtPath}', 'wb') as f: + f.write(clientTlscrt) + with open(f'{clientTlsKeyPath}', 'wb') as f: + f.write(clientTlskey) + with open(f'{clientCaCrtPath}', 'wb') as f: + f.write(clientCacrt) + provisioningWaiting = False + time.sleep(1) + + # The next step is to confirm the client. Poll status until complete. + confirmationWaiting = True + while confirmationWaiting: + response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=False, cert=(clientTlsCrtPath,clientTlsKeyPath)) + #print (response) + status = response.json()['state'] + #print("Waiting for client registration to be confirmed. Current status: %s" %status) + if status=="REGISTERED": + # print ("Client registration confirmed.") + confirmationWaiting = False + time.sleep(1) + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: Define helper functions to perform SLS actions: mirror AppPoint usage to SLS and obtain AppPoint usage from SLS +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# Update 'quantity' AppPoints used in SLS. After 24 hours, the AppPoints will be automatically returned (ensuring no orphaned AppPoints if your service instance disappears) +def spendAppPoints(quantity): + #print ("Let's spend "+str(quantity)+" AppPoints") + data = {'validity': 86400, 'quantity': quantity} + response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) + response.raise_for_status() + +# String representation of AppPoint usage by your client. TODO: SLS will shortly be providing an update to correctly report usage for your client. At present, it returns the overall usage across all clients. +def getUsage(): + response = requests.get(slsUrl+"/api/tokens/",verify=False,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner + response.raise_for_status() + j = json.loads(response.content.decode('utf-8')) + return j[0]['used'] + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: Interact with AppPoints +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# The ALM SaaS metering model is to perform the following actions every hour: +# - Locally calculate your AppPoint usage (based on your individual pricing model) +# - Check out the ValueMetric-Usage feature with (i) a licenseeId that is pseudo unique, (ii) an expiry of 86400 and (iii) a quantity matching the number of AppPoints +# +# Note that at each hour, you simply update your AppPoint usage: you do not need to explicitly return your previous hour's AppPoints +# +# As an example, let's suppose your AppPoints are charged on (i) users and (ii) IOPoints. +# 25 Users = 25 AppPoints +# 100 IOPoints = 10 AppPoints +# +# You can use one of two approaches to check out AppPoints. +# +# SIMPLE "single shot" +# -------------------- +# - Report AppPoint usage in one go. +# - This means a single, consistent licenseeId at the instance level - for example, licenseeId="eis-126745fvabv", i.e. reusing the SLS clientId is convenient. +# - Set the quantity to be 25+10 = 35 +# +# +# COMPLEX "breakdown by capability" +# --------------------------------- +# - Report AppPoint usage as a sequence of updates, per capability +# - Enables you to use SLS as a way to understand unequivocally breakdown of AppPoint usage by capability +# - Each capability has its own licenseeId - for example: +# - example: licenseeId="eis-126745fvabv-users",quantity=25 +# licenseeId="eis-126745fvabv-iopoints",quantity=10 +# +# In this code example, we'll do a single shot for simplicity and clarity + +# As described above, for simplicity in the single shot model, we set the licenseeId to the SLS clientId +licenseeId = slsClientId + +# Now let's define the license feature to check out. In SaaS ALM, this will be "MAS-ValueMetric-Usage" +productId = "MAS-ValueMetric-Usage" + +# +# Now let's start the day-to-day operation of the software - i.e., hourly updates of SLS of AppPoint usage, and hourly reporting of data (ready to send to DRO) +# + +#print ("Simulating a few hours' usage. Note that you will require an ALM-ready version of SLS, and an ALM-ready license, to report AppPoint usage correctly in shared environments.") + +# Hour 0: Check out 25+10=35 AppPoints +#print ("Hour 0") +#spendAppPoints(35) +# Get AppPoint usage from SLS (e.g., for reporting via DRO). +#print("Usage: "+str(getUsage())) + +# Hour 1: It's now 143 AppPoints overall +#print ("Hour 1") +#spendAppPoints(143) +#print("Usage: "+str(getUsage())) + +# Hour 2: It's now 133 AppPoints overall +#print ("Hour 2") +#spendAppPoints(133) +#print("Usage: "+str(getUsage())) + +# Hour 3: you get the picture :) +# +# ... + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: optional deprovision +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +# Now suppose a request has been made to deprovision your service. How do we gracefully decouple from SLS? +if deprovision: + +# At deprovision, return your licenses and unregister the client + print ("Preparing for deprovision: returning any AppPoints to the pool") + response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response.raise_for_status() + print("Usage after deprovisioning: "+str(getUsage())) + + print ("Preparing for deprovision: unregistering the SLS client") + response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response.raise_for_status() + #print ("Deleting redundant client certificates") + os.remove(clientTlsCrtPath) + os.remove(clientTlsKeyPath) + os.remove(clientCaCrtPath) + +else: + #print ("No deprovisioning will occur. Your client is still registered with SLS.") + pass + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Demo section: end +# - +# -------------------------------------------------------------------------------------------------------------------------------- +#print ("Demo over.") diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_apikey.sh b/ibm/mas_devops/roles/aibroker_app_config/files/create_apikey.sh new file mode 100755 index 0000000000..a644cb9511 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_apikey.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +###### +# +# Assume python3 is installed +# +####### + +echo "Usage ./create_apikey.sh tenant_name_in_lower_case" +TENANT=$1 +AIBROKER=$2 + +echo "TENANT = ${TENANT}" + +if [ -z ${TENANT} ]; then + echo "using default tenant name=aibrokeruser" + TENANT='aibrokeruser' + #echo "Usage ./create_apikey.sh tenant_name_in_lower_case" + #exit 1 +fi + +aibrokerapikey=$(python3 ../roles/aibroker/files/generate_api_key.py) +echo "aibrokerapikey=$aibrokerapikey" + +echo "Creating apikey secret" +oc create secret generic ${TENANT}----apikey-secret -n ${AIBROKER} \ + --from-literal=AIBROKER_APIKEY=${aibrokerapikey} diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_bucket.py b/ibm/mas_devops/roles/aibroker_app_config/files/create_bucket.py new file mode 100644 index 0000000000..0091549c31 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_bucket.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + + +# create a new file, ~/.aws/credentials: +# [default] +# aws_access_key_id = YOUR_ACCESS_KEY_ID +# aws_secret_access_key = YOUR_SECRET_ACCESS_KEY + +# Create a new file, ~/.aws/config: +# [default] +# region = YOUR_PREFERRED_REGION + + +import boto3 +import sys +import uuid + + +if len(sys.argv) != 5: + print( + "Usage: python create_bucket.py your-tenant-name access-key secret-key your-region" + ) + exit() + + +if len(sys.argv) == 5 and sys.argv[1] is not None and sys.argv[2] is not None: + YOUR_BUCKET_NAME = sys.argv[1] + "----" + str(uuid.uuid4()) + # print(YOUR_BUCKET_NAME) + + access_key = sys.argv[2] + secret_key = sys.argv[3] + region = sys.argv[4] +else: + exit() + +# if YOUR_BUCKET_NAME is None: +# print( +# "Usage: python create_bucket.py your-tenant-name access-key secret-key your-region" +# ) +# exit() + +# if region is None: +# print( +# "Usage: python create_bucket.py your-tenant-name access-key secret-key your-region" +# ) +# exit() + +# if access_key is None: +# print( +# "Usage: python create_bucket.py your-tenant-name access-key secret-key your-region" +# ) +# exit() + +# if secret_key is None: +# print( +# "Usage: python create_bucket.py your-tenant-name access-key secret-key your-region" +# ) +# exit() + + +credentials = {"aws_access_key_id": access_key, "aws_secret_access_key": secret_key} + +try: + s3_resource = boto3.resource("s3", region_name=region, **credentials) + + # Need to add try/catch + + bucket_response = s3_resource.create_bucket( + Bucket=YOUR_BUCKET_NAME, + CreateBucketConfiguration={"LocationConstraint": region}, + ) + + print(YOUR_BUCKET_NAME) +except Exception as e: + print("Error:", e) + +# print(bucket_response) diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_dro_token.sh b/ibm/mas_devops/roles/aibroker_app_config/files/create_dro_token.sh new file mode 100644 index 0000000000..4850440c7c --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_dro_token.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +###### +# +# create DRO token in k8s secret +# +####### + +echo "Usage ./create_dro_token.sh tenant_name_in_lower_case instance_id token" + +NAMESPACE='aibroker' + +TENANT=$1 +instance_id=$2 +token=$3 + +if [[ -z ${instance_id} ]]; then + #echo "❌ Missing instance_id " + #exit 1 + echo "using default namespace aibroker" + NAMESPACE='aibroker' + +else + NAMESPACE=mas-${instance_id}-aibroker +fi + +echo $NAMESPACE + +echo $token + +if [ -z ${TENANT} ]; then + echo "using default tenant name=aibroker-user" + TENANT='aibroker-user' + #echo "Usage ./create_apikey.sh tenant_name_in_lower_case" + #exit 1 +fi + +echo "Creating DRO token in k8s secret" +oc create secret generic ${TENANT}----dro-secret -n ${NAMESPACE} \ + --from-literal=DRO_TOKEN=${token} diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_entitlement.py b/ibm/mas_devops/roles/aibroker_app_config/files/create_entitlement.py new file mode 100644 index 0000000000..82688e3884 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_entitlement.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + +import datetime +import ibm_db +import sys +import os + +if len(sys.argv) < 8: + print("Usage: python3 create_entitlement.py entitlement_type model_type allowed_number_of_model_training_per_month allowed_number_of_deployed_model_instance allowed_number_of_concurrent_training allowed_number_of_input allowed_watsonx_call allowed_file_size") + print("Sample usage") + print("Usage: python3 create_entitlement.py standard pcc 10 1 1 10000 100 1MB") + exit() + +entitlement_type = sys.argv[1] +model_type = sys.argv[2] +allowed_number_of_model_training_per_month = sys.argv[3] +allowed_number_of_deployed_model_instance = sys.argv[4] +allowed_number_of_concurrent_training = sys.argv[5] +allowed_number_of_input = sys.argv[6] +allowed_watsonx_call = sys.argv[7] +allowed_file_size = sys.argv[8] + +if entitlement_type is None: + entitlement_type = 'standard' + +if model_type is None: + model_type = 'pcc' + +if allowed_number_of_model_training_per_month is None: + allowed_number_of_model_training_per_month = 10 + +if allowed_number_of_deployed_model_instance is None: + allowed_number_of_deployed_model_instance = 1 + +if allowed_number_of_concurrent_training is None: + allowed_number_of_concurrent_training = 1 + +if allowed_number_of_input is None: + allowed_number_of_input = 10000 + +if allowed_watsonx_call is None: + allowed_watsonx_call = 10000 + +if allowed_file_size is None: + allowed_file_size = '1MB' + + +# Retrieving schema +schema = os.getenv("NAMESPACE") +schema_uppercase = schema.upper() +print(schema_uppercase) +assert schema != None, "namespace env variable not found." + +# Retrieving DB Details from env vars +db_host = os.getenv("db_host") +db_port = os.getenv("db_port") +db_name = os.getenv("db2_db") +db_username = os.getenv("db_username") +db_password = os.getenv("db_password") + +assert db_host != None, "db_host env variable not found." +assert db_port != None, "db_port env variable not found." +assert db_name != None, "db_name env variable not found." +assert db_username != None, "db_username env variable not found." +assert db_password != None, "db_password env variable not found." + + + +# ct stores current time +ct = datetime.datetime.now() + +array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} + +# use certificate to connect to DB2 +conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) + +#statement_entitlement_string = "INSERT INTO AIBROKER.AIBROKER_ENTITLEMENT_TYPE (ENTITLEMENT_TYPE, MODEL_TYPE, ALLOWED_NUMBER_OF_MODEL_TRAINING_PER_MONTH, ALLOWED_NUMBER_OF_DEPLOYED_MODEL_INSTANCE, ALLOWED_NUMBER_OF_CONCURRENT_TRAINING, ALLOWED_NUMBER_OF_INPUT,ALLOWED_NUMBER_OF_WATSONX_CALL, ALLOWED_FILE_SIZE, CREATION_TIMESTAMP, UPDATED_TIMESTAMP) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?,?);" + +statement_entitlement_string = f"INSERT INTO \"{schema_uppercase}\".AIBROKER_ENTITLEMENT_TYPE (ENTITLEMENT_TYPE, MODEL_TYPE, ALLOWED_NUMBER_OF_MODEL_TRAINING_PER_MONTH, ALLOWED_NUMBER_OF_DEPLOYED_MODEL_INSTANCE, ALLOWED_NUMBER_OF_CONCURRENT_TRAINING, ALLOWED_NUMBER_OF_INPUT,ALLOWED_NUMBER_OF_WATSONX_CALL, ALLOWED_FILE_SIZE, CREATION_TIMESTAMP, UPDATED_TIMESTAMP) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?,?);" +stmt = ibm_db.prepare(conn, statement_entitlement_string) +ibm_db.bind_param(stmt, 1, entitlement_type) +ibm_db.bind_param(stmt, 2, model_type) +ibm_db.bind_param(stmt, 3, allowed_number_of_model_training_per_month) +ibm_db.bind_param(stmt, 4, allowed_number_of_deployed_model_instance) +ibm_db.bind_param(stmt, 5, allowed_number_of_concurrent_training) +ibm_db.bind_param(stmt, 6, allowed_number_of_input) +ibm_db.bind_param(stmt, 7, allowed_watsonx_call) +ibm_db.bind_param(stmt, 8, allowed_file_size) +ibm_db.bind_param(stmt, 9, ct) +ibm_db.bind_param(stmt, 10, ct) +try: + stmt = ibm_db.execute(stmt) +except Exception as e: + print("Transaction couldn't be completed.") + ibm_db.rollback(conn) +else: + ibm_db.commit(conn) + print("Transaction complete. AIBROKER_ENTITLEMENT_TYPE table has been populated.") diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_s3.sh b/ibm/mas_devops/roles/aibroker_app_config/files/create_s3.sh new file mode 100644 index 0000000000..4391404b8d --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_s3.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +TENANT=$1 +AIBROKER=$2 +STORAGE_HOST=$3 +STORAGE_ACCESSKEY=$4 +STORAGE_SECRETKEY=$5 +STORAGE_PROVIDER=$6 + +if [[ "$STORAGE_PROVIDER" == "minio" ]]; then + STORAGE_PORT=$7 + STORAGE_REGION=$8 +fi + +if [[ "$STORAGE_PROVIDER" == "aws" ]]; then + STORAGE_PORT=$8 + STORAGE_REGION=$7 +fi + +echo $TENANT +echo $AIBROKER +echo $STORAGE_HOST +echo $STORAGE_ACCESSKEY +echo $STORAGE_SECRETKEY +echo $STORAGE_PROVIDER +echo $STORAGE_PORT +echo $STORAGE_REGION + +if [ -z ${TENANT} ]; then + echo "using default tenant name=aibrokeruser" + TENANT='aibrokeruser' + #echo "Usage ./create_s3.sh tenant_name_in_lower_case" + #exit 1 +fi + +# echo -n "Enter S3 access-key > " +# read accesskey +# echo "You entered: $accesskey" + +# echo -n "Enter S3 secret-key > " +# read secretkey +# echo "You entered: $secretkey" + +# echo -n "Enter S3 region > " +# read region +# echo "You entered: $region" + +# echo -n "Enter S3 provider url(for example: https://s3.amazonaws.com ) > " +# read s3_url +# echo "You entered: $s3_url" + +# echo "TENANT = ${TENANT}" +# echo "access-key =${accesskey}" +# echo "secret-key =${secretkey}" +# echo "region =${region}" + +# if [ -z ${TENANT} ]; then +# echo "❌ Missing tenant name" +# exit 1 +# fi + +# if [ -z ${access-key} ]; then +# echo "❌ Missing S3 accesskey" +# exit 1 +# fi + +# if [ -z ${secret-key} ]; then +# echo "❌ Missing S3 secretkey" +# exit 1 +# fi + +# if [ -z ${s3_url} ]; then +# echo "❌ Missing S3 provider url" +# exit 1 +# fi + +# if [ -z ${region} ]; then +# echo "❌ Missing S3 region" +# exit 1 +# fi + +bucketpostfix='-training-bucket' +if [[ $STORAGE_HOST == *"amazonaws"* ]]; then + echo 'it is Amazon AWS' + bucketname=$(python3 ../roles/aibroker/files/create_bucket.py ${TENANT} ${STORAGE_ACCESSKEY} ${STORAGE_SECRETKEY} ${STORAGE_REGION}) +else + echo 'it is Non Amazon AWS' + bucketname="${TENANT}${bucketpostfix}" +fi + +echo "bucketname=$bucketname" + +if [[ "$bucketname" == "Error"* ]]; then + echo "The variable $bucketname starts with the word 'Error'. Can not create bucket. Please check your S3 credential." + exit 1 +else + echo "The credentail of S3 is valid. The bucketname $bucketname is created successfully." + # Add storage secrets + echo "Creating S3 secret in k8s" + echo ${STORAGE_PROVIDER} + if [ "$STORAGE_PROVIDER" == "aws" ]; then + # Create a Kubernetes secret with the AWS S3 credentials + oc create secret generic ${TENANT}----s3-secret -n ${AIBROKER} \ + --from-literal=ACCESS-KEY=${STORAGE_ACCESSKEY} \ + --from-literal=SECRET-KEY=${STORAGE_SECRETKEY} \ + --from-literal=URL=${STORAGE_HOST} \ + --from-literal=REGION=${STORAGE_REGION} \ + --from-literal=BUCKET-NAME=${bucketname} + # Check if the storage provider is Minio + elif [ "$STORAGE_PROVIDER" == "minio" ]; then + # Create a Kubernetes secret with the Minio credentials + oc create secret generic ${TENANT}----s3-secret -n ${AIBROKER} \ + --from-literal=ACCESS-KEY=${STORAGE_ACCESSKEY} \ + --from-literal=SECRET-KEY=${STORAGE_SECRETKEY} \ + --from-literal=URL=http://${STORAGE_HOST}:${STORAGE_PORT} \ + --from-literal=REGION=${STORAGE_REGION} \ + --from-literal=BUCKET-NAME=${bucketname} + else + echo "Invalid storage provider specified." + exit 1 + fi +fi diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker_app_config/files/create_sls_secret.sh new file mode 100644 index 0000000000..f942f3e026 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_sls_secret.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +TENANT=$1 +instance_id=$2 + +if [ -z ${TENANT} ]; then + #echo "❌ Missing tenant name" + #exit 1 + echo "Usage ./create_sls_secret.sh tenant-name aibroker-instance-id-from-CustomResourceDefinitions sls-url-from-sls-routes slsRegistrationKey-from-sls-api-pod-Environment path_to_the_ca_crt" + echo "for example" + echo "./create_sls_secret.sh aibroker-user aibdev https://sls.ibm-sls.xxx.ibm.com xxxx-xxxx-xxxx-xxx" + exit 1 +fi + +cwd=$(pwd) +echo "location pwd in script create secret " + +mkdir -p certs + +echo "creating SLS registration, please wait....." +# instanceIdentifier=`python3 {{ role_path }}/files/alm_sample_sls_use.py $3 $4` +instanceIdentifier=$(python3 ../roles/aibroker/files/alm_sample_sls_use.py $3 $4 $5) +# echo "SLS registration is created successfully." + +registrationKey=$4 + +# echo "----------" +# echo $instanceIdentifier +# echo "----------" +# Set variables +# Need to change +NAMESPACE=mas-${instance_id}-aibroker +slsClientId="aibroker"-$instanceIdentifier +# echo $slsClientId + +SECRET_NAME=${TENANT}----sls-secret +CA_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-ca.crt +TLS_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.crt +KEY_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.key + +# Create the secret +oc create secret generic $SECRET_NAME -n $NAMESPACE \ + --from-file=$CA_FILE_PATH \ + --from-file=$TLS_FILE_PATH \ + --from-file=$KEY_FILE_PATH \ + --from-literal=SLS_CLIENT_ID=${slsClientId} \ + --from-literal=SLS_REGISTRATION_KEY=${registrationKey} diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_tenant.py b/ibm/mas_devops/roles/aibroker_app_config/files/create_tenant.py new file mode 100644 index 0000000000..d36891e97d --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_tenant.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + +import datetime +import ibm_db +import sys +import os + +if len(sys.argv) < 3: + print("Usage: python3 create_tenant.py tenant_name sls_url dro_url ") + print("Sample usage") + print("Usage: python3 create_tenant.py kmodels-user3 sls_url dro_url") + exit() + +tenant_name = sys.argv[1] +sls_url = sys.argv[2] +dro_url = sys.argv[3] + + + +# Retrieving schema +schema = os.getenv("NAMESPACE") +schema_uppercase = schema.upper() +print(schema_uppercase) +assert schema != None, "namespace env variable not found." + +# Retrieving DB Details from env vars +db_host = os.getenv("db_host") +db_port = os.getenv("db_port") +db_name = os.getenv("db2_db") +db_username = os.getenv("db_username") +db_password = os.getenv("db_password") + +assert db_host != None, "db_host env variable not found." +assert db_port != None, "db_port env variable not found." +assert db_name != None, "db_name env variable not found." +assert db_username != None, "db_username env variable not found." +assert db_password != None, "db_password env variable not found." + +array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} +conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Adding entry in AIBROKER_TENANT table. +# - +# -------------------------------------------------------------------------------------------------------------------------------- + +#statement_entitlement_string = f"INSERT INTO {schema}.AIBROKER_TENANT(NAME, STATUS, SLS_URL, DRO_URL) VALUES(?,?,?,?);" +statement_entitlement_string = f"INSERT INTO \"{schema_uppercase}\".AIBROKER_TENANT(NAME, STATUS, SLS_URL, DRO_URL) VALUES(?,?,?,?);" + + +stmt = ibm_db.prepare(conn, statement_entitlement_string) +ibm_db.bind_param(stmt, 1, tenant_name) +ibm_db.bind_param(stmt, 2, 'active') +ibm_db.bind_param(stmt, 3, sls_url) +ibm_db.bind_param(stmt, 4, dro_url) + +try: + stmt = ibm_db.execute(stmt) +except Exception as e: + print("Transaction couldn't be completed. The Tenant has not been created") + ibm_db.rollback(conn) +else: + ibm_db.commit(conn) + print("Transaction complete. The Tenant has been created.") + + diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_tenant_entitlement.py b/ibm/mas_devops/roles/aibroker_app_config/files/create_tenant_entitlement.py new file mode 100644 index 0000000000..e71486a5dc --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_tenant_entitlement.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + +import datetime +import ibm_db +import sys +import os + +if len(sys.argv) < 5: + print("Usage: python3 create_tenant_entitlement.py tenant_name entitlement_type model_type entilement_start_date entilement_end_date ") + print("Sample usage") + print("Usage: python3 create_tenant_entitlement.py kmodels-user3 standard pcc 2024-09-12 2025-09-12") + exit() + +tenant_name = sys.argv[1] + +entitlement_type = sys.argv[2] +model_type = sys.argv[3] +entilement_start_date=sys.argv[4] +entilement_end_date=sys.argv[5] + + +# Retrieving schema +schema = os.getenv("NAMESPACE") +schema_uppercase = schema.upper() +print(schema_uppercase) +assert schema != None, "namespace env variable not found." + +# Retrieving DB Details from env vars +db_host = os.getenv("db_host") +db_port = os.getenv("db_port") +db_name = os.getenv("db2_db") +db_username = os.getenv("db_username") +db_password = os.getenv("db_password") + +assert db_host != None, "db_host env variable not found." +assert db_port != None, "db_port env variable not found." +assert db_name != None, "db_name env variable not found." +assert db_username != None, "db_username env variable not found." +assert db_password != None, "db_password env variable not found." + +array = {ibm_db.SQL_ATTR_AUTOCOMMIT: ibm_db.SQL_AUTOCOMMIT_OFF} +conn = ibm_db.pconnect(f"Security=SSL;SSLServerCertificate=/tmp/db2-certificate.pem;DATABASE={db_name};HOSTNAME={db_host};PORT={db_port};PROTOCOL=TCPIP;UID={db_username};PWD={db_password};", "", "", array) + +entitlement_type_string = f"SELECT ID FROM \"{schema_uppercase}\".AIBROKER_ENTITLEMENT_TYPE WHERE ENTITLEMENT_TYPE = ? AND MODEL_TYPE = ?;" +print(entitlement_type_string) +stmt = ibm_db.prepare(conn, entitlement_type_string) +ibm_db.bind_param(stmt, 1, entitlement_type) +ibm_db.bind_param(stmt, 2, model_type) + +try: + ibm_db.execute(stmt) + assert ibm_db.fetch_row(stmt) != False, "entitlement_type and model_type combination does not exist in database." + entitlement_id = int(ibm_db.result(stmt, "ID")) + print(f"Fetched entitlement id : {entitlement_id}") +except Exception as e: + print(ibm_db.stmt_errormsg(stmt)) + print(f"Couldn't retrieve entitlement type from database. Exception : {e}") + + + + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - Retrirving tenant id using tenant_name +# - +# -------------------------------------------------------------------------------------------------------------------------------- + + +#tenant_string = f"SELECT ID FROM {schema}.AIBROKER_TENANT WHERE NAME = ?;" +tenant_string = f"SELECT ID FROM \"{schema_uppercase}\".AIBROKER_TENANT WHERE NAME = ?;" +stmt = ibm_db.prepare(conn, tenant_string) +ibm_db.bind_param(stmt, 1, tenant_name) + + +try: + ibm_db.execute(stmt) + assert ibm_db.fetch_row(stmt) != False, "tenant_name does not exist in database."+tenant_name + tenant_id = int(ibm_db.result(stmt, "ID")) + print(f"Fetched tenant id : {tenant_id}") +except Exception as e: + print(ibm_db.stmt_errormsg(stmt)) + print(f"Couldn't retrieve tenant id from database. Exception : {e}") + + +# -------------------------------------------------------------------------------------------------------------------------------- +# - +# - insert into AIBROKER_TENANT_ENTITLEMENT +# - +# -------------------------------------------------------------------------------------------------------------------------------- + + +#statement_tenant_entitlement = f"INSERT INTO {schema}.AIBROKER_TENANT_ENTITLEMENT(TENANT_ID, ENTITLEMENT_ID, START_DATE, EXPIRE_DATE) VALUES(?,?,?,?);" +statement_tenant_entitlement = f"INSERT INTO \"{schema_uppercase}\".AIBROKER_TENANT_ENTITLEMENT(TENANT_ID, ENTITLEMENT_ID, START_DATE, EXPIRE_DATE) VALUES(?,?,?,?);" + +stmt = ibm_db.prepare(conn, statement_tenant_entitlement) +ibm_db.bind_param(stmt, 1, tenant_id) +ibm_db.bind_param(stmt, 2, entitlement_id) +ibm_db.bind_param(stmt, 3, entilement_start_date ) +ibm_db.bind_param(stmt, 4, entilement_end_date) + +try: + stmt = ibm_db.execute(stmt) +except Exception as e: + print("Transaction couldn't be completed. The relationship of tenant and entitlement has not been created") + ibm_db.rollback(conn) +else: + ibm_db.commit(conn) + print("Transaction complete. The relationship of tenant and entitlement has been created.") diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_wx.sh b/ibm/mas_devops/roles/aibroker_app_config/files/create_wx.sh new file mode 100755 index 0000000000..6edba73990 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/create_wx.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# echo "Usage ./create_apikey.sh tenant_name_in_lower_case" + +TENANT=$1 +AIBROKER=$2 +WATSONXAI_APIKEY=$3 +WATSONXAI_URL=$4 +WATSONXAI_PROJECT_ID=$5 + +# if [ -z ${TENANT} ]; then +# #echo "❌ Missing tenant name" +# echo "using default tenant name=aibrokeruser" +# TENANT='aibrokeruser' +# #exit 1 +# fi + +# if [[ "$TENANT" =~ [[:upper:]] ]]; then +# echo "❌ uppercase character found" +# exit 1 +# fi + +# echo -n "Enter watsonxai apikey > " +# read watsonxaiapikey +# echo "You entered: $watsonxaiapikey" + +# echo -n "Enter watsonxai url > " +# read watsonxai_url +# echo "You entered: $watsonxai_url" + +# echo -n "Enter watsonxai project id > " +# read watsonxai_project_id +# echo "You entered: $watsonxai_project_id" + +# echo "TENANT = ${TENANT}" +# echo "watsonxaiapikey =${watsonxaiapikey}" +# echo "watsonxai_url =${watsonxai_url}" +# echo "watsonxai_project_id =${watsonxai_project_id}" + +# if [ -z ${TENANT} ]; then +# #echo "❌ Missing tenant name" +# echo "using default tenant name=aibrokeruser" +# TENANT='aibrokeruser' +# #exit 1 +# fi + +# if [ -z ${watsonxaiapikey} ]; then +# echo "❌ Missing watsonxai apikey" +# exit 1 +# fi + +# if [ -z ${watsonxai_project_id} ]; then +# echo "❌ Missing watsonxai project id" +# exit 1 +# fi + +# Add testing code to make sure the watsonx apikey works +# python3 ../roles/aibroker/files/access_wx.py $WATSONXAI_APIKEY $WATSONXAI_URL $WATSONXAI_PROJECT_ID +if [ $? -eq 0 ]; then + echo "Successfully executed script" + echo "Creating watsonx secret in k8s" + oc create secret generic ${TENANT}----wx-secret -n ${AIBROKER} \ + --from-literal=wx_apikey=${WATSONXAI_APIKEY} \ + --from-literal=wx_url=${WATSONXAI_URL} \ + --from-literal=wx_project_id=${WATSONXAI_PROJECT_ID} + echo "Created watsonx secret successfully in k8s." +else + # Redirect stdout from echo command to stderr. + echo "Script exited with error. The secret for the watsonx is not created in k8s. Please fix the error and rerun the script." >&2 +fi diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/delete_s3.py b/ibm/mas_devops/roles/aibroker_app_config/files/delete_s3.py new file mode 100644 index 0000000000..135214568c --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/delete_s3.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + + +# create a new file, ~/.aws/credentials: +# [default] +# aws_access_key_id = YOUR_ACCESS_KEY_ID +# aws_secret_access_key = YOUR_SECRET_ACCESS_KEY + +# Create a new file, ~/.aws/config: +# [default] +# region = YOUR_PREFERRED_REGION + +import boto3 +import sys + +s3_resource = boto3.resource('s3') + +if len(sys.argv) != 2: + print("Usage: python delete_bucket.py your-bucket-name") + sys.exit(1) + +if len(sys.argv) == 2 and sys.argv[1] is not None: + # Need to read from k8s + YOUR_BUCKET_NAME = sys.argv[1] +else: + sys.exit(1) + +print('will delete the bucket name='+YOUR_BUCKET_NAME) +try: + print("deleting") + s3_resource = boto3.resource('s3') + bucket = s3_resource.Bucket(YOUR_BUCKET_NAME) + count = 0 + for key in bucket.objects.all(): + count += 1 + print(key.key) + print("Number of objects in bucket=", count) + bucket.object_versions.delete() + bucket.delete() + sys.exit(0) +except Exception as e: + print("Not granted") + print(e) + sys.exit(1) diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/generate_api_key.py b/ibm/mas_devops/roles/aibroker_app_config/files/generate_api_key.py new file mode 100755 index 0000000000..b196049d6a --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/files/generate_api_key.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# Licensed Materials - Property of IBM +# 5737-M66, 5900-AAA +# (C) Copyright IBM Corp. 2024 All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication, or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. + +import secrets +import string +import random + + +def generate_api_key(): + alphabet = string.ascii_letters + string.digits + return "".join(secrets.choice(alphabet) for i in range(32)) + + +if __name__ == "__main__": + print(generate_api_key()) diff --git a/ibm/mas_devops/roles/aibroker_app_config/meta/main.yml b/ibm/mas_devops/roles/aibroker_app_config/meta/main.yml new file mode 100644 index 0000000000..65ba5e8570 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/meta/main.yml @@ -0,0 +1,21 @@ +galaxy_info: + author: Karol Czarnecki (@karol-czarnecki) + description: Deploy IBM AiBroker + company: IBM + + license: EPL-2.0 + + min_ansible_version: 2.10 + + platforms: + - name: GenericLinux + versions: + - all + + galaxy_tags: + - ibm + - mas + - devops + +dependencies: + - role: ibm.mas_devops.ansible_version_check diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml new file mode 100644 index 0000000000..67d7af15ec --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml @@ -0,0 +1,20 @@ +--- +- name: "Read sls config file" + ansible.builtin.set_fact: + slscfg: "{{ lookup('file', '{{ mas_config_dir }}/sls.yml') | from_yaml_all }}" + +- name: "Set sls mandatory vars" + ansible.builtin.set_fact: + slscfg: + sls_secret_name: "{{ slscfg[0].metadata.name }}" + sls_registration_key: "{{ slscfg[0].stringData.registrationKey }}" + sls_url: "{{ slscfg[1].spec.config.url }}" + sls_cert_ca: "{{ slscfg[1].spec.certificates }}" + +- name: "Debug: sls information" #TODO: remove before PR + debug: + msg: + - "sls secret name ......... {{ slscfg.sls_secret_name }}" + - "sls registration key .... {{ slscfg.sls_registration_key }}" + - "sls registration key .... {{ slscfg.sls_url }}" + - "sls cert ca ............. {{ slscfg.sls_cert_ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml new file mode 100644 index 0000000000..d102e47fe4 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Load variables (main) + include_vars: "vars/main.yml" + +# Create config for SLS +- include_tasks: tasks/config_sls/main.yml diff --git a/ibm/mas_devops/roles/aibroker_app_config/vars/main.yml b/ibm/mas_devops/roles/aibroker_app_config/vars/main.yml new file mode 100644 index 0000000000..29d86e90d3 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/vars/main.yml @@ -0,0 +1,50 @@ +--- +# aibroker vars +aibroker_namespace: 'mas-{{ mas_instance_id }}-aibroker' +mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" +aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" +pullSecretName: 'ibm-entitlement' + +# Whether to perform an install or remove tenant +tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" + +# Create Api Key vars +mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" +tenantNamespace: 'aibroker-{{ mas_aibroker_tenant_name }}' +# app_domain: "{{ lookup('env', 'APP_DOMAIN') }}" +mas_aibroker_apikey_action: "{{ lookup('env', 'MAS_AIBROKER_APIKEY_ACTION') | default('install', true) }}" + +# Main variables +ibm_entitlement_username: "{{ lookup('env','IBM_ENTITLEMENT_USERNAME') }}" +ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" + +# Source container registry +# ----------------------------------------------------------------------------- +mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" + +# WatsonX +mas_aibroker_watsonxai_apikey: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY') }}" +mas_aibroker_watsonxai_url: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') }}" +mas_aibroker_watsonxai_project_id: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') }}" +mas_aibroker_watsonx_action: "{{ lookup('env', 'MAS_AIBROKER_WATSONX_ACTION') | default('install', true) }}" + +# S3 +mas_aibroker_storage_host: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_HOST') | default('', true) }}" +mas_aibroker_storage_accesskey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') | default('', true) }}" +mas_aibroker_storage_secretkey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') | default('', true) }}" +mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | default('', true) }}" +mas_aibroker_s3_action: "{{ lookup('env', 'MAS_AIBROKER_S3_ACTION') | default('install', true) }}" +mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" +mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" + +# SAAS +mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" +mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" +mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" +mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" +mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" +mas_aibroker_path_ca_crt: './certs' +mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" From e7a2e4141c1a1c865edeada8dcb7f5356fb75976 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 21 Jan 2025 10:57:09 +0000 Subject: [PATCH 087/169] [patch] added get config tasks for dro/sls/db2 --- .../tasks/config_db2/main.yml | 20 +++++++++++++++++++ .../tasks/config_dro/main.yml | 20 +++++++++++++++++++ .../tasks/config_sls/main.yml | 2 +- .../roles/aibroker_app_config/tasks/main.yml | 6 ++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml new file mode 100644 index 0000000000..cdf00294f5 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml @@ -0,0 +1,20 @@ +--- +- name: "Read db2 config file" + ansible.builtin.set_fact: + db2cfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_instance_id }}-db2u.yml') | from_yaml_all }}" + +- name: "Set dro mandatory vars" + ansible.builtin.set_fact: + db2cfg: + db2_username: "{{ db2cfg[0].data.username }}" + db2_password: "{{ db2cfg[0].data.password }}" + db2_url: "{{ db2cfg[1].spec.config.url }}" + db2_cert_ca: "{{ db2cfg[1].spec.certificates }}" + +- name: "Debug: DB2 information" #TODO: remove before PR + debug: + msg: + - "DB2 username ............. {{ db2cfg.db2_username }}" + - "DB2 password ............. {{ db2cfg.db2_password }}" + - "DB2 url .................. {{ db2cfg.db2_url }}" + - "DB2 cert ca .............. {{ db2cfg.db2_cert_ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml new file mode 100644 index 0000000000..f8be05f622 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml @@ -0,0 +1,20 @@ +--- +- name: "Read dro config file" + ansible.builtin.set_fact: + drocfg: "{{ lookup('file', '{{ mas_config_dir }}/dro.yml') | from_yaml_all }}" + +- name: "Set dro mandatory vars" + ansible.builtin.set_fact: + drocfg: + dro_secret_name: "{{ drocfg[0].metadata.name }}" + dro_api_key: "{{ drocfg[0].stringData.api_key }}" + dro_url: "{{ drocfg[1].spec.config.url }}" + dro_cert_ca: "{{ drocfg[1].spec.certificates }}" + +- name: "Debug: dro information" #TODO: remove before PR + debug: + msg: + - "dro secret name ......... {{ drocfg.dro_secret_name }}" + - "dro api key ............. {{ drocfg.dro_api_key }}" + - "dro url ................. {{ drocfg.dro_url }}" + - "dro cert ca ............. {{ drocfg.dro_cert_ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml index 67d7af15ec..537edb00b3 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml @@ -16,5 +16,5 @@ msg: - "sls secret name ......... {{ slscfg.sls_secret_name }}" - "sls registration key .... {{ slscfg.sls_registration_key }}" - - "sls registration key .... {{ slscfg.sls_url }}" + - "sls url ................. {{ slscfg.sls_url }}" - "sls cert ca ............. {{ slscfg.sls_cert_ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml index d102e47fe4..02953b789b 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml @@ -4,3 +4,9 @@ # Create config for SLS - include_tasks: tasks/config_sls/main.yml + +# Create config for DRO +- include_tasks: tasks/config_dro/main.yml + +# Create config for DRO +- include_tasks: tasks/config_db2/main.yml From b1a3d86fa78e684901364f38743c6ff5297552f7 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 21 Jan 2025 11:39:37 +0000 Subject: [PATCH 088/169] [patch] update db2 name --- ibm/mas_devops/roles/aibroker/defaults/main.yml | 1 - .../roles/aibroker_app_config/tasks/config_db2/main.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/defaults/main.yml b/ibm/mas_devops/roles/aibroker/defaults/main.yml index 760a9ec042..3b14fcde35 100644 --- a/ibm/mas_devops/roles/aibroker/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker/defaults/main.yml @@ -2,7 +2,6 @@ mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" - # Custom Labels # ----------------------------------------------------------------------------- custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string | ibm.mas_devops.string2dict() }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml index cdf00294f5..e2b03eaf09 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml @@ -1,7 +1,7 @@ --- - name: "Read db2 config file" ansible.builtin.set_fact: - db2cfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_instance_id }}-db2u.yml') | from_yaml_all }}" + db2cfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_app_id }}-db2u.yml') | from_yaml_all }}" - name: "Set dro mandatory vars" ansible.builtin.set_fact: From 4a16c5eaa872ec266c087a52aca0806934c13e75 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 21 Jan 2025 11:59:58 +0000 Subject: [PATCH 089/169] [patch] add b64decode for db2 --- .../roles/aibroker_app_config/tasks/config_db2/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml index e2b03eaf09..d4bf1b37e7 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml @@ -6,8 +6,8 @@ - name: "Set dro mandatory vars" ansible.builtin.set_fact: db2cfg: - db2_username: "{{ db2cfg[0].data.username }}" - db2_password: "{{ db2cfg[0].data.password }}" + db2_username: "{{ db2cfg[0].data.username | b64decode}}" + db2_password: "{{ db2cfg[0].data.password | b64decode}}" db2_url: "{{ db2cfg[1].spec.config.url }}" db2_cert_ca: "{{ db2cfg[1].spec.certificates }}" From 3976347399bb995c89682e80b28e2520cb1a2aa7 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 21 Jan 2025 12:01:50 +0000 Subject: [PATCH 090/169] [patch] update playbook --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 412c9d083e..3392e792c1 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -76,4 +76,4 @@ # - ibm.mas_devops.odh # - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker - # - ibm.mas_devops.aibroker_app_config + - ibm.mas_devops.aibroker_app_config From d8e57e2e836295437099fbffd533b3c40a60a6b6 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 21 Jan 2025 12:39:56 +0000 Subject: [PATCH 091/169] [patch] update doc for current state of disable MAS from Airoker --- docs/playbooks/oneclick-aibroker.md | 151 +++++++++------------------- 1 file changed, 45 insertions(+), 106 deletions(-) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index fb161be730..1f7804e864 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -2,26 +2,28 @@ ## Prerequisites -You will need a RedHat OpenShift v4.14 or above with IBM Maximo Application Suite Core v9.x already be installed, the [oneclick-core](oneclick-core.md) playbook can be used to set this up. +You will need a RedHat OpenShift v4.14 or above. ### Dependencies: +* IBM Suite License Service installed on OCP cluster or external instance +* IBM Data Reporter Operator installed on OCP cluster or external instance * Object Storage + Minio (installed on the same cluster what aibroker) + AWS S3 (if customer use AWS S3 bucket bucket) buckets needs to have unique names * MariaDB database (installed in cluster where aibroker instance) or in AWS -* IBM Maximo Application Suite Core v9.x ## Overview -This playbook will add **AI Broker v1.0.x** to an existing IBM Maximo Application Suite Core installation. +This playbook will add **AI Broker v9.1.x** to OCP cluster. This playbook can be ran against any OCP cluster regardless of its type; whether it's running in IBM Cloud, Azure, AWS, or your local datacenter. * Install dependencies: - + Install IBM Maximo Application Suite Core v9.x (~30 Minutes) - + Install MariaDB (~5 minutes) - + Install Minio (~5 minutes) + + IBM Suite License Service (~10 Minutes) **optional** + + IBM Data Reporter Operator (~10 Minutes) **optional** + + Install MariaDB (~5 minutes) **optional** + + Install Minio (~5 minutes) **optional** * Install AI broker application (using playbook): + Install application (~10 Minutes) + Configure AI Broker (kmodels, tenant, etc) (~5 Minutes) @@ -98,117 +100,55 @@ source /tmp/venv/bin/activate python3 -m pip install boto3 ``` -#### Run playbooks for deploy AI Broker from internal registry ex. `docker-na-public.artifactory.swg-devops.com` - -```bash -export MAS_CATALOG_VERSION="" -export ARTIFACTORY_USERNAME="" -export ARTIFACTORY_TOKEN="" -export MAS_ICR_CP="" -export MAS_ICR_CPOPEN="" -export MAS_INSTANCE_ID="" -export MAS_APP_CHANNEL="9.0.x" -export MAS_AIBROKER_STORAGE_ACCESSKEY="" -export MAS_AIBROKER_STORAGE_SECRETKEY="" -export MAS_AIBROKER_STORAGE_HOST="" -export MAS_AIBROKER_STORAGE_SSL="true or false" -export MAS_AIBROKER_STORAGE_REGION="" -export MAS_AIBROKER_STORAGE_PROVIDER="" -export MAS_AIBROKER_STORAGE_PORT="" -export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" -export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" -export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" -export MAS_AIBROKER_WATSONXAI_APIKEY="" -export MAS_AIBROKER_WATSONXAI_URL="" -export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" -export MAS_AIBROKER_DB_HOST="" -export MAS_AIBROKER_DB_PORT="" -export MAS_AIBROKER_DB_USER="" -export MAS_AIBROKER_DB_DATABASE="" -export MAS_AIBROKER_DB_SECRET_NAME="" -export MAS_AIBROKER_DB_SECRET_VALUE="" - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook playbooks/oneclick_add_aibroker.yml -``` - -#### Run playbooks for deploy AI Broker from public registry ex. `icr.io` - -```bash -export MAS_CATALOG_VERSION="" -export MAS_ENTITLEMENT_USERNAME="" -export MAS_ENTITLEMENT_KEY="" -export MAS_INSTANCE_ID="" -export MAS_APP_CHANNEL="9.0.x" -export MAS_AIBROKER_STORAGE_ACCESSKEY="" -export MAS_AIBROKER_STORAGE_SECRETKEY="" -export MAS_AIBROKER_STORAGE_HOST="" -export MAS_AIBROKER_STORAGE_SSL="true or false" -export MAS_AIBROKER_STORAGE_REGION="" -export MAS_AIBROKER_STORAGE_PROVIDER="" -export MAS_AIBROKER_STORAGE_PORT="" -export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" -export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" -export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" -export MAS_AIBROKER_WATSONXAI_APIKEY="" -export MAS_AIBROKER_WATSONXAI_URL="" -export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" -export MAS_AIBROKER_DB_HOST="" -export MAS_AIBROKER_DB_PORT="" -export MAS_AIBROKER_DB_USER="" -export MAS_AIBROKER_DB_DATABASE="" -export MAS_AIBROKER_DB_SECRET_NAME="" -export MAS_AIBROKER_DB_SECRET_VALUE="" - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook playbooks/oneclick_add_aibroker.yml -``` - #### Run playbooks for deploy AI Broker on SAAS ```bash -export MAS_CATALOG_VERSION="v9-master-amd64" +export ARTIFACTORY_USERNAME="" +export ARTIFACTORY_TOKEN="" +export MAS_ICR_CP="docker-na-public.artifactory.swg-devops.com/wiotp-docker-local" +export MAS_ICR_CPOPEN="docker-na-public.artifactory.swg-devops.com/wiotp-docker-local/cpopen" +export MAS_CATALOG_VERSION="v9-xxx-amd64" export MAS_ENTITLEMENT_USERNAME="" export MAS_ENTITLEMENT_KEY="" export IBM_ENTITLEMENT_KEY=${MAS_ENTITLEMENT_KEY} export MAS_INSTANCE_ID="" export MAS_APP_CHANNEL="9.1.x" export MAS_CONFIG_DIR="config_path_location" -export MAS_AIBROKER_SAAS="true" -export MAS_AIBROKER_DOMAIN="apps.domain" -export MAS_AIBROKER_SLS_URL="https://sls.ibm-sls.ibm-sls."${MAS_AIBROKER_DOMAIN} -export MAS_AIBROKER_SLS_REGISTRATION_KEY="xxxxxxx" -export MAS_AIBROKER_DRO_URL="https://ibm-data-reporter-redhat-marketplace."${MAS_AIBROKER_DOMAIN} -export MAS_AIBROKER_DRO_TOKEN="xxx" -export DB2_INSTANCE_NAME="aibroker" -export MAS_AIBROKER_STORAGE_ACCESSKEY="" -export MAS_AIBROKER_STORAGE_SECRETKEY="" -export MAS_AIBROKER_STORAGE_HOST="" -export MAS_AIBROKER_STORAGE_SSL="true or false" -export MAS_AIBROKER_STORAGE_REGION="" -export MAS_AIBROKER_STORAGE_PROVIDER="" -export MAS_AIBROKER_STORAGE_PORT="" -export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" -export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" -export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" -export MAS_AIBROKER_WATSONXAI_APIKEY="" -export MAS_AIBROKER_WATSONXAI_URL="" -export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" -export MAS_AIBROKER_DB_HOST="" -export MAS_AIBROKER_DB_PORT="" -export MAS_AIBROKER_DB_USER="" -export MAS_AIBROKER_DB_DATABASE="" -export MAS_AIBROKER_DB_SECRET_NAME="" -export MAS_AIBROKER_DB_SECRET_VALUE="" +export IBM_ENTITLEMENT_KEY=${MAS_ENTITLEMENT_KEY} +# export MAS_AIBROKER_SAAS="true" +# export MAS_AIBROKER_DOMAIN="apps.domain" +# export MAS_AIBROKER_SLS_URL="https://sls.ibm-sls.ibm-sls."${MAS_AIBROKER_DOMAIN} +# export MAS_AIBROKER_SLS_REGISTRATION_KEY="xxxxxxx" +# export MAS_AIBROKER_DRO_URL="https://ibm-data-reporter-redhat-marketplace."${MAS_AIBROKER_DOMAIN} +# export MAS_AIBROKER_DRO_TOKEN="xxx" +# export DB2_INSTANCE_NAME="aibroker" +# export MAS_AIBROKER_STORAGE_ACCESSKEY="" +# export MAS_AIBROKER_STORAGE_SECRETKEY="" +# export MAS_AIBROKER_STORAGE_HOST="" +# export MAS_AIBROKER_STORAGE_SSL="true or false" +# export MAS_AIBROKER_STORAGE_REGION="" +# export MAS_AIBROKER_STORAGE_PROVIDER="" +# export MAS_AIBROKER_STORAGE_PORT="" +# export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" +# export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" +# export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" +# export MAS_AIBROKER_WATSONXAI_APIKEY="" +# export MAS_AIBROKER_WATSONXAI_URL="" +# export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" +# export MAS_AIBROKER_DB_HOST="" +# export MAS_AIBROKER_DB_PORT="" +# export MAS_AIBROKER_DB_USER="" +# export MAS_AIBROKER_DB_DATABASE="" +# export MAS_AIBROKER_DB_SECRET_NAME="" +# export MAS_AIBROKER_DB_SECRET_VALUE="" oc login --token=xxxx --server=https://myocpserver ansible-playbook playbooks/oneclick_add_aibroker.yml ``` -- `MAS_AIBROKER_SLS_REGISTRATION_KEY` - value can be found in `ibm-sls` namespace, in pod `sls-api-licensing-85699fb57-9lmrq` please look in environments tab, then value `REGISTRATION_KEY` -- `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` -- in `AWS` for `MAS_AIBROKER_STORAGE_PIPELINES_BUCKET`, `MAS_AIBROKER_STORAGE_TENANTS_BUCKET`, `MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET` user need to create S3 buckets with unique name - +* `MAS_AIBROKER_SLS_REGISTRATION_KEY` - value can be found in `ibm-sls` namespace, in pod `sls-api-licensing-85699fb57-9lmrq` please look in environments tab, then value `REGISTRATION_KEY` +* `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` +* in `AWS` for `MAS_AIBROKER_STORAGE_PIPELINES_BUCKET`, `MAS_AIBROKER_STORAGE_TENANTS_BUCKET`, `MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET` user need to create S3 buckets with unique name ## NOTICE: playbook oneclick_add_aibroker.yml will run three roles: @@ -370,8 +310,7 @@ export ROLE_NAME="aibroker" ansible-playbook playbooks/run_role.yml ``` -- `MAS_AIBROKER_SLS_REGISTRATION_KEY` - value can be found in `ibm-sls` namespace, in pod `sls-api-licensing-85699fb57-9lmrq` please look in environments tab, then value `REGISTRATION_KEY` -- `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` - +* `MAS_AIBROKER_SLS_REGISTRATION_KEY` - value can be found in `ibm-sls` namespace, in pod `sls-api-licensing-85699fb57-9lmrq` please look in environments tab, then value `REGISTRATION_KEY` +* `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` **NOTE:** for create addidional tenants we don't need to specify buckets From 39c73a46e289684089621336d0fb75fb5f9388c2 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 21 Jan 2025 16:48:30 +0000 Subject: [PATCH 092/169] [patch] update vars --- .../tasks/config_db2/main.yml | 24 +++++++++---------- .../tasks/config_dro/main.yml | 8 +++---- .../tasks/config_sls/main.yml | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml index d4bf1b37e7..a0ab46b48d 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml @@ -1,20 +1,20 @@ --- -- name: "Read db2 config file" +- name: "Read jdbc config file" ansible.builtin.set_fact: - db2cfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_app_id }}-db2u.yml') | from_yaml_all }}" + jdbccfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_app_id }}-db2u.yml') | from_yaml_all }}" - name: "Set dro mandatory vars" ansible.builtin.set_fact: - db2cfg: - db2_username: "{{ db2cfg[0].data.username | b64decode}}" - db2_password: "{{ db2cfg[0].data.password | b64decode}}" - db2_url: "{{ db2cfg[1].spec.config.url }}" - db2_cert_ca: "{{ db2cfg[1].spec.certificates }}" + jdbccfg: + jdbc_username: "{{ jdbccfg[0].data.username | b64decode}}" + jdbc_password: "{{ jdbccfg[0].data.password | b64decode}}" + jdbc_url: "{{ jdbccfg[1].spec.config.url }}" + jdbc_cert_ca: "{{ jdbccfg[1].spec.certificates }}" -- name: "Debug: DB2 information" #TODO: remove before PR +- name: "Debug: jdbc information" #TODO: remove before PR debug: msg: - - "DB2 username ............. {{ db2cfg.db2_username }}" - - "DB2 password ............. {{ db2cfg.db2_password }}" - - "DB2 url .................. {{ db2cfg.db2_url }}" - - "DB2 cert ca .............. {{ db2cfg.db2_cert_ca }}" + - "JDBC username ............. {{ jdbccfg.jdbc_username }}" + - "JDBC password ............. {{ jdbccfg.jdbc_password }}" + - "JDBC url .................. {{ jdbccfg.jdbc_url }}" + - "JDBC cert ca .............. {{ jdbccfg.jdbc_cert_ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml index f8be05f622..b78ee78ef1 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml @@ -7,14 +7,14 @@ ansible.builtin.set_fact: drocfg: dro_secret_name: "{{ drocfg[0].metadata.name }}" - dro_api_key: "{{ drocfg[0].stringData.api_key }}" + dro_registration_key: "{{ drocfg[0].stringData.api_key }}" dro_url: "{{ drocfg[1].spec.config.url }}" - dro_cert_ca: "{{ drocfg[1].spec.certificates }}" + dro_ca: "{{ drocfg[1].spec.certificates }}" - name: "Debug: dro information" #TODO: remove before PR debug: msg: - "dro secret name ......... {{ drocfg.dro_secret_name }}" - - "dro api key ............. {{ drocfg.dro_api_key }}" + - "dro api key ............. {{ drocfg.dro_registration_key }}" - "dro url ................. {{ drocfg.dro_url }}" - - "dro cert ca ............. {{ drocfg.dro_cert_ca }}" + - "dro cert ca ............. {{ drocfg.dro_ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml index 537edb00b3..6800d00c4e 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml @@ -9,7 +9,7 @@ sls_secret_name: "{{ slscfg[0].metadata.name }}" sls_registration_key: "{{ slscfg[0].stringData.registrationKey }}" sls_url: "{{ slscfg[1].spec.config.url }}" - sls_cert_ca: "{{ slscfg[1].spec.certificates }}" + sls_ca: "{{ slscfg[1].spec.certificates }}" - name: "Debug: sls information" #TODO: remove before PR debug: @@ -17,4 +17,4 @@ - "sls secret name ......... {{ slscfg.sls_secret_name }}" - "sls registration key .... {{ slscfg.sls_registration_key }}" - "sls url ................. {{ slscfg.sls_url }}" - - "sls cert ca ............. {{ slscfg.sls_cert_ca }}" + - "sls cert ca ............. {{ slscfg.sls_ca }}" From b1f72838bbb07219a343c1503f2c7a2dc5a9de86 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 21 Jan 2025 17:16:31 +0000 Subject: [PATCH 093/169] [patch] update aibroker CR --- .../templates/aibroker/aibrokerapp.yml.j2 | 16 ++++++++++++++-- .../tasks/config_db2/main.yml | 18 ++++++++++-------- .../tasks/config_dro/main.yml | 16 ++++++++-------- .../tasks/config_sls/main.yml | 16 ++++++++-------- 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index 9a18da45fe..32c472b2de 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -8,8 +8,20 @@ metadata: mas.ibm.com/applicationId: aibroker mas.ibm.com/instanceId: "{{ mas_instance_id }}" spec: - bindings: {} settings: icr: cp: "{{ mas_icr_cp }}" - cpopen: "{{ mas_icr_cpopen }}" \ No newline at end of file + cpopen: "{{ mas_icr_cpopen }}" + sls: + url: "{{ slscfg.url }}" + registration_key: "{{ slscfg.registration_key }}" + ca: "{{ slscfg.ca }}" + dro: + url: "{{ drocfg.url }}" + registration_key: "{{ drocfg.registration_key }}" + ca: "{{ drocfg.ca }}" + jdbc: + url: "{{ jdbccfg.url }}" + username: "{{ jdbccfg.username }}" + password: "{{ jdbccfg.password }}" + ca: "{{ jdbccfg.ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml index a0ab46b48d..28876edafc 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml @@ -6,15 +6,17 @@ - name: "Set dro mandatory vars" ansible.builtin.set_fact: jdbccfg: - jdbc_username: "{{ jdbccfg[0].data.username | b64decode}}" - jdbc_password: "{{ jdbccfg[0].data.password | b64decode}}" - jdbc_url: "{{ jdbccfg[1].spec.config.url }}" - jdbc_cert_ca: "{{ jdbccfg[1].spec.certificates }}" + # username: "{{ jdbccfg[0].data.username | b64decode}}" + # password: "{{ jdbccfg[0].data.password | b64decode}}" + username: "{{ jdbccfg[0].data.username }}" + password: "{{ jdbccfg[0].data.password }}" + url: "{{ jdbccfg[1].spec.config.url }}" + ca: "{{ jdbccfg[1].spec.certificates }}" - name: "Debug: jdbc information" #TODO: remove before PR debug: msg: - - "JDBC username ............. {{ jdbccfg.jdbc_username }}" - - "JDBC password ............. {{ jdbccfg.jdbc_password }}" - - "JDBC url .................. {{ jdbccfg.jdbc_url }}" - - "JDBC cert ca .............. {{ jdbccfg.jdbc_cert_ca }}" + - "JDBC username ............. {{ jdbccfg.username }}" + - "JDBC password ............. {{ jdbccfg.password }}" + - "JDBC url .................. {{ jdbccfg.url }}" + - "JDBC cert ca .............. {{ jdbccfg.ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml index b78ee78ef1..52608b1010 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml @@ -6,15 +6,15 @@ - name: "Set dro mandatory vars" ansible.builtin.set_fact: drocfg: - dro_secret_name: "{{ drocfg[0].metadata.name }}" - dro_registration_key: "{{ drocfg[0].stringData.api_key }}" - dro_url: "{{ drocfg[1].spec.config.url }}" - dro_ca: "{{ drocfg[1].spec.certificates }}" + secret_name: "{{ drocfg[0].metadata.name }}" + registration_key: "{{ drocfg[0].stringData.api_key }}" + url: "{{ drocfg[1].spec.config.url }}" + ca: "{{ drocfg[1].spec.certificates }}" - name: "Debug: dro information" #TODO: remove before PR debug: msg: - - "dro secret name ......... {{ drocfg.dro_secret_name }}" - - "dro api key ............. {{ drocfg.dro_registration_key }}" - - "dro url ................. {{ drocfg.dro_url }}" - - "dro cert ca ............. {{ drocfg.dro_ca }}" + - "dro secret name ......... {{ drocfg.secret_name }}" + - "dro api key ............. {{ drocfg.registration_key }}" + - "dro url ................. {{ drocfg.url }}" + - "dro cert ca ............. {{ drocfg.ca }}" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml index 6800d00c4e..f260373798 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml @@ -6,15 +6,15 @@ - name: "Set sls mandatory vars" ansible.builtin.set_fact: slscfg: - sls_secret_name: "{{ slscfg[0].metadata.name }}" - sls_registration_key: "{{ slscfg[0].stringData.registrationKey }}" - sls_url: "{{ slscfg[1].spec.config.url }}" - sls_ca: "{{ slscfg[1].spec.certificates }}" + secret_name: "{{ slscfg[0].metadata.name }}" + registration_key: "{{ slscfg[0].stringData.registrationKey }}" + url: "{{ slscfg[1].spec.config.url }}" + ca: "{{ slscfg[1].spec.certificates }}" - name: "Debug: sls information" #TODO: remove before PR debug: msg: - - "sls secret name ......... {{ slscfg.sls_secret_name }}" - - "sls registration key .... {{ slscfg.sls_registration_key }}" - - "sls url ................. {{ slscfg.sls_url }}" - - "sls cert ca ............. {{ slscfg.sls_ca }}" + - "sls secret name ......... {{ slscfg.secret_name }}" + - "sls registration key .... {{ slscfg.registration_key }}" + - "sls url ................. {{ slscfg.url }}" + - "sls cert ca ............. {{ slscfg.ca }}" From f1a1702e98bd85eb7afe1314e7a8e08d7431336e Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 22 Jan 2025 10:47:01 +0000 Subject: [PATCH 094/169] [patch] add update CR task --- .../roles/aibroker_app_config/tasks/main.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml index 02953b789b..fa520c0fec 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml @@ -10,3 +10,29 @@ # Create config for DRO - include_tasks: tasks/config_db2/main.yml + +# Update CR +- name: Update AibrokerApp CR + kubernetes.core.k8s: + state: patched + api_version: apps.mas.ibm.com/v1 + kind: AiBrokerApp + name: "{{ mas_instance_id }}" + namespace: "{{ aibroker_namespace }}" + definition: + spec: + settings: + sls: + url: "{{ slscfg.url }}" + registration_key: "{{ slscfg.registration_key }}" + ca: "{{ slscfg.ca }}" + dro: + url: "{{ drocfg.url }}" + registration_key: "{{ drocfg.registration_key }}" + ca: "{{ drocfg.ca }}" + jdbc: + url: "{{ jdbccfg.url }}" + username: "{{ jdbccfg.username }}" + password: "{{ jdbccfg.password }}" + ca: "{{ jdbccfg.ca }}" + ignore_errors: yes From 4c08ff2042759321489e55cc9aa74833baed8f63 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 24 Jan 2025 11:08:44 +0000 Subject: [PATCH 095/169] [patch] remove temp update CR --- .../roles/aibroker_app_config/tasks/main.yml | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml index fa520c0fec..af90664b8e 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml @@ -12,27 +12,27 @@ - include_tasks: tasks/config_db2/main.yml # Update CR -- name: Update AibrokerApp CR - kubernetes.core.k8s: - state: patched - api_version: apps.mas.ibm.com/v1 - kind: AiBrokerApp - name: "{{ mas_instance_id }}" - namespace: "{{ aibroker_namespace }}" - definition: - spec: - settings: - sls: - url: "{{ slscfg.url }}" - registration_key: "{{ slscfg.registration_key }}" - ca: "{{ slscfg.ca }}" - dro: - url: "{{ drocfg.url }}" - registration_key: "{{ drocfg.registration_key }}" - ca: "{{ drocfg.ca }}" - jdbc: - url: "{{ jdbccfg.url }}" - username: "{{ jdbccfg.username }}" - password: "{{ jdbccfg.password }}" - ca: "{{ jdbccfg.ca }}" - ignore_errors: yes +# - name: Update AibrokerApp CR +# kubernetes.core.k8s: +# state: patched +# api_version: apps.mas.ibm.com/v1 +# kind: AiBrokerApp +# name: "{{ mas_instance_id }}" +# namespace: "{{ aibroker_namespace }}" +# definition: +# spec: +# settings: +# sls: +# url: "{{ slscfg.url }}" +# registration_key: "{{ slscfg.registration_key }}" +# ca: "{{ slscfg.ca }}" +# dro: +# url: "{{ drocfg.url }}" +# registration_key: "{{ drocfg.registration_key }}" +# ca: "{{ drocfg.ca }}" +# jdbc: +# url: "{{ jdbccfg.url }}" +# username: "{{ jdbccfg.username }}" +# password: "{{ jdbccfg.password }}" +# ca: "{{ jdbccfg.ca }}" +# ignore_errors: yes From 0838c0d8a9fe45eef00bdb19cc51dbfad21aa0fb Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 24 Jan 2025 14:14:41 +0000 Subject: [PATCH 096/169] [patch] add mongo role temp --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 3392e792c1..23af90a666 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -68,6 +68,7 @@ roles: - ibm.mas_devops.ibm_catalogs - ibm.mas_devops.cert_manager + - ibm.mas_devops.mongodb - ibm.mas_devops.sls - ibm.mas_devops.dro - ibm.mas_devops.db2 From cb53eb131e807df13bc2ea7440aad0cec6282a5c Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 24 Jan 2025 16:11:30 +0000 Subject: [PATCH 097/169] [patch] move get cfgs to aibroker role --- .../playbooks/oneclick_add_aibroker.yml | 2 +- .../roles/aibroker/tasks/config_sls/main.yml | 20 +++++++++++++++++++ ibm/mas_devops/roles/aibroker/tasks/main.yml | 9 +++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 23af90a666..0fb9fbf7bf 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -77,4 +77,4 @@ # - ibm.mas_devops.odh # - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker - - ibm.mas_devops.aibroker_app_config + # - ibm.mas_devops.aibroker_app_config diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml new file mode 100644 index 0000000000..f260373798 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml @@ -0,0 +1,20 @@ +--- +- name: "Read sls config file" + ansible.builtin.set_fact: + slscfg: "{{ lookup('file', '{{ mas_config_dir }}/sls.yml') | from_yaml_all }}" + +- name: "Set sls mandatory vars" + ansible.builtin.set_fact: + slscfg: + secret_name: "{{ slscfg[0].metadata.name }}" + registration_key: "{{ slscfg[0].stringData.registrationKey }}" + url: "{{ slscfg[1].spec.config.url }}" + ca: "{{ slscfg[1].spec.certificates }}" + +- name: "Debug: sls information" #TODO: remove before PR + debug: + msg: + - "sls secret name ......... {{ slscfg.secret_name }}" + - "sls registration key .... {{ slscfg.registration_key }}" + - "sls url ................. {{ slscfg.url }}" + - "sls cert ca ............. {{ slscfg.ca }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 4a90d9dde5..e21f5ba392 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -2,6 +2,15 @@ - name: Load variables (main) include_vars: "vars/main.yml" +# Create config for SLS +- include_tasks: tasks/config_sls/main.yml + +# Create config for DRO +- include_tasks: tasks/config_dro/main.yml + +# Create config for DB2 +- include_tasks: tasks/config_db2/main.yml + # install AI Broker api - include_tasks: tasks/aibroker/main.yml From 376de392d3c75fc3e8a23ad2c3315f038ab9bb40 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 24 Jan 2025 16:17:26 +0000 Subject: [PATCH 098/169] [patch] move roles dro db2 config to aibroker --- .../roles/aibroker/tasks/config_db2/main.yml | 22 +++++++++++++++++++ .../roles/aibroker/tasks/config_dro/main.yml | 20 +++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml create mode 100644 ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml new file mode 100644 index 0000000000..28876edafc --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -0,0 +1,22 @@ +--- +- name: "Read jdbc config file" + ansible.builtin.set_fact: + jdbccfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_app_id }}-db2u.yml') | from_yaml_all }}" + +- name: "Set dro mandatory vars" + ansible.builtin.set_fact: + jdbccfg: + # username: "{{ jdbccfg[0].data.username | b64decode}}" + # password: "{{ jdbccfg[0].data.password | b64decode}}" + username: "{{ jdbccfg[0].data.username }}" + password: "{{ jdbccfg[0].data.password }}" + url: "{{ jdbccfg[1].spec.config.url }}" + ca: "{{ jdbccfg[1].spec.certificates }}" + +- name: "Debug: jdbc information" #TODO: remove before PR + debug: + msg: + - "JDBC username ............. {{ jdbccfg.username }}" + - "JDBC password ............. {{ jdbccfg.password }}" + - "JDBC url .................. {{ jdbccfg.url }}" + - "JDBC cert ca .............. {{ jdbccfg.ca }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml new file mode 100644 index 0000000000..52608b1010 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml @@ -0,0 +1,20 @@ +--- +- name: "Read dro config file" + ansible.builtin.set_fact: + drocfg: "{{ lookup('file', '{{ mas_config_dir }}/dro.yml') | from_yaml_all }}" + +- name: "Set dro mandatory vars" + ansible.builtin.set_fact: + drocfg: + secret_name: "{{ drocfg[0].metadata.name }}" + registration_key: "{{ drocfg[0].stringData.api_key }}" + url: "{{ drocfg[1].spec.config.url }}" + ca: "{{ drocfg[1].spec.certificates }}" + +- name: "Debug: dro information" #TODO: remove before PR + debug: + msg: + - "dro secret name ......... {{ drocfg.secret_name }}" + - "dro api key ............. {{ drocfg.registration_key }}" + - "dro url ................. {{ drocfg.url }}" + - "dro cert ca ............. {{ drocfg.ca }}" From fb8746b147d40417f1e3ed99110839483acb7f76 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 24 Jan 2025 16:53:07 +0000 Subject: [PATCH 099/169] [patch] update CR --- ibm/mas_devops/roles/aibroker/defaults/main.yml | 1 + .../roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/defaults/main.yml b/ibm/mas_devops/roles/aibroker/defaults/main.yml index 3b14fcde35..fc4c738430 100644 --- a/ibm/mas_devops/roles/aibroker/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker/defaults/main.yml @@ -1,6 +1,7 @@ --- mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" +mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" # Custom Labels # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index 32c472b2de..3601bbae90 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -7,7 +7,7 @@ metadata: labels: mas.ibm.com/applicationId: aibroker mas.ibm.com/instanceId: "{{ mas_instance_id }}" -spec: +spec: settings: icr: cp: "{{ mas_icr_cp }}" From 6b16be7533cae6814f04eeb43cfd7199163505b9 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 11:19:10 +0000 Subject: [PATCH 100/169] [patch] update set fact to use string for certs --- .../roles/aibroker/tasks/config_db2/main.yml | 10 +++++----- .../roles/aibroker/tasks/config_dro/main.yml | 2 +- .../roles/aibroker/tasks/config_sls/main.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml index 28876edafc..6987e1f0c3 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -6,12 +6,12 @@ - name: "Set dro mandatory vars" ansible.builtin.set_fact: jdbccfg: - # username: "{{ jdbccfg[0].data.username | b64decode}}" - # password: "{{ jdbccfg[0].data.password | b64decode}}" - username: "{{ jdbccfg[0].data.username }}" - password: "{{ jdbccfg[0].data.password }}" + username: "{{ jdbccfg[0].data.username | b64decode}}" + password: "{{ jdbccfg[0].data.password | b64decode}}" + # username: "{{ jdbccfg[0].data.username }}" + # password: "{{ jdbccfg[0].data.password }}" url: "{{ jdbccfg[1].spec.config.url }}" - ca: "{{ jdbccfg[1].spec.certificates }}" + ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') }}" - name: "Debug: jdbc information" #TODO: remove before PR debug: diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml index 52608b1010..e19884c8d3 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml @@ -9,7 +9,7 @@ secret_name: "{{ drocfg[0].metadata.name }}" registration_key: "{{ drocfg[0].stringData.api_key }}" url: "{{ drocfg[1].spec.config.url }}" - ca: "{{ drocfg[1].spec.certificates }}" + ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') }}" - name: "Debug: dro information" #TODO: remove before PR debug: diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml index f260373798..2f90e500fd 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml @@ -9,7 +9,7 @@ secret_name: "{{ slscfg[0].metadata.name }}" registration_key: "{{ slscfg[0].stringData.registrationKey }}" url: "{{ slscfg[1].spec.config.url }}" - ca: "{{ slscfg[1].spec.certificates }}" + ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') }}" - name: "Debug: sls information" #TODO: remove before PR debug: From 2396c0631abd9263b61d5abdde91cff03241262a Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 12:15:47 +0000 Subject: [PATCH 101/169] [patch] remove join list for DRO/SLS/DB2 --- ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml | 3 ++- ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml | 3 ++- ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml index 6987e1f0c3..c2571c343b 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -11,7 +11,8 @@ # username: "{{ jdbccfg[0].data.username }}" # password: "{{ jdbccfg[0].data.password }}" url: "{{ jdbccfg[1].spec.config.url }}" - ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" + # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') }}" - name: "Debug: jdbc information" #TODO: remove before PR debug: diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml index e19884c8d3..3d321fda6a 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml @@ -9,7 +9,8 @@ secret_name: "{{ drocfg[0].metadata.name }}" registration_key: "{{ drocfg[0].stringData.api_key }}" url: "{{ drocfg[1].spec.config.url }}" - ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + # ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') }}" - name: "Debug: dro information" #TODO: remove before PR debug: diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml index 2f90e500fd..91a9e651fa 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml @@ -9,7 +9,8 @@ secret_name: "{{ slscfg[0].metadata.name }}" registration_key: "{{ slscfg[0].stringData.registrationKey }}" url: "{{ slscfg[1].spec.config.url }}" - ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + # ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') }}" - name: "Debug: sls information" #TODO: remove before PR debug: From 53f011f738f3edaa982142cf57aa59b8d7c9c1fa Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 14:07:04 +0000 Subject: [PATCH 102/169] [patch] added secret for DRO --- ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml | 5 +++++ .../aibroker/templates/aibroker/operator-group.yml.j2 | 1 + .../roles/aibroker/templates/dro/dro-token.yml.j2 | 9 +++++++++ ibm/mas_devops/roles/aibroker/vars/main.yml | 3 +++ 4 files changed, 18 insertions(+) create mode 100644 ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml index 3d321fda6a..1cedb43c35 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml @@ -19,3 +19,8 @@ - "dro api key ............. {{ drocfg.registration_key }}" - "dro url ................. {{ drocfg.url }}" - "dro cert ca ............. {{ drocfg.ca }}" + +- name: "Create secret for Dro registration key" + kubernetes.core.k8s: + apply: yes + template: "templates/dro/dro-token.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/operator-group.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/operator-group.yml.j2 index 30ecdb6739..fcf430d336 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/operator-group.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/operator-group.yml.j2 @@ -1,3 +1,4 @@ +--- apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: diff --git a/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 new file mode 100644 index 0000000000..7bf681bf1f --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ mas_aibroker_dro_token }} + namespace: mas-{{ mas_instance_id }}-aibroker +data: + DRO_TOKEN: {{ drocfg.registration_key }} +type: Opaque \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 29d86e90d3..1ad2b6bbae 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -48,3 +48,6 @@ mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRAT mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" mas_aibroker_path_ca_crt: './certs' mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" + +# DRO +mas_aibroker_dro_token: "dro_token" From e61bc71342c0de9e68764b098b959ecb8d99e25a Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 14:27:28 +0000 Subject: [PATCH 103/169] [patch] add dro secrets --- ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 | 2 +- ibm/mas_devops/roles/aibroker/vars/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 index 7bf681bf1f..ad78906e58 100644 --- a/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 @@ -5,5 +5,5 @@ metadata: name: {{ mas_aibroker_dro_token }} namespace: mas-{{ mas_instance_id }}-aibroker data: - DRO_TOKEN: {{ drocfg.registration_key }} + DRO_TOKEN: "{{ drocfg.registration_key | b64encode }}" type: Opaque \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 1ad2b6bbae..c5fa78c466 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -50,4 +50,4 @@ mas_aibroker_path_ca_crt: './certs' mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" # DRO -mas_aibroker_dro_token: "dro_token" +mas_aibroker_dro_token: "dro-token" From 6b5d7a4396858c747320788b7270901fa4ba2f77 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 14:31:37 +0000 Subject: [PATCH 104/169] [patch] remove duplicate --- ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 | 2 +- ibm/mas_devops/roles/aibroker/vars/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 index ad78906e58..e6684c1714 100644 --- a/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/dro/dro-token.yml.j2 @@ -2,7 +2,7 @@ kind: Secret apiVersion: v1 metadata: - name: {{ mas_aibroker_dro_token }} + name: {{ mas_aibroker_dro_token_secret }} namespace: mas-{{ mas_instance_id }}-aibroker data: DRO_TOKEN: "{{ drocfg.registration_key | b64encode }}" diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index c5fa78c466..574e703b64 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -50,4 +50,4 @@ mas_aibroker_path_ca_crt: './certs' mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" # DRO -mas_aibroker_dro_token: "dro-token" +mas_aibroker_dro_token_secret: "dro-token" From a22eaa59b4cfad613c4ea3dbaf22478d1e515fcb Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 14:41:57 +0000 Subject: [PATCH 105/169] [patch] added secret for db2 admin --- .../roles/aibroker/tasks/config_db2/main.yml | 13 +++++++++---- .../templates/jdbc/jdbc-admin-credentials.yml.j2 | 10 ++++++++++ ibm/mas_devops/roles/aibroker/vars/main.yml | 3 +++ 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml index c2571c343b..04300b1638 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -6,10 +6,10 @@ - name: "Set dro mandatory vars" ansible.builtin.set_fact: jdbccfg: - username: "{{ jdbccfg[0].data.username | b64decode}}" - password: "{{ jdbccfg[0].data.password | b64decode}}" - # username: "{{ jdbccfg[0].data.username }}" - # password: "{{ jdbccfg[0].data.password }}" + # username: "{{ jdbccfg[0].data.username | b64decode}}" + # password: "{{ jdbccfg[0].data.password | b64decode}}" + username: "{{ jdbccfg[0].data.username }}" + password: "{{ jdbccfg[0].data.password }}" url: "{{ jdbccfg[1].spec.config.url }}" ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') }}" @@ -21,3 +21,8 @@ - "JDBC password ............. {{ jdbccfg.password }}" - "JDBC url .................. {{ jdbccfg.url }}" - "JDBC cert ca .............. {{ jdbccfg.ca }}" + +- name: "Create secret for JDBC admin credentials" + kubernetes.core.k8s: + apply: yes + template: "templates/jdbc/jdbc-admin-credentials.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 new file mode 100644 index 0000000000..555703aecd --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 @@ -0,0 +1,10 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ mas_aibroker_dro_jdbc_secret }} + namespace: mas-{{ mas_instance_id }}-aibroker +data: + user: "{{ jdbccfg.username | b64encode }}" + password: "{{ jdbccfg.password | b64encode }}" +type: Opaque diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 574e703b64..85fde4b6b9 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -51,3 +51,6 @@ mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT' # DRO mas_aibroker_dro_token_secret: "dro-token" + +# JDBC +mas_aibroker_dro_jdbc_secret: "jdbc-admin-credentials" \ No newline at end of file From 790fb945a8ba47a9f8d03f8ad5631e9a6423cc7d Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 14:49:34 +0000 Subject: [PATCH 106/169] [patch] update secret --- .../roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 | 2 +- ibm/mas_devops/roles/aibroker/vars/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 index 555703aecd..263def1801 100644 --- a/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 @@ -5,6 +5,6 @@ metadata: name: {{ mas_aibroker_dro_jdbc_secret }} namespace: mas-{{ mas_instance_id }}-aibroker data: - user: "{{ jdbccfg.username | b64encode }}" + username: "{{ jdbccfg.username }}" password: "{{ jdbccfg.password | b64encode }}" type: Opaque diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 85fde4b6b9..3a589ece84 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -53,4 +53,4 @@ mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT' mas_aibroker_dro_token_secret: "dro-token" # JDBC -mas_aibroker_dro_jdbc_secret: "jdbc-admin-credentials" \ No newline at end of file +mas_aibroker_dro_jdbc_secret: "jdbc-admin-credentials" From 8e96c37435284508303ad4b13f041e7cc84b83a1 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 15:23:21 +0000 Subject: [PATCH 107/169] [patch] added sls registration key secret --- ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml | 5 +++++ .../aibroker/templates/sls/sls-tenant-details.yml.j2 | 9 +++++++++ ibm/mas_devops/roles/aibroker/vars/main.yml | 3 +++ 3 files changed, 17 insertions(+) create mode 100644 ibm/mas_devops/roles/aibroker/templates/sls/sls-tenant-details.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml index 91a9e651fa..cbb2761612 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml @@ -19,3 +19,8 @@ - "sls registration key .... {{ slscfg.registration_key }}" - "sls url ................. {{ slscfg.url }}" - "sls cert ca ............. {{ slscfg.ca }}" + +- name: "Create secret for SLS registration key" + kubernetes.core.k8s: + apply: yes + template: "templates/sls/sls-tenant-details.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker/templates/sls/sls-tenant-details.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/sls/sls-tenant-details.yml.j2 new file mode 100644 index 0000000000..238a157ce0 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/templates/sls/sls-tenant-details.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ mas_aibroker_dro_token_secret }} + namespace: mas-{{ mas_instance_id }}-aibroker +data: + SLS_REGISTRATION_KEY: "{{ slscfg.registration_key | b64encode }}" +type: Opaque diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 3a589ece84..9079f498a5 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -54,3 +54,6 @@ mas_aibroker_dro_token_secret: "dro-token" # JDBC mas_aibroker_dro_jdbc_secret: "jdbc-admin-credentials" + +# SLS +mas_aibroker_sls_registration_key_secret: "sls-registration-key" From 1ac1d86b244c606e2ba2c5ec17d4f3162cd4e86f Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 15:25:33 +0000 Subject: [PATCH 108/169] [patch] fix typo for sls secret name --- .../roles/aibroker/templates/sls/sls-tenant-details.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/sls/sls-tenant-details.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/sls/sls-tenant-details.yml.j2 index 238a157ce0..35f6285706 100644 --- a/ibm/mas_devops/roles/aibroker/templates/sls/sls-tenant-details.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/sls/sls-tenant-details.yml.j2 @@ -2,7 +2,7 @@ kind: Secret apiVersion: v1 metadata: - name: {{ mas_aibroker_dro_token_secret }} + name: {{ mas_aibroker_sls_registration_key_secret }} namespace: mas-{{ mas_instance_id }}-aibroker data: SLS_REGISTRATION_KEY: "{{ slscfg.registration_key | b64encode }}" From 4e2f2e6b7e0e6748a051d3be77347ed97316f670 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 15:51:33 +0000 Subject: [PATCH 109/169] [patch] update CR --- .../roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index 3601bbae90..520684c371 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -12,16 +12,11 @@ spec: icr: cp: "{{ mas_icr_cp }}" cpopen: "{{ mas_icr_cpopen }}" - sls: - url: "{{ slscfg.url }}" - registration_key: "{{ slscfg.registration_key }}" - ca: "{{ slscfg.ca }}" dro: url: "{{ drocfg.url }}" - registration_key: "{{ drocfg.registration_key }}" + dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" ca: "{{ drocfg.ca }}" jdbc: url: "{{ jdbccfg.url }}" - username: "{{ jdbccfg.username }}" - password: "{{ jdbccfg.password }}" + credentials_secret: "{{ mas_aibroker_dro_jdbc_secret }}" ca: "{{ jdbccfg.ca }}" From a0f7269cac7a5bffe1f45a1d8db1e7e3a1c8b6a4 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 16:12:49 +0000 Subject: [PATCH 110/169] [patch] add operator group for AIbroker --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 404ec57fe3..28c3a51c07 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -36,6 +36,11 @@ # catalog_source: "{{ mas_catalog_source }}" # register: subscription +- name: "Install AI Broker Operator Group" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/aibroker/operator-group.yml.j2') }}" + - name: "Install AI Broker Operator" kubernetes.core.k8s: apply: yes From e86ea5260aeba69c8247afbbeab0943e2d921c03 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 16:28:54 +0000 Subject: [PATCH 111/169] [patch] disable OG for AIbroker --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 28c3a51c07..699831b448 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -36,10 +36,10 @@ # catalog_source: "{{ mas_catalog_source }}" # register: subscription -- name: "Install AI Broker Operator Group" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/aibroker/operator-group.yml.j2') }}" +# - name: "Install AI Broker Operator Group" +# kubernetes.core.k8s: +# apply: yes +# definition: "{{ lookup('template', 'templates/aibroker/operator-group.yml.j2') }}" - name: "Install AI Broker Operator" kubernetes.core.k8s: From 2f23f6d4ad7a2d8966db2a9b6ec72dbde70fa57d Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 16:38:10 +0000 Subject: [PATCH 112/169] [patch] add create aibroker namespace to main yml file --- .../roles/aibroker/tasks/aibroker/main.yml | 16 ++++++++-------- ibm/mas_devops/roles/aibroker/tasks/main.yml | 9 +++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 699831b448..d2cf1e853e 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -8,14 +8,14 @@ - "Channel ........................ {{ mas_app_channel }}" - "MAS Instance Id ................ {{ mas_instance_id }}" -# Create IBM Maximo IBM Maximo AI Broker -# ----------------------------------------------------------------------------- -- name: "Create IBM Maximo AI Broker namespace" - kubernetes.core.k8s: - state: present - api_version: v1 - kind: Namespace - name: "{{ aibroker_namespace }}" +# # Create IBM Maximo IBM Maximo AI Broker +# # ----------------------------------------------------------------------------- +# - name: "Create IBM Maximo AI Broker namespace" +# kubernetes.core.k8s: +# state: present +# api_version: v1 +# kind: Namespace +# name: "{{ aibroker_namespace }}" # Install the operator & create entitlement secret diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index e21f5ba392..2110d7bf21 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -2,6 +2,15 @@ - name: Load variables (main) include_vars: "vars/main.yml" +# Create IBM Maximo IBM Maximo AI Broker +# ----------------------------------------------------------------------------- +- name: "Create IBM Maximo AI Broker namespace" + kubernetes.core.k8s: + state: present + api_version: v1 + kind: Namespace + name: "{{ aibroker_namespace }}" + # Create config for SLS - include_tasks: tasks/config_sls/main.yml From 654f38115b22609c2cea0c695f9a90c7165caa6a Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 27 Jan 2025 16:46:45 +0000 Subject: [PATCH 113/169] [patch] fix OG for AIbroker --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 8 ++++---- .../aibroker/templates/aibroker/operator-group.yml.j2 | 8 +------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index d2cf1e853e..e251145976 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -36,10 +36,10 @@ # catalog_source: "{{ mas_catalog_source }}" # register: subscription -# - name: "Install AI Broker Operator Group" -# kubernetes.core.k8s: -# apply: yes -# definition: "{{ lookup('template', 'templates/aibroker/operator-group.yml.j2') }}" +- name: "Install AI Broker Operator Group" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/aibroker/operator-group.yml.j2') }}" - name: "Install AI Broker Operator" kubernetes.core.k8s: diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/operator-group.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/operator-group.yml.j2 index fcf430d336..5aab10febd 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/operator-group.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/operator-group.yml.j2 @@ -2,14 +2,8 @@ apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: - name: mas-{{ mas_instance_id }}-aibroker-operator-group + name: mas-{{ mas_instance_id }}-aibroker namespace: "{{ aibroker_namespace }}" -{% if custom_labels is defined and custom_labels.items() %} - labels: -{% for key, value in custom_labels.items() %} - "{{ key }}": "{{ value }}" -{% endfor %} -{% endif %} spec: targetNamespaces: - "{{ aibroker_namespace }}" From 9bfc4229b76fd4dea8d460f6759ffd26e1b84f66 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 28 Jan 2025 09:52:30 +0000 Subject: [PATCH 114/169] [patch] parse cert to string for sls/dro/db2 --- ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml | 4 ++-- ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml | 4 ++-- ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml index 04300b1638..58ec6b6eb7 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -11,8 +11,8 @@ username: "{{ jdbccfg[0].data.username }}" password: "{{ jdbccfg[0].data.password }}" url: "{{ jdbccfg[1].spec.config.url }}" - ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" - # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" + ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') }}" - name: "Debug: jdbc information" #TODO: remove before PR debug: diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml index 1cedb43c35..18438c865c 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml @@ -9,8 +9,8 @@ secret_name: "{{ drocfg[0].metadata.name }}" registration_key: "{{ drocfg[0].stringData.api_key }}" url: "{{ drocfg[1].spec.config.url }}" - # ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') }}" - ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') }}" + ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + # ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') }}" - name: "Debug: dro information" #TODO: remove before PR debug: diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml index cbb2761612..c5a5d5252e 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml @@ -9,8 +9,8 @@ secret_name: "{{ slscfg[0].metadata.name }}" registration_key: "{{ slscfg[0].stringData.registrationKey }}" url: "{{ slscfg[1].spec.config.url }}" - # ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') }}" - ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') }}" + ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + # ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') }}" - name: "Debug: sls information" #TODO: remove before PR debug: From 968cf5cb787f20c5755dc87d37ec4ef0d5ea20dd Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 28 Jan 2025 10:07:00 +0000 Subject: [PATCH 115/169] [patch] update cert format in CR --- ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml | 2 +- ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml | 2 +- ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml index 58ec6b6eb7..98c16283d2 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -12,7 +12,7 @@ password: "{{ jdbccfg[0].data.password }}" url: "{{ jdbccfg[1].spec.config.url }}" # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" - ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n')}}" - name: "Debug: jdbc information" #TODO: remove before PR debug: diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml index 18438c865c..7a2a60a520 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml @@ -9,7 +9,7 @@ secret_name: "{{ drocfg[0].metadata.name }}" registration_key: "{{ drocfg[0].stringData.api_key }}" url: "{{ drocfg[1].spec.config.url }}" - ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" # ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') }}" - name: "Debug: dro information" #TODO: remove before PR diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml index c5a5d5252e..f32195ca35 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml @@ -9,7 +9,7 @@ secret_name: "{{ slscfg[0].metadata.name }}" registration_key: "{{ slscfg[0].stringData.registrationKey }}" url: "{{ slscfg[1].spec.config.url }}" - ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') }}" + ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" # ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') }}" - name: "Debug: sls information" #TODO: remove before PR From 56b13bc00258e6b50e7d3d78fb269d3b5c97d65f Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 28 Jan 2025 10:07:31 +0000 Subject: [PATCH 116/169] [patch] add format for db2 --- ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml index 98c16283d2..69ed491313 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -12,7 +12,7 @@ password: "{{ jdbccfg[0].data.password }}" url: "{{ jdbccfg[1].spec.config.url }}" # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" - ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n')}}" + ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" - name: "Debug: jdbc information" #TODO: remove before PR debug: From b4e2e9b7c21f44a431a13afcbe2c9877c8af96a3 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 28 Jan 2025 11:14:43 +0000 Subject: [PATCH 117/169] [patch] increate time for wait for aibroker-api pod --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 4 ++-- ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index e251145976..d07014d0b8 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -90,7 +90,7 @@ register: _is_not_exists until: _is_not_exists.stdout|int == 0 retries: 60 - delay: 10 + delay: 30 - name: "Wait for aibroker-api pod to be ready (10s delay)" shell: > @@ -98,4 +98,4 @@ register: _is_not_ready until: _is_not_ready.stdout|int == 0 retries: 60 - delay: 10 + delay: 30 diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index b94d78ea4d..796fe927dd 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -106,7 +106,7 @@ register: _is_not_exists until: _is_not_exists.stdout|int == 0 retries: 60 - delay: 10 + delay: 30 - name: "Wait for aibroker-api pod to be ready (10s delay)" shell: > @@ -114,7 +114,7 @@ register: _is_not_ready until: _is_not_ready.stdout|int == 0 retries: 60 - delay: 10 + delay: 30 - name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri uri: From b021db7477c1627240d6791c09a65a7de1b292ec Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 28 Jan 2025 11:18:08 +0000 Subject: [PATCH 118/169] [patch] increase time waiting for pod ready --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 2 +- ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index d07014d0b8..41e3ebc22b 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -92,7 +92,7 @@ retries: 60 delay: 30 -- name: "Wait for aibroker-api pod to be ready (10s delay)" +- name: "Wait for aibroker-api pod to be ready (30s delay)" shell: > oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l register: _is_not_ready diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 796fe927dd..41ec997e4e 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -108,7 +108,7 @@ retries: 60 delay: 30 -- name: "Wait for aibroker-api pod to be ready (10s delay)" +- name: "Wait for aibroker-api pod to be ready (30s delay)" shell: > oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l register: _is_not_ready From 4ef131c967d2cddd1e65e25757361210a4723b76 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 24 Feb 2025 15:44:43 +0000 Subject: [PATCH 119/169] [patch] update playbook to create CR for aibroker workspace --- .../roles/aibroker/tasks/aibroker/main.yml | 11 +++++++ .../templates/aibroker/aibrokerapp.yml.j2 | 2 +- .../aibroker/aibrokerworkspace.yml.j2 | 30 +++++++++++++++++++ .../jdbc/jdbc-admin-credentials.yml.j2 | 2 +- ibm/mas_devops/roles/aibroker/vars/main.yml | 8 +++-- 5 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 41e3ebc22b..9fd96e7282 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -80,6 +80,17 @@ retries: 45 # approx 45 minutes before we give up delay: 60 # 1 minute +# IBM Maximo AI Broker workspace +# ----------------------------------------------------------------------------- +- name: Create AI Broker workspace CR + vars: + annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}" + kubernetes.core.k8s: + state: present + namespace: "{{ aibroker_namespace }}" + template: templates/aibroker/aibrokerworkspace.yml.j2 + register: aibrokerWorkspaceResult + - name: Await to aibroker-api pod created pause: minutes: 3 diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index 520684c371..889ab3929e 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -18,5 +18,5 @@ spec: ca: "{{ drocfg.ca }}" jdbc: url: "{{ jdbccfg.url }}" - credentials_secret: "{{ mas_aibroker_dro_jdbc_secret }}" + credentials_secret: "{{ mas_aibroker_jdbc_secret }}" ca: "{{ jdbccfg.ca }}" diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 new file mode 100644 index 0000000000..8c40b313cd --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 @@ -0,0 +1,30 @@ +--- +apiVersion: apps.mas.ibm.com/v1 +kind: AiBrokerWorkspace +metadata: + name: "{{ mas_instance_id }}" + namespace: "{{ aibroker_namespace }}" + labels: + mas.ibm.com/applicationId: aibroker + mas.ibm.com/instanceId: "{{ mas_instance_id }}" +spec: + settings: + icr: + cp: "{{ mas_icr_cp }}" + cpopen: "{{ mas_icr_cpopen }}" + dro: + url: "{{ drocfg.url }}" + dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" + ca: "{{ drocfg.ca }}" + jdbc: + url: "{{ jdbccfg.url }}" + credentials_secret: "{{ mas_aibroker_dro_jdbc_secret }}" + ca: "{{ jdbccfg.ca }}" + sls: + url: "{{ slscfg.url }}" + sls_regitration_key_secret: "{{ slscfg.registration_key }}" + ca: "{{ slscfg.ca }}" + tenant: + tenant_id: "{{ mas_aibroker_tenant_name }}" + subscription_id: "{{ mas_aibroker_sls_subscription_id }}" + instance_id: "{{ mas_instance_id }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 index 263def1801..7e0ac3bf5f 100644 --- a/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 @@ -2,7 +2,7 @@ kind: Secret apiVersion: v1 metadata: - name: {{ mas_aibroker_dro_jdbc_secret }} + name: {{ mas_aibroker_jdbc_secret }} namespace: mas-{{ mas_instance_id }}-aibroker data: username: "{{ jdbccfg.username }}" diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index 9079f498a5..c934f3c7f3 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -53,7 +53,11 @@ mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT' mas_aibroker_dro_token_secret: "dro-token" # JDBC -mas_aibroker_dro_jdbc_secret: "jdbc-admin-credentials" +mas_aibroker_jdbc_secret: "jdbc-admin-credentials" # SLS -mas_aibroker_sls_registration_key_secret: "sls-registration-key" +mas_aibroker_sls_registration_key_secret: "sls-registration-key" +mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" + +# TENANT +mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" \ No newline at end of file From a504e0f763fc20d27335f13f70a47fe0e7795246 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 24 Feb 2025 15:49:47 +0000 Subject: [PATCH 120/169] [patch] fix no new line character at the end of file --- ibm/mas_devops/roles/aibroker/vars/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index c934f3c7f3..defda35239 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -58,6 +58,3 @@ mas_aibroker_jdbc_secret: "jdbc-admin-credentials" # SLS mas_aibroker_sls_registration_key_secret: "sls-registration-key" mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" - -# TENANT -mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" \ No newline at end of file From cd3ccba3ead133427bbd45d9a0657944cc8430da Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 24 Feb 2025 15:50:47 +0000 Subject: [PATCH 121/169] [patch] fix var name --- .../roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 index 8c40b313cd..8e30f247a1 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 @@ -18,7 +18,7 @@ spec: ca: "{{ drocfg.ca }}" jdbc: url: "{{ jdbccfg.url }}" - credentials_secret: "{{ mas_aibroker_dro_jdbc_secret }}" + credentials_secret: "{{ mas_aibroker_jdbc_secret }}" ca: "{{ jdbccfg.ca }}" sls: url: "{{ slscfg.url }}" From 8d6f0db04ef4cdb7258a6a1e9238220064cbb842 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 25 Feb 2025 14:29:17 +0000 Subject: [PATCH 122/169] [patch] add task for wait until workspace CR finish --- .../roles/aibroker/tasks/aibroker/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 9fd96e7282..6c7ce06f55 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -91,6 +91,22 @@ template: templates/aibroker/aibrokerworkspace.yml.j2 register: aibrokerWorkspaceResult +# Wait IBM Maximo AI Broker custom resource to be complete +# ----------------------------------------------------------------------------- +- name: "Wait for AiBrokerWorkspace custom resource to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_instance_id }}" + namespace: "{{ aibroker_namespace }}" + kind: AiBrokerWorkspace + register: aibrokerworkspace_cr_result + until: + - aibrokerworkspace_cr_result_cr_result.resources is defined and aibrokerworkspace_cr_result.resources | length == 1 + - aibrokerworkspace_cr_result.resources[0].status is defined + - aibrokerworkspace_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 + retries: 45 # approx 45 minutes before we give up + delay: 60 # 1 minute + - name: Await to aibroker-api pod created pause: minutes: 3 From 132855d3989b9c082dc84a41d16fa264c21249fc Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 5 Mar 2025 17:46:36 +0000 Subject: [PATCH 123/169] [patch] fix wait for workspace condition --- ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index 6c7ce06f55..ebe99c6e35 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -101,7 +101,7 @@ kind: AiBrokerWorkspace register: aibrokerworkspace_cr_result until: - - aibrokerworkspace_cr_result_cr_result.resources is defined and aibrokerworkspace_cr_result.resources | length == 1 + - aibrokerworkspace_cr_result.resources is defined and aibrokerworkspace_cr_result.resources | length == 1 - aibrokerworkspace_cr_result.resources[0].status is defined - aibrokerworkspace_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 retries: 45 # approx 45 minutes before we give up From ea7939dd12ad0b358c617f3b6c203a2c659acfbe Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 5 Mar 2025 17:54:55 +0000 Subject: [PATCH 124/169] [patch] update tenant name in CR --- .../roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 index 8e30f247a1..0cb94a3c28 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 @@ -25,6 +25,6 @@ spec: sls_regitration_key_secret: "{{ slscfg.registration_key }}" ca: "{{ slscfg.ca }}" tenant: - tenant_id: "{{ mas_aibroker_tenant_name }}" + tenant_id: "{{ tenantNamespace }}" subscription_id: "{{ mas_aibroker_sls_subscription_id }}" instance_id: "{{ mas_instance_id }}" \ No newline at end of file From 188ac80990bb8966629a2f928bd5074e8016d639 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 14 Mar 2025 14:10:26 +0000 Subject: [PATCH 125/169] [patch] add new role for create tenanr --- .../playbooks/oneclick_add_aibroker.yml | 17 +- ibm/mas_devops/roles/aibroker/tasks/main.yml | 12 +- .../roles/aibroker_app_config/tasks/main.yml | 38 ---- .../README.md | 0 .../defaults/main.yml | 8 +- .../files/access_wx.py | 0 .../files/alm_sample_sls_use.py | 0 .../files/create_apikey.sh | 0 .../files/create_bucket.py | 0 .../files/create_dro_token.sh | 0 .../files/create_entitlement.py | 0 .../files/create_s3.sh | 0 .../files/create_sls_secret.sh | 0 .../files/create_tenant.py | 0 .../files/create_tenant_entitlement.py | 0 .../files/create_wx.sh | 0 .../files/delete_s3.py | 0 .../files/generate_api_key.py | 0 .../meta/main.yml | 0 .../aibroker_tenant/tasks/aibroker/main.yml | 128 ++++++++++++ .../tasks/apikey/install/main.yml | 13 ++ .../aibroker_tenant/tasks/apikey/main.yml | 15 ++ .../tasks/apikey/remove/main.yml | 6 + .../tasks/apikey/update/main.yml | 16 ++ .../tasks/config_db2/main.yml | 8 +- .../tasks/config_dro/main.yml | 8 +- .../tasks/config_sls/main.yml | 8 +- .../roles/aibroker_tenant/tasks/main.yml | 40 ++++ .../aibroker_tenant/tasks/s3/install/main.yml | 14 ++ .../roles/aibroker_tenant/tasks/s3/main.yml | 14 ++ .../aibroker_tenant/tasks/s3/remove/main.yml | 55 +++++ .../roles/aibroker_tenant/tasks/saas/main.yml | 25 +++ .../tasks/tenant/install/main.yml | 197 ++++++++++++++++++ .../aibroker_tenant/tasks/tenant/main.yml | 14 ++ .../tasks/tenant/remove/main.yml | 50 +++++ .../tasks/watsonx/install/main.yml | 19 ++ .../aibroker_tenant/tasks/watsonx/main.yml | 14 ++ .../tasks/watsonx/remove/main.yml | 6 + .../templates/aibroker/aibrokerapp.yml.j2 | 22 ++ .../aibroker/aibrokerworkspace.yml.j2 | 30 +++ .../templates/aibroker/operator-group.yml.j2 | 9 + .../templates/aibroker/subscription.yml.j2 | 12 ++ .../templates/dro/dro-token.yml.j2 | 9 + .../jdbc/jdbc-admin-credentials.yml.j2 | 10 + .../templates/sls/sls-tenant-details.yml.j2 | 9 + .../tenant/connector-configmap.yml.j2 | 7 + .../templates/tenant/km-s3-sa.yml.j2 | 8 + .../tenant/kserve-huggingfaceserver.yml.j2 | 33 +++ .../templates/tenant/kserve-lgbserver.yml.j2 | 33 +++ .../tenant/kserve-sklearnserver.yml.j2 | 32 +++ .../templates/tenant/kserve-xgbserver.yml.j2 | 33 +++ .../templates/tenant/rbac.yml.j2 | 104 +++++++++ .../vars/main.yml | 10 + 53 files changed, 1025 insertions(+), 61 deletions(-) delete mode 100644 ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/README.md (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/defaults/main.yml (74%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/access_wx.py (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/alm_sample_sls_use.py (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_apikey.sh (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_bucket.py (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_dro_token.sh (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_entitlement.py (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_s3.sh (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_sls_secret.sh (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_tenant.py (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_tenant_entitlement.py (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/create_wx.sh (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/delete_s3.py (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/files/generate_api_key.py (100%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/meta/main.yml (100%) create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/aibroker/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/install/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/remove/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/update/main.yml rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/tasks/config_db2/main.yml (71%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/tasks/config_dro/main.yml (69%) rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/tasks/config_sls/main.yml (68%) create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/s3/install/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/s3/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/s3/remove/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/saas/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/remove/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/install/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/remove/main.yml create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerapp.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/operator-group.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/subscription.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-token.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/jdbc/jdbc-admin-credentials.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-tenant-details.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/connector-configmap.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/km-s3-sa.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-huggingfaceserver.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-lgbserver.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-sklearnserver.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-xgbserver.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/rbac.yml.j2 rename ibm/mas_devops/roles/{aibroker_app_config => aibroker_tenant}/vars/main.yml (91%) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 0fb9fbf7bf..b407f05516 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -17,7 +17,7 @@ mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" # Control if saas release - # mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" + mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" # DB2 # ----------------------------------------------------------------------------- @@ -47,11 +47,6 @@ # - lookup('env', 'MAS_AIBROKER_DB_DATABASE') != "" # - lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') != "" # - lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') != "" - # - lookup('env', 'MAS_AIBROKER_DOMAIN') != "" - # - lookup('env', 'MAS_AIBROKER_SLS_URL') != "" - # - lookup('env', 'MAS_AIBROKER_DRO_URL') != "" - # - lookup('env', 'MAS_CONFIG_DIR') != "" # db2 support - # - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" # db2 support fail_msg: "One or more required environment variables are not defined" # - name: Check for required saas environment variables @@ -72,9 +67,9 @@ - ibm.mas_devops.sls - ibm.mas_devops.dro - ibm.mas_devops.db2 - # - ibm.mas_devops.minio - # - ibm.mas_devops.mariadb - # - ibm.mas_devops.odh - # - ibm.mas_devops.kmodels + - ibm.mas_devops.minio + - ibm.mas_devops.mariadb + - ibm.mas_devops.odh + - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker - # - ibm.mas_devops.aibroker_app_config + - ibm.mas_devops.aibroker_tenant \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 2110d7bf21..3a55ca97e3 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -11,14 +11,14 @@ kind: Namespace name: "{{ aibroker_namespace }}" -# Create config for SLS -- include_tasks: tasks/config_sls/main.yml +# # Create config for SLS +# - include_tasks: tasks/config_sls/main.yml -# Create config for DRO -- include_tasks: tasks/config_dro/main.yml +# # Create config for DRO +# - include_tasks: tasks/config_dro/main.yml -# Create config for DB2 -- include_tasks: tasks/config_db2/main.yml +# # Create config for DB2 +# - include_tasks: tasks/config_db2/main.yml # install AI Broker api - include_tasks: tasks/aibroker/main.yml diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml b/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml deleted file mode 100644 index af90664b8e..0000000000 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/main.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- name: Load variables (main) - include_vars: "vars/main.yml" - -# Create config for SLS -- include_tasks: tasks/config_sls/main.yml - -# Create config for DRO -- include_tasks: tasks/config_dro/main.yml - -# Create config for DRO -- include_tasks: tasks/config_db2/main.yml - -# Update CR -# - name: Update AibrokerApp CR -# kubernetes.core.k8s: -# state: patched -# api_version: apps.mas.ibm.com/v1 -# kind: AiBrokerApp -# name: "{{ mas_instance_id }}" -# namespace: "{{ aibroker_namespace }}" -# definition: -# spec: -# settings: -# sls: -# url: "{{ slscfg.url }}" -# registration_key: "{{ slscfg.registration_key }}" -# ca: "{{ slscfg.ca }}" -# dro: -# url: "{{ drocfg.url }}" -# registration_key: "{{ drocfg.registration_key }}" -# ca: "{{ drocfg.ca }}" -# jdbc: -# url: "{{ jdbccfg.url }}" -# username: "{{ jdbccfg.username }}" -# password: "{{ jdbccfg.password }}" -# ca: "{{ jdbccfg.ca }}" -# ignore_errors: yes diff --git a/ibm/mas_devops/roles/aibroker_app_config/README.md b/ibm/mas_devops/roles/aibroker_tenant/README.md similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/README.md rename to ibm/mas_devops/roles/aibroker_tenant/README.md diff --git a/ibm/mas_devops/roles/aibroker_app_config/defaults/main.yml b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml similarity index 74% rename from ibm/mas_devops/roles/aibroker_app_config/defaults/main.yml rename to ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml index 0b78b5f25c..fc4c738430 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml @@ -1,6 +1,6 @@ --- mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" -aibroker_namespace: 'mas-{{ mas_instance_id }}-aibroker' +aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" # Custom Labels @@ -9,8 +9,10 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # Source container registry # ----------------------------------------------------------------------------- -mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" -mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" +# mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +# mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" +icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" # MAS Entitlement # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/access_wx.py b/ibm/mas_devops/roles/aibroker_tenant/files/access_wx.py similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/access_wx.py rename to ibm/mas_devops/roles/aibroker_tenant/files/access_wx.py diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/alm_sample_sls_use.py rename to ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_apikey.sh b/ibm/mas_devops/roles/aibroker_tenant/files/create_apikey.sh similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_apikey.sh rename to ibm/mas_devops/roles/aibroker_tenant/files/create_apikey.sh diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_bucket.py b/ibm/mas_devops/roles/aibroker_tenant/files/create_bucket.py similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_bucket.py rename to ibm/mas_devops/roles/aibroker_tenant/files/create_bucket.py diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_dro_token.sh b/ibm/mas_devops/roles/aibroker_tenant/files/create_dro_token.sh similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_dro_token.sh rename to ibm/mas_devops/roles/aibroker_tenant/files/create_dro_token.sh diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_entitlement.py b/ibm/mas_devops/roles/aibroker_tenant/files/create_entitlement.py similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_entitlement.py rename to ibm/mas_devops/roles/aibroker_tenant/files/create_entitlement.py diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_s3.sh b/ibm/mas_devops/roles/aibroker_tenant/files/create_s3.sh similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_s3.sh rename to ibm/mas_devops/roles/aibroker_tenant/files/create_s3.sh diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_sls_secret.sh rename to ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_tenant.py b/ibm/mas_devops/roles/aibroker_tenant/files/create_tenant.py similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_tenant.py rename to ibm/mas_devops/roles/aibroker_tenant/files/create_tenant.py diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_tenant_entitlement.py b/ibm/mas_devops/roles/aibroker_tenant/files/create_tenant_entitlement.py similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_tenant_entitlement.py rename to ibm/mas_devops/roles/aibroker_tenant/files/create_tenant_entitlement.py diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/create_wx.sh b/ibm/mas_devops/roles/aibroker_tenant/files/create_wx.sh similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/create_wx.sh rename to ibm/mas_devops/roles/aibroker_tenant/files/create_wx.sh diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/delete_s3.py b/ibm/mas_devops/roles/aibroker_tenant/files/delete_s3.py similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/delete_s3.py rename to ibm/mas_devops/roles/aibroker_tenant/files/delete_s3.py diff --git a/ibm/mas_devops/roles/aibroker_app_config/files/generate_api_key.py b/ibm/mas_devops/roles/aibroker_tenant/files/generate_api_key.py similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/files/generate_api_key.py rename to ibm/mas_devops/roles/aibroker_tenant/files/generate_api_key.py diff --git a/ibm/mas_devops/roles/aibroker_app_config/meta/main.yml b/ibm/mas_devops/roles/aibroker_tenant/meta/main.yml similarity index 100% rename from ibm/mas_devops/roles/aibroker_app_config/meta/main.yml rename to ibm/mas_devops/roles/aibroker_tenant/meta/main.yml diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/aibroker/main.yml new file mode 100644 index 0000000000..ebe99c6e35 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/aibroker/main.yml @@ -0,0 +1,128 @@ +--- +# Provide Debug information +# ----------------------------------------------------------------------------- +- name: "Debug information - IBM Maximo AI Broker" + debug: + msg: + - "Namespace ...................... {{ aibroker_namespace }}" + - "Channel ........................ {{ mas_app_channel }}" + - "MAS Instance Id ................ {{ mas_instance_id }}" + +# # Create IBM Maximo IBM Maximo AI Broker +# # ----------------------------------------------------------------------------- +# - name: "Create IBM Maximo AI Broker namespace" +# kubernetes.core.k8s: +# state: present +# api_version: v1 +# kind: Namespace +# name: "{{ aibroker_namespace }}" + + +# Install the operator & create entitlement secret +# ----------------------------------------------------------------------------- +- name: "Create IBM Entitlement Key" + ibm.mas_devops.update_ibm_entitlement: + namespace: "{{ aibroker_namespace }}" + icr_username: "{{ mas_entitlement_username }}" + icr_password: "{{ mas_entitlement_key }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + +# - name: "Create ibm-mas-aibroker Subscription" +# ibm.mas_devops.apply_subscription: +# namespace: "{{ aibroker_namespace }}" +# package_name: ibm-mas-aibroker +# package_channel: "{{ aibroker_channel }}" +# catalog_source: "{{ mas_catalog_source }}" +# register: subscription + +- name: "Install AI Broker Operator Group" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/aibroker/operator-group.yml.j2') }}" + +- name: "Install AI Broker Operator" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/aibroker/subscription.yml.j2') }}" + +# Wait until the IBM Maximo AI Broker CRD is available +# ----------------------------------------------------------------------------- +- name: "Wait until the IBM Maximo AI Broker Operator CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: aibrokerapps.apps.mas.ibm.com + +# IBM Maximo AI Broker installation +# ----------------------------------------------------------------------------- +- name: Create AI Broker CR + vars: + annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}" + kubernetes.core.k8s: + state: present + namespace: "{{ aibroker_namespace }}" + template: templates/aibroker/aibrokerapp.yml.j2 + register: aibrokerAppResult + +# Wait IBM Maximo AI Broker custom resource to be complete +# ----------------------------------------------------------------------------- +- name: "Wait for AiBrokerApp custom resource to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_instance_id }}" + namespace: "{{ aibroker_namespace }}" + kind: AiBrokerApp + register: aibrokerapp_cr_result + until: + - aibrokerapp_cr_result.resources is defined and aibrokerapp_cr_result.resources | length == 1 + - aibrokerapp_cr_result.resources[0].status is defined + - aibrokerapp_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 + retries: 45 # approx 45 minutes before we give up + delay: 60 # 1 minute + +# IBM Maximo AI Broker workspace +# ----------------------------------------------------------------------------- +- name: Create AI Broker workspace CR + vars: + annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}" + kubernetes.core.k8s: + state: present + namespace: "{{ aibroker_namespace }}" + template: templates/aibroker/aibrokerworkspace.yml.j2 + register: aibrokerWorkspaceResult + +# Wait IBM Maximo AI Broker custom resource to be complete +# ----------------------------------------------------------------------------- +- name: "Wait for AiBrokerWorkspace custom resource to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_instance_id }}" + namespace: "{{ aibroker_namespace }}" + kind: AiBrokerWorkspace + register: aibrokerworkspace_cr_result + until: + - aibrokerworkspace_cr_result.resources is defined and aibrokerworkspace_cr_result.resources | length == 1 + - aibrokerworkspace_cr_result.resources[0].status is defined + - aibrokerworkspace_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 + retries: 45 # approx 45 minutes before we give up + delay: 60 # 1 minute + +- name: Await to aibroker-api pod created + pause: + minutes: 3 + prompt: "Waiting for aibroker-api pod" + +- name: "Wait for aibroker-api pod" + shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} + register: _is_not_exists + until: _is_not_exists.stdout|int == 0 + retries: 60 + delay: 30 + +- name: "Wait for aibroker-api pod to be ready (30s delay)" + shell: > + oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l + register: _is_not_ready + until: _is_not_ready.stdout|int == 0 + retries: 60 + delay: 30 diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/install/main.yml new file mode 100644 index 0000000000..a45f748433 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/install/main.yml @@ -0,0 +1,13 @@ +--- +- name: "Check if API key secret exists in namespace: {{ aibroker_namespace }}" + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "{{ tenantNamespace }}----apikey-secret" + namespace: "{{ aibroker_namespace }}" + register: apikey_secret_info + +- name: Create API Key + script: "{{ role_path }}/files/create_apikey.sh {{ tenantNamespace }} {{ aibroker_namespace }}" + when: + - apikey_secret_info.resources | length == 0 diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/main.yml new file mode 100644 index 0000000000..cefd059d41 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/main.yml @@ -0,0 +1,15 @@ +--- +- name: "API Key: Debug properties" + debug: + msg: + - "Api Key action ........................ {{ mas_aibroker_apikey_action }}" + +# 1. Perform the selected action +# ----------------------------------------------------------------------------- +# Two actions are supported: +# - install +# - remove +# - update +- include_tasks: "apikey/{{ mas_aibroker_apikey_action }}/main.yml" + when: + - mas_aibroker_apikey_action != "none" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/remove/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/remove/main.yml new file mode 100644 index 0000000000..cad8206ed5 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/remove/main.yml @@ -0,0 +1,6 @@ +--- +- name: Delete API key secret + command: oc delete secret "{{ tenantNamespace }}----apikey-secret" -n {{ aibroker_namespace }} + register: delete_result + ignore_errors: yes + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/update/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/update/main.yml new file mode 100644 index 0000000000..7c60abccc5 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/apikey/update/main.yml @@ -0,0 +1,16 @@ +--- +- name: Delete API Key Secret + command: oc delete secret "{{ tenantNamespace }}----apikey-secret" -n {{ aibroker_namespace }} + register: delete_result + ignore_errors: yes + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + +- name: Generate new API key + command: python3 ../roles/aibroker/files/generate_api_key.py + register: new_apikey + ignore_errors: yes + +- name: Create new API Key Secret + command: oc create secret generic "{{ tenantNamespace }}----apikey-secret" -n "{{ aibroker_namespace }}" --from-literal=AIBROKER_APIKEY="{{ new_apikey.stdout }}" + register: create_secret_result + ignore_errors: yes diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml similarity index 71% rename from ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml rename to ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml index 28876edafc..69ed491313 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml @@ -11,7 +11,8 @@ username: "{{ jdbccfg[0].data.username }}" password: "{{ jdbccfg[0].data.password }}" url: "{{ jdbccfg[1].spec.config.url }}" - ca: "{{ jdbccfg[1].spec.certificates }}" + # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" + ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" - name: "Debug: jdbc information" #TODO: remove before PR debug: @@ -20,3 +21,8 @@ - "JDBC password ............. {{ jdbccfg.password }}" - "JDBC url .................. {{ jdbccfg.url }}" - "JDBC cert ca .............. {{ jdbccfg.ca }}" + +- name: "Create secret for JDBC admin credentials" + kubernetes.core.k8s: + apply: yes + template: "templates/jdbc/jdbc-admin-credentials.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml similarity index 69% rename from ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml rename to ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml index 52608b1010..7a2a60a520 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml @@ -9,7 +9,8 @@ secret_name: "{{ drocfg[0].metadata.name }}" registration_key: "{{ drocfg[0].stringData.api_key }}" url: "{{ drocfg[1].spec.config.url }}" - ca: "{{ drocfg[1].spec.certificates }}" + ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" + # ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') }}" - name: "Debug: dro information" #TODO: remove before PR debug: @@ -18,3 +19,8 @@ - "dro api key ............. {{ drocfg.registration_key }}" - "dro url ................. {{ drocfg.url }}" - "dro cert ca ............. {{ drocfg.ca }}" + +- name: "Create secret for Dro registration key" + kubernetes.core.k8s: + apply: yes + template: "templates/dro/dro-token.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml similarity index 68% rename from ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml rename to ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml index f260373798..f32195ca35 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml @@ -9,7 +9,8 @@ secret_name: "{{ slscfg[0].metadata.name }}" registration_key: "{{ slscfg[0].stringData.registrationKey }}" url: "{{ slscfg[1].spec.config.url }}" - ca: "{{ slscfg[1].spec.certificates }}" + ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" + # ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') }}" - name: "Debug: sls information" #TODO: remove before PR debug: @@ -18,3 +19,8 @@ - "sls registration key .... {{ slscfg.registration_key }}" - "sls url ................. {{ slscfg.url }}" - "sls cert ca ............. {{ slscfg.ca }}" + +- name: "Create secret for SLS registration key" + kubernetes.core.k8s: + apply: yes + template: "templates/sls/sls-tenant-details.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml new file mode 100644 index 0000000000..9abf73d078 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: Load variables (main) + include_vars: "vars/main.yml" + +# # Create IBM Maximo IBM Maximo AI Broker +# # ----------------------------------------------------------------------------- +# - name: "Create IBM Maximo AI Broker namespace" +# kubernetes.core.k8s: +# state: present +# api_version: v1 +# kind: Namespace +# name: "{{ aibroker_namespace }}" + +# Create config for SLS +- include_tasks: tasks/config_sls/main.yml + +# Create config for DRO +- include_tasks: tasks/config_dro/main.yml + +# Create config for DB2 +- include_tasks: tasks/config_db2/main.yml + +# # install AI Broker api +# - include_tasks: tasks/aibroker/main.yml + +# run api calls for aibroker saas +- include_tasks: tasks/saas/main.yml + when: mas_aibroker_saas == true + +# create AI Broker tenant +- include_tasks: tasks/tenant/main.yml + +# create AI Broker api key +- include_tasks: tasks/apikey/main.yml + +# create s3 api key +- include_tasks: tasks/s3/main.yml + +# create wx api +- include_tasks: tasks/watsonx/main.yml diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/install/main.yml new file mode 100644 index 0000000000..a5662de1ad --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/install/main.yml @@ -0,0 +1,14 @@ +--- +- name: "Check if S3 key secret exists in namespace: {{ aibroker_namespace }}" + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "{{ tenantNamespace }}----s3-secret" + namespace: "{{ aibroker_namespace }}" + register: s3_apikey_secret_info + +- name: Create S3 API Key + script: "{{ role_path }}/files/create_s3.sh {{ tenantNamespace }} {{ aibroker_namespace }} {{ mas_aibroker_storage_host }} {{ mas_aibroker_storage_accesskey }} {{ mas_aibroker_storage_secretkey }} {{ mas_aibroker_storage_provider }} {{ mas_aibroker_storage_port }} {{ mas_aibroker_storage_region }}" + when: + - s3_apikey_secret_info.resources | length == 0 + register: script_output diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/main.yml new file mode 100644 index 0000000000..0169f649c8 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/main.yml @@ -0,0 +1,14 @@ +--- +- name: "S3: Debug properties" + debug: + msg: + - "S3 action ........................ {{ mas_aibroker_s3_action }}" + +# 1. Perform the selected action +# ----------------------------------------------------------------------------- +# Two actions are supported: +# - install +# - remove +- include_tasks: "s3/{{ mas_aibroker_s3_action }}/main.yml" + when: + - mas_aibroker_s3_action != "none" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/remove/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/remove/main.yml new file mode 100644 index 0000000000..927a909810 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/s3/remove/main.yml @@ -0,0 +1,55 @@ +--- +- name: "Check if S3 key secret exists in namespace: {{ aibroker_namespace }}" + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "{{ tenantNamespace }}----s3-secret" + namespace: "{{ aibroker_namespace }}" + register: s3_apikey_secret_info + +- name: Get secret value + command: oc get secret "{{ tenantNamespace }}----s3-secret" -n "{{ aibroker_namespace }}" -o jsonpath='{.data.BUCKET-NAME}' + register: secret_value + when: + - s3_apikey_secret_info.resources | length != 0 + +- name: Decode secret value + shell: "echo '{{ secret_value.stdout | default('') }}' | base64 -d" + register: decoded_secret_value + changed_when: false + +- name: Create AWS credentials directory + file: + path: "{{ ansible_env.HOME }}/.aws" + state: directory + mode: "0700" + +- name: Create AWS credentials file + copy: + dest: "{{ ansible_env.HOME }}/.aws/credentials" + content: | + [default] + aws_access_key_id = {{ mas_aibroker_storage_accesskey }} + aws_secret_access_key = {{ mas_aibroker_storage_secretkey }} + mode: "0600" + +- name: Create AWS config file + copy: + dest: "{{ ansible_env.HOME }}/.aws/config" + content: | + [default] + region = {{ mas_aibroker_storage_region }} + mode: "0600" + +- name: Delete S3 bucket if script succeeds + command: python3 {{ role_path }}/files/delete_s3.py "{{ decoded_secret_value.stdout }}" + register: delete_result + ignore_errors: yes + when: + - s3_apikey_secret_info.resources | length != 0 + +- name: Delete secret if bucket deleted successfully + command: oc delete secret "{{ tenantNamespace }}----s3-secret" -n "{{ aibroker_namespace }}" + register: secret_delete_result + when: + - s3_apikey_secret_info.resources | length != 0 diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/saas/main.yml new file mode 100644 index 0000000000..49b80c6dc6 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/saas/main.yml @@ -0,0 +1,25 @@ +--- +- name: "Check if API key secret exists in namespace: {{ mas_aibroker_provision_tenant }}" + k8s_info: + api_version: v1 + kind: Secret + name: "{{ mas_aibroker_provision_tenant }}----apikey-secret" + namespace: "{{ aibroker_namespace }}" + register: apikey_secret_info + +- name: Create API Key + script: "{{ role_path }}/files/create_apikey.sh {{ mas_aibroker_provision_tenant }} {{ aibroker_namespace }}" + when: + - apikey_secret_info.resources | length == 0 + +- name: "Lookup tenant {{ mas_aibroker_provision_tenant }} api key" + shell: oc get secret {{ mas_aibroker_provision_tenant }}----apikey-secret -n {{ aibroker_namespace }} -o json | jq -r '.data.AIBROKER_APIKEY' | base64 -d + register: provision_tenant_api_key_output + +- name: "Set aibroker api key variable" + set_fact: + provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" + +- name: Debug {{ mas_aibroker_provision_tenant }} api key #TODO: remove on final commit + debug: + msg: "Provision tenant api key: {{ provision_tenant_api_key }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml new file mode 100644 index 0000000000..41ec997e4e --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -0,0 +1,197 @@ +--- +- name: "Check if tenant namespace: {{ tenantNamespace }} exists" + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + name: "{{ tenantNamespace }}" + register: namespace_info + +# Create tenant namespace +- name: "Create a tenant namespace: {{ tenantNamespace }}" + kubernetes.core.k8s: + name: "{{ tenantNamespace }}" + api_version: v1 + kind: Namespace + when: + - namespace_info.resources | length == 0 + +# Copy secrets to tenant namespace +- name: "Copy secrets to namespace: {{ tenantNamespace }}" + shell: 'oc get secret {{ item }} -n mas-{{ mas_instance_id }}-aibroker -o yaml | sed "s/namespace: .*/namespace: {{ tenantNamespace }}/" | oc apply --force -f -' + with_items: + - km-s3-secret + - "{{ pullSecretName }}" + +# Adding inference server runtimes +- name: "Create config map for connector config" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/connector-configmap.yml.j2') }}" + +- name: "Create SA for server runtimes" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/km-s3-sa.yml.j2') }}" + +- name: "Create huggingfaceserver server runtime" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/kserve-huggingfaceserver.yml.j2') }}" + +- name: "Create lgbserver server runtime" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/kserve-lgbserver.yml.j2') }}" + +- name: "Create sklearnserver server runtime" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/kserve-sklearnserver.yml.j2') }}" + +- name: "Create xgbserver server runtime" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/kserve-xgbserver.yml.j2') }}" + +# Applay RBAC roles +- name: "Apply RBAC for tenant" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/rbac.yml.j2') }}" + +- name: "Check if SLS key secret exists in namespace: {{ aibroker_namespace }}" + k8s_info: + api_version: v1 + kind: Secret + name: "{{ tenantNamespace }}----sls-secret" + namespace: "{{ aibroker_namespace }}" + register: sls_secret_info + +- name: Create SLS secret + script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} {{ mas_aibroker_path_ca_crt }}" + when: + - sls_secret_info.resources | length == 0 + register: sls_secret_output + +- name: Debug Create sls secret for AI Broker + debug: + msg: "{{ sls_secret_output }}" + +- name: Delete DRO secret + command: oc delete secret "{{ tenantNamespace }}----dro-secret" -n {{ aibroker_namespace }} + register: delete_result + ignore_errors: yes + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + +- name: Create DRO token + script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_dro_token }}" + register: dro_token_output + +- name: Debug Create DRO token + debug: + msg: "{{ dro_token_output }}" + +# - name: Wait for service /service/rest/api/v1/tenant #need more accurate way to check if service working +# pause: +# minutes: 8 +# prompt: "Waiting for for service /service/rest/api/v1/tenant" + +- name: Await to aibroker-api pod created + pause: + minutes: 3 + prompt: "Waiting for aibroker-api pod" + +- name: "Wait for aibroker-api pod" + shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} + register: _is_not_exists + until: _is_not_exists.stdout|int == 0 + retries: 60 + delay: 30 + +- name: "Wait for aibroker-api pod to be ready (30s delay)" + shell: > + oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l + register: _is_not_ready + until: _is_not_ready.stdout|int == 0 + retries: 60 + delay: 30 + +- name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri + uri: + url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" + validate_certs: no + method: POST + headers: + apikey: "{{ provision_tenant_api_key }}" + accept: "*/*" + Content-Type: "application/json" + tenantid: "{{ mas_aibroker_provision_tenant }}" + body_format: "json" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"sls_url\": \"{{ mas_aibroker_sls_url }}\", \"dro_url\": \"{{ mas_aibroker_dro_url }}\"}" + status_code: [201] + timeout: 30 + register: add_tenant_to_db2 + +- name: Debug tenant #TODO: remove on final commit + debug: + msg: "{{ add_tenant_to_db2 }}" + +- name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri + uri: + url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + validate_certs: no + method: POST + headers: + apikey: "{{ provision_tenant_api_key }}" + accept: "*/*" + Content-Type: "application/json" + tenantid: "{{ mas_aibroker_provision_tenant }}" + body_format: "json" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"similarity\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" + status_code: [201] + timeout: 30 + register: add_tenant_entitlements_to_db2 + +- name: Debug tenant entitlement #TODO: remove on final commit + debug: + msg: "{{ add_tenant_entitlements_to_db2 }}" + +- name: Create tenant {{ tenantNamespace }} and add to db2 for pcc - using uri + uri: + url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + validate_certs: no + method: POST + headers: + apikey: "{{ provision_tenant_api_key }}" + accept: "*/*" + Content-Type: "application/json" + tenantid: "{{ mas_aibroker_provision_tenant }}" + body_format: "json" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"pcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" + status_code: [201] + timeout: 30 + register: add_tenant_entitlements_to_db2_pcc + +- name: Debug tenant entitlement #TODO: remove on final commit + debug: + msg: "{{ add_tenant_entitlements_to_db2_pcc }}" + +- name: Create tenant {{ tenantNamespace }} and add to db2 for mcc - using uri + uri: + url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + validate_certs: no + method: POST + headers: + apikey: "{{ provision_tenant_api_key }}" + accept: "*/*" + Content-Type: "application/json" + tenantid: "{{ mas_aibroker_provision_tenant }}" + body_format: "json" + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"mcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" + status_code: [201] + timeout: 30 + register: add_tenant_entitlements_to_db2_mcc + +- name: Debug tenant entitlement #TODO: remove on final commit + debug: + msg: "{{ add_tenant_entitlements_to_db2_mcc }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/main.yml new file mode 100644 index 0000000000..2161a6a02b --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/main.yml @@ -0,0 +1,14 @@ +--- +- name: "Tenant: Debug properties" + debug: + msg: + - "Tenant action ........................ {{ tenant_action }}" + +# 1. Perform the selected action +# ----------------------------------------------------------------------------- +# Two actions are supported: +# - install + +- include_tasks: "tenant/{{ tenant_action }}/main.yml" + when: + - tenant_action != "none" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/remove/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/remove/main.yml new file mode 100644 index 0000000000..db88c99c5c --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/remove/main.yml @@ -0,0 +1,50 @@ +--- +# Tasks to remove tenant resources +- name: Check if models exist before deleting kmodels + shell: oc get isvc,ig -n {{ tenantNamespace }} -o name + register: result + +- name: Check if any models exist + fail: + msg: "Please remove all models before deleting kmodels" + when: result.rc == 0 and result.stdout != '' + +- name: Delete monitor-config ConfigMap + shell: oc delete cm monitor-config -n {{ tenantNamespace }} + ignore_errors: true + register: delete_result + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + +- name: Delete connector-config ConfigMap + shell: oc delete cm connector-config -n {{ tenantNamespace }} + ignore_errors: true + register: delete_result + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + +- name: Delete km-s3-sa ServiceAccount + shell: oc delete sa km-s3-sa -n {{ tenantNamespace }} + ignore_errors: true + register: delete_result + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + +- name: Delete km-s3-secret Secret + shell: oc delete secret km-s3-secret -n {{ tenantNamespace }} + ignore_errors: true + register: delete_result + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + +- name: Delete regcred Secret + shell: oc delete secret regcred -n {{ tenantNamespace }} + ignore_errors: true + register: delete_result + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + +- name: Delete tenant namespace + shell: oc delete ns {{ tenantNamespace }} + ignore_errors: true + register: delete_result + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr + +- name: Tenant status + debug: + msg: "Tenant was deleted" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/install/main.yml new file mode 100644 index 0000000000..3476be827b --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/install/main.yml @@ -0,0 +1,19 @@ +--- +- name: "Check if WatsonX AI API key secret exists in namespace: {{ aibroker_namespace }}" + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "{{ tenantNamespace }}----wx-secret" + namespace: "{{ aibroker_namespace }}" + register: wx_apikey_secret_info + +- name: Create WatsonX AI API Key + script: "{{ role_path }}/files/create_wx.sh {{ tenantNamespace }} {{ aibroker_namespace }} {{ mas_aibroker_watsonxai_apikey }} {{ mas_aibroker_watsonxai_url }} {{ mas_aibroker_watsonxai_project_id }}" + when: + - wx_apikey_secret_info.resources | length == 0 + +# Copy secrets to tenant namespace +- name: 'Copy secret {{ tenantNamespace }}----wx-secret to namespace: {{ tenantNamespace }}' + shell: 'oc get secret {{ item }} -n mas-{{ mas_instance_id }}-aibroker -o yaml | sed "s/namespace: .*/namespace: {{ tenantNamespace }}/" | oc apply --force -f -' + with_items: + - '{{ tenantNamespace }}----wx-secret' diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/main.yml new file mode 100644 index 0000000000..2418e3193e --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/main.yml @@ -0,0 +1,14 @@ +--- +- name: "Watsonx: Debug properties" + debug: + msg: + - "Watsonx action ........................ {{ mas_aibroker_watsonx_action }}" + +# 1. Perform the selected action +# ----------------------------------------------------------------------------- +# Two actions are supported: +# - install +# - remove +- include_tasks: "watsonx/{{ mas_aibroker_watsonx_action }}/main.yml" + when: + - mas_aibroker_watsonx_action != "none" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/remove/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/remove/main.yml new file mode 100644 index 0000000000..fadd539752 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/watsonx/remove/main.yml @@ -0,0 +1,6 @@ +--- +- name: Delete WatsonX.AI secret + command: oc delete secret "{{ tenantNamespace }}----wx-secret" -n {{ aibroker_namespace }} + register: delete_result + ignore_errors: yes + failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerapp.yml.j2 new file mode 100644 index 0000000000..889ab3929e --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerapp.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: apps.mas.ibm.com/v1 +kind: AiBrokerApp +metadata: + name: "{{ mas_instance_id }}" + namespace: "{{ aibroker_namespace }}" + labels: + mas.ibm.com/applicationId: aibroker + mas.ibm.com/instanceId: "{{ mas_instance_id }}" +spec: + settings: + icr: + cp: "{{ mas_icr_cp }}" + cpopen: "{{ mas_icr_cpopen }}" + dro: + url: "{{ drocfg.url }}" + dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" + ca: "{{ drocfg.ca }}" + jdbc: + url: "{{ jdbccfg.url }}" + credentials_secret: "{{ mas_aibroker_jdbc_secret }}" + ca: "{{ jdbccfg.ca }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 new file mode 100644 index 0000000000..0cb94a3c28 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -0,0 +1,30 @@ +--- +apiVersion: apps.mas.ibm.com/v1 +kind: AiBrokerWorkspace +metadata: + name: "{{ mas_instance_id }}" + namespace: "{{ aibroker_namespace }}" + labels: + mas.ibm.com/applicationId: aibroker + mas.ibm.com/instanceId: "{{ mas_instance_id }}" +spec: + settings: + icr: + cp: "{{ mas_icr_cp }}" + cpopen: "{{ mas_icr_cpopen }}" + dro: + url: "{{ drocfg.url }}" + dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" + ca: "{{ drocfg.ca }}" + jdbc: + url: "{{ jdbccfg.url }}" + credentials_secret: "{{ mas_aibroker_jdbc_secret }}" + ca: "{{ jdbccfg.ca }}" + sls: + url: "{{ slscfg.url }}" + sls_regitration_key_secret: "{{ slscfg.registration_key }}" + ca: "{{ slscfg.ca }}" + tenant: + tenant_id: "{{ tenantNamespace }}" + subscription_id: "{{ mas_aibroker_sls_subscription_id }}" + instance_id: "{{ mas_instance_id }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/operator-group.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/operator-group.yml.j2 new file mode 100644 index 0000000000..5aab10febd --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/operator-group.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: mas-{{ mas_instance_id }}-aibroker + namespace: "{{ aibroker_namespace }}" +spec: + targetNamespaces: + - "{{ aibroker_namespace }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/subscription.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/subscription.yml.j2 new file mode 100644 index 0000000000..dc7a429078 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/subscription.yml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: ibm-mas-aibroker + namespace: "{{ aibroker_namespace }}" +spec: + channel: "{{ mas_app_channel }}" + installPlanApproval: Automatic + name: ibm-mas-aibroker + source: "{{ mas_catalog_source }}" + sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-token.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-token.yml.j2 new file mode 100644 index 0000000000..e6684c1714 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-token.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ mas_aibroker_dro_token_secret }} + namespace: mas-{{ mas_instance_id }}-aibroker +data: + DRO_TOKEN: "{{ drocfg.registration_key | b64encode }}" +type: Opaque \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/jdbc/jdbc-admin-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/jdbc/jdbc-admin-credentials.yml.j2 new file mode 100644 index 0000000000..7e0ac3bf5f --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/jdbc/jdbc-admin-credentials.yml.j2 @@ -0,0 +1,10 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ mas_aibroker_jdbc_secret }} + namespace: mas-{{ mas_instance_id }}-aibroker +data: + username: "{{ jdbccfg.username }}" + password: "{{ jdbccfg.password | b64encode }}" +type: Opaque diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-tenant-details.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-tenant-details.yml.j2 new file mode 100644 index 0000000000..35f6285706 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-tenant-details.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ mas_aibroker_sls_registration_key_secret }} + namespace: mas-{{ mas_instance_id }}-aibroker +data: + SLS_REGISTRATION_KEY: "{{ slscfg.registration_key | b64encode }}" +type: Opaque diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/connector-configmap.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/connector-configmap.yml.j2 new file mode 100644 index 0000000000..a4fdee577d --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/connector-configmap.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: connector-config + namespace: mas-{{ mas_instance_id }}-aibroker +data: {} diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/km-s3-sa.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/km-s3-sa.yml.j2 new file mode 100644 index 0000000000..db2d7a4edf --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/km-s3-sa.yml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: km-s3-sa + namespace: "{{ tenantNamespace }}" +secrets: + - name: km-s3-secret diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-huggingfaceserver.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-huggingfaceserver.yml.j2 new file mode 100644 index 0000000000..1ddc436b8e --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-huggingfaceserver.yml.j2 @@ -0,0 +1,33 @@ +--- +apiVersion: serving.kserve.io/v1alpha1 +kind: ServingRuntime +metadata: + name: kserve-huggingfaceserver + namespace: "{{ tenantNamespace }}" +spec: + annotations: + prometheus.kserve.io/port: "8080" + prometheus.kserve.io/path: "/metrics" + supportedModelFormats: + - name: huggingface + version: "1" + autoSelect: true + priority: 1 + protocolVersions: + - v2 + - v1 + containers: + - name: kserve-container + image: kserve/huggingfaceserver:latest + args: + - '--model_name={{".Name"}}' + env: + - name: HUGGINGFACE_HUB_CACHE + value: /tmp/huggingface/hub + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "1" + memory: 2Gi diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-lgbserver.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-lgbserver.yml.j2 new file mode 100644 index 0000000000..a79769f1c5 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-lgbserver.yml.j2 @@ -0,0 +1,33 @@ +--- +apiVersion: serving.kserve.io/v1alpha1 +kind: ServingRuntime +metadata: + name: kserve-lgbserver + namespace: "{{ tenantNamespace }}" +spec: + annotations: + prometheus.kserve.io/port: "8080" + prometheus.kserve.io/path: "/metrics" + supportedModelFormats: + - name: lightgbm + version: "3" + autoSelect: true + priority: 1 + protocolVersions: + - v1 + - v2 + containers: + - name: kserve-container + image: kserve/lgbserver:latest + args: + - '--model_name={{".Name"}}' + - --model_dir=/mnt/models + - --http_port=8080 + - --nthread=1 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "1" + memory: 2Gi diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-sklearnserver.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-sklearnserver.yml.j2 new file mode 100644 index 0000000000..2a05ee1fc8 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-sklearnserver.yml.j2 @@ -0,0 +1,32 @@ +--- +apiVersion: serving.kserve.io/v1alpha1 +kind: ServingRuntime +metadata: + name: kserve-sklearnserver + namespace: "{{ tenantNamespace }}" +spec: + annotations: + prometheus.kserve.io/port: "8080" + prometheus.kserve.io/path: "/metrics" + supportedModelFormats: + - name: sklearn + version: "1" + autoSelect: true + priority: 1 + protocolVersions: + - v1 + - v2 + containers: + - name: kserve-container + image: kserve/sklearnserver:latest + args: + - '--model_name={{".Name"}}' + - --model_dir=/mnt/models + - --http_port=8080 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "1" + memory: 2Gi diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-xgbserver.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-xgbserver.yml.j2 new file mode 100644 index 0000000000..daec36ac28 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-xgbserver.yml.j2 @@ -0,0 +1,33 @@ +--- +apiVersion: serving.kserve.io/v1alpha1 +kind: ServingRuntime +metadata: + name: kserve-xgbserver + namespace: "{{ tenantNamespace }}" +spec: + annotations: + prometheus.kserve.io/port: "8080" + prometheus.kserve.io/path: "/metrics" + supportedModelFormats: + - name: xgboost + version: "1" + autoSelect: true + priority: 1 + protocolVersions: + - v1 + - v2 + containers: + - name: kserve-container + image: kserve/xgbserver:latest + args: + - '--model_name={{".Name"}}' + - --model_dir=/mnt/models + - --http_port=8080 + - --nthread=1 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "1" + memory: 2Gi diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/rbac.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/rbac.yml.j2 new file mode 100644 index 0000000000..fa28085ab0 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/rbac.yml.j2 @@ -0,0 +1,104 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: km-controller-role + namespace: {{ tenantNamespace }} +rules: + - verbs: + - list + apiGroups: + - '' + resources: + - pods + - configmaps + - secrets + - verbs: + - delete + apiGroups: + - '' + resources: + - pods + - verbs: + - list + apiGroups: + - metrics.k8s.io + resources: + - nodes + - pods + - verbs: + - list + apiGroups: + - apps + resources: + - deployments + - verbs: + - list + - create + - delete + apiGroups: + - batch + resources: + - jobs + - cronjobs + - verbs: + - create + - list + - delete + - get + - patch + apiGroups: + - serving.kserve.io + resources: + - inferenceservices +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: km-controller-binding + namespace: {{ tenantNamespace }} +subjects: + - kind: ServiceAccount + name: km-controller + namespace: mas-{{ mas_instance_id }}-aibroker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: km-controller-role +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: km-watcher-role + namespace: {{ tenantNamespace }} +rules: + - verbs: + - list + - watch + apiGroups: + - '' + resources: + - events + - pods + - verbs: + - list + - watch + apiGroups: + - batch + resources: + - cronjobs + - jobs +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: km-watcher-binding + namespace: {{ tenantNamespace }} +subjects: + - kind: ServiceAccount + name: km-watcher + namespace: mas-{{ mas_instance_id }}-aibroker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: km-watcher-role diff --git a/ibm/mas_devops/roles/aibroker_app_config/vars/main.yml b/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml similarity index 91% rename from ibm/mas_devops/roles/aibroker_app_config/vars/main.yml rename to ibm/mas_devops/roles/aibroker_tenant/vars/main.yml index 29d86e90d3..defda35239 100644 --- a/ibm/mas_devops/roles/aibroker_app_config/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml @@ -48,3 +48,13 @@ mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRAT mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" mas_aibroker_path_ca_crt: './certs' mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" + +# DRO +mas_aibroker_dro_token_secret: "dro-token" + +# JDBC +mas_aibroker_jdbc_secret: "jdbc-admin-credentials" + +# SLS +mas_aibroker_sls_registration_key_secret: "sls-registration-key" +mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" From 613a943bde968c470df3d05512662d695be4ec23 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 14 Mar 2025 14:15:08 +0000 Subject: [PATCH 126/169] [patch] fix new-line-at-end-of-file --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index b407f05516..6c6e9fbc99 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -72,4 +72,4 @@ - ibm.mas_devops.odh - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker - - ibm.mas_devops.aibroker_tenant \ No newline at end of file + - ibm.mas_devops.aibroker_tenant From 39b974f3e525531dabc28522774b6b39c77431c8 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Tue, 18 Mar 2025 15:38:16 +0530 Subject: [PATCH 127/169] [patch] Adding jdbccfg, dro, certificate and truststore task (#1683) --- .../roles/aibroker/tasks/config_db2/main.yml | 8 +- .../roles/aibroker/tasks/config_dro/main.yml | 8 +- ibm/mas_devops/roles/aibroker/tasks/main.yml | 33 ++++++-- .../templates/aibroker/aibrokerapp.yml.j2 | 6 +- .../aibroker/aibrokerworkspace.yml.j2 | 4 +- .../aibroker/templates/dro/dro-ca-cert.yml.j2 | 9 +++ .../jdbc/jdbc-admin-credentials.yml.j2 | 9 ++- ibm/mas_devops/roles/aibroker/vars/main.yml | 3 +- ibm/mas_devops/roles/mariadb/README.md | 68 ++++++++++++++++ .../roles/mariadb/defaults/main.yml | 27 +++++++ ibm/mas_devops/roles/mariadb/meta/main.yml | 21 +++++ .../tasks/determine-storage-classes.yml | 20 +++++ ibm/mas_devops/roles/mariadb/tasks/main.yml | 49 ++++++++++++ .../mariadb/mariadb-deployment.yml.j2 | 79 +++++++++++++++++++ .../templates/mariadb/mariadb-np.yml.j2 | 38 +++++++++ .../templates/mariadb/mariadb-ns.yml.j2 | 5 ++ .../templates/mariadb/mariadb-pvc.yml.j2 | 16 ++++ .../templates/mariadb/mariadb-sa.yml.j2 | 9 +++ .../templates/mariadb/mariadb-secret.yml.j2 | 9 +++ .../templates/mariadb/mariadb-service.yml.j2 | 18 +++++ ibm/mas_devops/roles/mariadb/vars/main.yml | 1 + ibm/mas_devops/roles/minio/README.md | 53 +++++++++++++ ibm/mas_devops/roles/minio/defaults/main.yml | 26 ++++++ ibm/mas_devops/roles/minio/meta/main.yml | 21 +++++ .../minio/tasks/determine-storage-classes.yml | 20 +++++ ibm/mas_devops/roles/minio/tasks/main.yml | 37 +++++++++ .../templates/minio/minio-deployment.yml.j2 | 35 ++++++++ .../minio/templates/minio/minio-ns.yml.j2 | 5 ++ .../minio/templates/minio/minio-pvc.yml.j2 | 14 ++++ .../minio/templates/minio/minio-route.yml.j2 | 16 ++++ .../templates/minio/minio-service.yml.j2 | 19 +++++ ibm/mas_devops/roles/minio/vars/main.yml | 1 + 32 files changed, 667 insertions(+), 20 deletions(-) create mode 100644 ibm/mas_devops/roles/aibroker/templates/dro/dro-ca-cert.yml.j2 create mode 100644 ibm/mas_devops/roles/mariadb/README.md create mode 100644 ibm/mas_devops/roles/mariadb/defaults/main.yml create mode 100644 ibm/mas_devops/roles/mariadb/meta/main.yml create mode 100644 ibm/mas_devops/roles/mariadb/tasks/determine-storage-classes.yml create mode 100644 ibm/mas_devops/roles/mariadb/tasks/main.yml create mode 100644 ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-deployment.yml.j2 create mode 100644 ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-np.yml.j2 create mode 100644 ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-ns.yml.j2 create mode 100644 ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-pvc.yml.j2 create mode 100644 ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-sa.yml.j2 create mode 100644 ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-secret.yml.j2 create mode 100644 ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-service.yml.j2 create mode 100644 ibm/mas_devops/roles/mariadb/vars/main.yml create mode 100644 ibm/mas_devops/roles/minio/README.md create mode 100644 ibm/mas_devops/roles/minio/defaults/main.yml create mode 100644 ibm/mas_devops/roles/minio/meta/main.yml create mode 100644 ibm/mas_devops/roles/minio/tasks/determine-storage-classes.yml create mode 100644 ibm/mas_devops/roles/minio/tasks/main.yml create mode 100644 ibm/mas_devops/roles/minio/templates/minio/minio-deployment.yml.j2 create mode 100644 ibm/mas_devops/roles/minio/templates/minio/minio-ns.yml.j2 create mode 100644 ibm/mas_devops/roles/minio/templates/minio/minio-pvc.yml.j2 create mode 100644 ibm/mas_devops/roles/minio/templates/minio/minio-route.yml.j2 create mode 100644 ibm/mas_devops/roles/minio/templates/minio/minio-service.yml.j2 create mode 100644 ibm/mas_devops/roles/minio/vars/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml index 69ed491313..88a8adf973 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -6,13 +6,11 @@ - name: "Set dro mandatory vars" ansible.builtin.set_fact: jdbccfg: - # username: "{{ jdbccfg[0].data.username | b64decode}}" - # password: "{{ jdbccfg[0].data.password | b64decode}}" username: "{{ jdbccfg[0].data.username }}" password: "{{ jdbccfg[0].data.password }}" - url: "{{ jdbccfg[1].spec.config.url }}" - # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" - ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" + url: "{{ jdbccfg[1].spec.config.url | b64encode }}" + sslenabled: "{{ jdbccfg[1].spec.config.sslEnabled | b64encode }}" + ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') | b64encode }}" - name: "Debug: jdbc information" #TODO: remove before PR debug: diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml index 7a2a60a520..5ac7e1d06e 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml @@ -9,8 +9,7 @@ secret_name: "{{ drocfg[0].metadata.name }}" registration_key: "{{ drocfg[0].stringData.api_key }}" url: "{{ drocfg[1].spec.config.url }}" - ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" - # ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') }}" + ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') | regex_replace('\\n\\s*\\n', '\\n') }}" - name: "Debug: dro information" #TODO: remove before PR debug: @@ -24,3 +23,8 @@ kubernetes.core.k8s: apply: yes template: "templates/dro/dro-token.yml.j2" + +- name: "Create secret for Dro CA Cert" + kubernetes.core.k8s: + apply: yes + template: "templates/dro/dro-ca-cert.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 3a55ca97e3..4ff9e365f9 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -2,6 +2,27 @@ - name: Load variables (main) include_vars: "vars/main.yml" +# Get cluster domain +# ----------------------------------------------------------------------------- +- name: "Get cluster domain" + kubernetes.core.k8s_info: + api_version: config.openshift.io/v1 + kind: DNS + name: cluster + register: _cluster_dns + +- name: "Set fact: cluster domain" + set_fact: + cluster_domain: "{{ _cluster_dns.resources[0].spec.baseDomain }}" + +- name: "Debug: cluster domain" + debug: + msg: "Cluster domain ........................ {{ cluster_domain }}" + +- name: "Set fact: cluster domain" + set_fact: + in_saas_env: "{{ lookup('env', 'INSAASENV') | default('false', true) | bool }}" + # Create IBM Maximo IBM Maximo AI Broker # ----------------------------------------------------------------------------- - name: "Create IBM Maximo AI Broker namespace" @@ -11,14 +32,14 @@ kind: Namespace name: "{{ aibroker_namespace }}" -# # Create config for SLS -# - include_tasks: tasks/config_sls/main.yml +# Create config for SLS +- include_tasks: tasks/config_sls/main.yml -# # Create config for DRO -# - include_tasks: tasks/config_dro/main.yml +# Create config for DRO +- include_tasks: tasks/config_dro/main.yml -# # Create config for DB2 -# - include_tasks: tasks/config_db2/main.yml +# Create config for DB2 +- include_tasks: tasks/config_db2/main.yml # install AI Broker api - include_tasks: tasks/aibroker/main.yml diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index 889ab3929e..c108a8e1ec 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -17,6 +17,8 @@ spec: dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" ca: "{{ drocfg.ca }}" jdbc: - url: "{{ jdbccfg.url }}" + url: "{{ jdbccfg.url | b64decode }}" credentials_secret: "{{ mas_aibroker_jdbc_secret }}" - ca: "{{ jdbccfg.ca }}" + ca: "{{ jdbccfg.ca | b64decode }}" + cluster_domain: "{{ cluster_domain }}" + in_saas_env: "{{ in_saas_env }}" diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 index 0cb94a3c28..cdbba6ff91 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 @@ -27,4 +27,6 @@ spec: tenant: tenant_id: "{{ tenantNamespace }}" subscription_id: "{{ mas_aibroker_sls_subscription_id }}" - instance_id: "{{ mas_instance_id }}" \ No newline at end of file + instance_id: "{{ mas_instance_id }}" + cluster_domain: "{{ cluster_domain }}" + in_saas_env: "{{ in_saas_env }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/templates/dro/dro-ca-cert.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/dro/dro-ca-cert.yml.j2 new file mode 100644 index 0000000000..55b674742b --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/templates/dro/dro-ca-cert.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ mas_aibroker_dro_cacert_secret }} + namespace: mas-{{ mas_instance_id }}-aibroker +data: + ca.crt: "{{ drocfg.ca | b64encode }}" +type: Opaque \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 index 7e0ac3bf5f..1b16e4ec8f 100644 --- a/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/jdbc/jdbc-admin-credentials.yml.j2 @@ -2,9 +2,12 @@ kind: Secret apiVersion: v1 metadata: - name: {{ mas_aibroker_jdbc_secret }} + name: "{{ mas_aibroker_jdbc_secret }}" namespace: mas-{{ mas_instance_id }}-aibroker data: username: "{{ jdbccfg.username }}" - password: "{{ jdbccfg.password | b64encode }}" -type: Opaque + password: "{{ jdbccfg.password }}" + url: "{{ jdbccfg.url }}" + sslenabled: "{{ jdbccfg.sslenabled }}" + certificate: "{{ jdbccfg.ca }}" +type: Opaque \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index defda35239..c17fa30044 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -51,9 +51,10 @@ mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT' # DRO mas_aibroker_dro_token_secret: "dro-token" +mas_aibroker_dro_cacert_secret: "dro-certificates" # JDBC -mas_aibroker_jdbc_secret: "jdbc-admin-credentials" +mas_aibroker_jdbc_secret: "{{ mas_app_id }}-jdbccfg" # SLS mas_aibroker_sls_registration_key_secret: "sls-registration-key" diff --git a/ibm/mas_devops/roles/mariadb/README.md b/ibm/mas_devops/roles/mariadb/README.md new file mode 100644 index 0000000000..6afc6ee3f4 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/README.md @@ -0,0 +1,68 @@ +# Mariadb +===== + +This role provides support to install and configure MariaDB + +* Install MariaDB + +Role Variables +-------------- + +### MAS core instance +Action to be performed by mariadb role. Valid values are `string`. + +* **Required** +* Environment Variable: `MAS_INSTANCE_ID` +* Default Value: "" + +### mariadb_namespace +Action to be performed by mariadb role. Valid values are `string`. + +* Optional +* Environment Variable: `MARIADB_NAMESPACE` +* Default Value: `mariadb` + +### mariadb_instance_name +Action to be performed by mariadb role. Valid values are `string`. + +* Optional +* Environment Variable: `MARIADB_INSTANCE_NAME` +* Default Value: `mariadb-instance` + +### mariadb_user +Action to be performed by mariadb role. Valid values are `string`. + +* Optional +* Environment Variable: `MARIADB_USER` +* Default Value: `mariadb` + +### mariadb_password +Action to be performed by mariadb role. Valid values are `string`. + +* **Required** +* Environment Variable: `MARIADB_PASSWORD` +* Default Value: `` + +### mariadb_database +Action to be performed by mariadb role. Valid values are `string`. + +* Optional +* Environment Variable: `MARIADB_DATABASE` +* Default Value: `kmpipeline` + + +Example Playbook +---------------- + +```yaml +--- +- hosts: localhost + any_errors_fatal: true + roles: + - ibm.mas_devops.mariadb +``` + +License +------- + +EPL-2.0 diff --git a/ibm/mas_devops/roles/mariadb/defaults/main.yml b/ibm/mas_devops/roles/mariadb/defaults/main.yml new file mode 100644 index 0000000000..1ad37c00e8 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/defaults/main.yml @@ -0,0 +1,27 @@ +--- +# When these are defined we will generate a MAS MongoCfg template +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" +mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" + +# Source container registry +# ----------------------------------------------------------------------------- +mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" + +# MAS Entitlement +# ----------------------------------------------------------------------------- +mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}" +ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" +mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" + +# MAS Annotation block +# ----------------------------------------------------------------------------- +mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" + +# MariaDB +mariadb_namespace: "{{ lookup('env', 'MARIADB_NAMESPACE') | default('mariadb', True) }}" +mariadb_instance_name: "{{ lookup('env', 'MARIADB_INSTANCE_NAME') | default('mariadb-instance', True) }}" +mariadb_user: "{{ lookup('env', 'MARIADB_USER') | default('mariadb', True) }}" +mariadb_password: "{{ lookup('env', 'MARIADB_PASSWORD') | default('', True) }}" +mariadb_database: "{{ lookup('env', 'MARIADB_DATABASE') | default('kmpipeline', True) }}" +mariadb_version: "registry.redhat.io/rhel8/mariadb-103:1-188" diff --git a/ibm/mas_devops/roles/mariadb/meta/main.yml b/ibm/mas_devops/roles/mariadb/meta/main.yml new file mode 100644 index 0000000000..45ea110058 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/meta/main.yml @@ -0,0 +1,21 @@ +galaxy_info: + author: Karol Czarnecki (@karol-czarnecki) + description: Deploy Minio storage + company: IBM + + license: EPL-2.0 + + min_ansible_version: 2.10 + + platforms: + - name: GenericLinux + versions: + - all + + galaxy_tags: + - ibm + - mas + - devops + +dependencies: + - role: ibm.mas_devops.ansible_version_check diff --git a/ibm/mas_devops/roles/mariadb/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/mariadb/tasks/determine-storage-classes.yml new file mode 100644 index 0000000000..a0a82bf3f1 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/tasks/determine-storage-classes.yml @@ -0,0 +1,20 @@ +# Lookup storage class availabiity +# ----------------------------------------------------------------------------- +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" + +# Set Primary Storage (Required) +# ----------------------------------------------------------------------------- +- name: "determine-storage-classes : Default Primary Storage if not set by user" + when: primary_storage_class is not defined or primary_storage_class == "" + set_fact: + primary_storage_class: "{{ defaultStorageClasses.rwx }}" + +- name: "determine-storage-classes : Assert that primary storage class has been defined" + assert: + that: primary_storage_class is defined and primary_storage_class != "" + fail_msg: "primary_storage_class must be defined" + +- name: Set primary StorageClass + debug: + msg: "Primary storage class ...... {{ primary_storage_class }}" diff --git a/ibm/mas_devops/roles/mariadb/tasks/main.yml b/ibm/mas_devops/roles/mariadb/tasks/main.yml new file mode 100644 index 0000000000..771d0d37bd --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: Load variables (main) + include_vars: "vars/main.yml" + +# Load default storage class (if not provided by the user) +# ----------------------------------------------------------------------------- +- include_tasks: tasks/determine-storage-classes.yml + +# create mariadb namespace +- name: 'Create mariadb namespace' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/mariadb/mariadb-ns.yml.j2') }}" + +# create mariadb PVC +- name: 'Create mariadb pvc' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/mariadb/mariadb-pvc.yml.j2') }}" + +# create mariadb sa +- name: 'Create mariadb sa' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/mariadb/mariadb-sa.yml.j2') }}" + +# create mariadb np +- name: 'Create mariadb np' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/mariadb/mariadb-np.yml.j2') }}" + +# create mariadb secret +- name: 'Create mariadb secret' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/mariadb/mariadb-secret.yml.j2') }}" + +# create mariadb deployment +- name: 'Create mariadb deployment' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/mariadb/mariadb-deployment.yml.j2') }}" + +# create mariadb service +- name: 'Create mariadb service' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/mariadb/mariadb-service.yml.j2') }}" diff --git a/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-deployment.yml.j2 b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-deployment.yml.j2 new file mode 100644 index 0000000000..eca734c288 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-deployment.yml.j2 @@ -0,0 +1,79 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "{{ mariadb_instance_name }}" + namespace: "{{ mariadb_namespace }}" + labels: + app: "{{ mariadb_instance_name }}" + component: data-science-pipelines + dspa: instance +spec: + strategy: + type: Recreate + selector: + matchLabels: + app: "{{ mariadb_instance_name }}" + component: data-science-pipelines + dspa: instance + template: + metadata: + labels: + app: "{{ mariadb_instance_name }}" + component: data-science-pipelines + dspa: instance + spec: + serviceAccountName: ds-pipelines-mariadb-sa-instance + containers: + - name: mariadb + image: "{{ mariadb_version }}" + ports: + - containerPort: 3306 + readinessProbe: + exec: + command: + - /bin/sh + - "-i" + - "-c" + - >- + MYSQL_PWD=$MYSQL_PASSWORD mysql -h 127.0.0.1 -u $MYSQL_USER -D + $MYSQL_DATABASE -e 'SELECT 1' + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: 3306 + timeoutSeconds: 1 + env: + - name: MYSQL_USER + value: "{{ mariadb_user }}" + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: "mariadb-instance" + - name: MYSQL_DATABASE + value: "{{ mariadb_database }}" + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "true" + resources: + requests: + cpu: 300m + memory: 800Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: mariadb-persistent-storage + mountPath: /var/lib/mysql + volumes: + - name: mariadb-persistent-storage + persistentVolumeClaim: + claimName: mariadb-pvc diff --git a/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-np.yml.j2 b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-np.yml.j2 new file mode 100644 index 0000000000..a166dccf96 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-np.yml.j2 @@ -0,0 +1,38 @@ +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: "{{ mariadb_instance_name }}" + namespace: "{{ mariadb_namespace }}" +spec: + podSelector: + matchLabels: + app: "{{ mariadb_instance_name }}" + component: data-science-pipelines + ingress: + - ports: + - protocol: TCP + port: 3306 + from: + - podSelector: + matchLabels: + app.kubernetes.io/name: data-science-pipelines-operator + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: opendatahub + - podSelector: + matchLabels: + app: ds-pipeline-instance + component: data-science-pipelines + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: "mas-{{ mas_instance_id }}-aibroker" + - podSelector: + matchLabels: + app: ds-pipeline-metadata-grpc-instance + component: data-science-pipelines + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: "mas-{{ mas_instance_id }}-aibroker" + policyTypes: + - Ingress diff --git a/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-ns.yml.j2 b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-ns.yml.j2 new file mode 100644 index 0000000000..50704c6490 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-ns.yml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: "{{ mariadb_namespace }}" diff --git a/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-pvc.yml.j2 b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-pvc.yml.j2 new file mode 100644 index 0000000000..4b6cf52aa2 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-pvc.yml.j2 @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mariadb-pvc + namespace: "{{ mariadb_namespace }}" + labels: + app: "{{ mariadb_instance_name }}" + component: data-science-pipelines +spec: + accessModes: + - ReadWriteOnce + storageClassName: "{{ primary_storage_class }}" + resources: + requests: + storage: 20Gi diff --git a/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-sa.yml.j2 b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-sa.yml.j2 new file mode 100644 index 0000000000..83255ebabc --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-sa.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ds-pipelines-mariadb-sa-instance + namespace: "{{ mariadb_namespace }}" + labels: + app: "{{ mariadb_instance_name }}" + component: data-science-pipelines \ No newline at end of file diff --git a/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-secret.yml.j2 b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-secret.yml.j2 new file mode 100644 index 0000000000..267f1e12b8 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-secret.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: "{{ mariadb_instance_name }}" + namespace: "{{ mariadb_namespace }}" +data: + password: "{{ mariadb_password | b64encode }}" +type: Opaque diff --git a/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-service.yml.j2 b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-service.yml.j2 new file mode 100644 index 0000000000..787c25a703 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-service.yml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: "{{ mariadb_instance_name }}" + namespace: "{{ mariadb_namespace }}" + labels: + app: "{{ mariadb_instance_name }}" + component: data-science-pipelines +spec: + ports: + - name: http + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: "{{ mariadb_instance_name }}" + component: data-science-pipelines \ No newline at end of file diff --git a/ibm/mas_devops/roles/mariadb/vars/main.yml b/ibm/mas_devops/roles/mariadb/vars/main.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ibm/mas_devops/roles/mariadb/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/ibm/mas_devops/roles/minio/README.md b/ibm/mas_devops/roles/minio/README.md new file mode 100644 index 0000000000..16db228378 --- /dev/null +++ b/ibm/mas_devops/roles/minio/README.md @@ -0,0 +1,53 @@ +# Minio +===== + +This role provides support to install and configure Minio Storage + +* Install Minio storage + +Role Variables +-------------- + +### minio_namespace +Action to be performed by minio role. Valid values are `string`. + +* Optional +* Environment Variable: `MINIO_NAMESPACE` +* Default Value: `minio` + +### minio_instance_name +Action to be performed by minio role. Valid values are `string`. + +* Optional +* Environment Variable: `MINIO_INSTANCE_NAME` +* Default Value: `minio` + +### minio_root_user +Action to be performed by minio role. Valid values are `string`. + +* Optional +* Environment Variable: `MINIO_ROOT_USER` +* Default Value: `minio` + +### minio_root_password +Action to be performed by minio role. Valid values are `string`. + +* **Required** +* Environment Variable: `MINIO_ROOT_PASSWORD` +* Default Value: `` + +Example Playbook +---------------- + +```yaml +--- +- hosts: localhost + any_errors_fatal: true + roles: + - ibm.mas_devops.minio +``` + +License +------- + +EPL-2.0 diff --git a/ibm/mas_devops/roles/minio/defaults/main.yml b/ibm/mas_devops/roles/minio/defaults/main.yml new file mode 100644 index 0000000000..1746b12a47 --- /dev/null +++ b/ibm/mas_devops/roles/minio/defaults/main.yml @@ -0,0 +1,26 @@ +--- +# When these are defined we will generate a MAS MongoCfg template +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" +mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" + +# Source container registry +# ----------------------------------------------------------------------------- +mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" + +# MAS Entitlement +# ----------------------------------------------------------------------------- +mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}" +ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" +mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" + +# MAS Annotation block +# ----------------------------------------------------------------------------- +mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" + +# MINIO +minio_namespace: "{{ lookup('env', 'MINIO_NAMESPACE') | default('minio', True) }}" +minio_instance_name: "{{ lookup('env', 'MINIO_INSTANCE_NAME') | default('minio', True) }}" +minio_root_user: "{{ lookup('env', 'MINIO_ROOT_USER') | default('minio', True) }}" +minio_root_password: "{{ lookup('env', 'MINIO_ROOT_PASSWORD') | default('', True) }}" +minio_version: "quay.io/minio/minio:latest" diff --git a/ibm/mas_devops/roles/minio/meta/main.yml b/ibm/mas_devops/roles/minio/meta/main.yml new file mode 100644 index 0000000000..45ea110058 --- /dev/null +++ b/ibm/mas_devops/roles/minio/meta/main.yml @@ -0,0 +1,21 @@ +galaxy_info: + author: Karol Czarnecki (@karol-czarnecki) + description: Deploy Minio storage + company: IBM + + license: EPL-2.0 + + min_ansible_version: 2.10 + + platforms: + - name: GenericLinux + versions: + - all + + galaxy_tags: + - ibm + - mas + - devops + +dependencies: + - role: ibm.mas_devops.ansible_version_check diff --git a/ibm/mas_devops/roles/minio/tasks/determine-storage-classes.yml b/ibm/mas_devops/roles/minio/tasks/determine-storage-classes.yml new file mode 100644 index 0000000000..a0a82bf3f1 --- /dev/null +++ b/ibm/mas_devops/roles/minio/tasks/determine-storage-classes.yml @@ -0,0 +1,20 @@ +# Lookup storage class availabiity +# ----------------------------------------------------------------------------- +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" + +# Set Primary Storage (Required) +# ----------------------------------------------------------------------------- +- name: "determine-storage-classes : Default Primary Storage if not set by user" + when: primary_storage_class is not defined or primary_storage_class == "" + set_fact: + primary_storage_class: "{{ defaultStorageClasses.rwx }}" + +- name: "determine-storage-classes : Assert that primary storage class has been defined" + assert: + that: primary_storage_class is defined and primary_storage_class != "" + fail_msg: "primary_storage_class must be defined" + +- name: Set primary StorageClass + debug: + msg: "Primary storage class ...... {{ primary_storage_class }}" diff --git a/ibm/mas_devops/roles/minio/tasks/main.yml b/ibm/mas_devops/roles/minio/tasks/main.yml new file mode 100644 index 0000000000..fa3879b88a --- /dev/null +++ b/ibm/mas_devops/roles/minio/tasks/main.yml @@ -0,0 +1,37 @@ +--- +- name: Load variables (main) + include_vars: "vars/main.yml" + +# Load default storage class (if not provided by the user) +# ----------------------------------------------------------------------------- +- include_tasks: tasks/determine-storage-classes.yml + +# create minio namespace +- name: 'Create minio namespace' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/minio/minio-ns.yml.j2') }}" + +# create minio PVC +- name: 'Create minio pvc' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/minio/minio-pvc.yml.j2') }}" + +# create minio deployment +- name: 'Create minio deployment' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/minio/minio-deployment.yml.j2') }}" + +# create minio service +- name: 'Create minio service' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/minio/minio-service.yml.j2') }}" + +# create minio route +- name: 'Create minio route' + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/minio/minio-route.yml.j2') }}" diff --git a/ibm/mas_devops/roles/minio/templates/minio/minio-deployment.yml.j2 b/ibm/mas_devops/roles/minio/templates/minio/minio-deployment.yml.j2 new file mode 100644 index 0000000000..8853bd3368 --- /dev/null +++ b/ibm/mas_devops/roles/minio/templates/minio/minio-deployment.yml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "{{ minio_instance_name }}" + namespace: "{{ minio_namespace }}" +spec: + selector: + matchLabels: + app: "{{ minio_instance_name }}" + template: + metadata: + labels: + app: "{{ minio_instance_name }}" + spec: + volumes: + - name: storage + persistentVolumeClaim: + claimName: minio-pvc + containers: + - name: "{{ minio_instance_name }}" + image: "{{ minio_version }}" + command: + - /bin/bash + - -c + args: + - minio server /data --console-address :9090 + env: + - name: MINIO_ROOT_USER + value: "{{ minio_root_user }}" + - name: MINIO_ROOT_PASSWORD + value: "{{ minio_root_password }}" + volumeMounts: + - mountPath: /data + name: storage diff --git a/ibm/mas_devops/roles/minio/templates/minio/minio-ns.yml.j2 b/ibm/mas_devops/roles/minio/templates/minio/minio-ns.yml.j2 new file mode 100644 index 0000000000..1c39b209dc --- /dev/null +++ b/ibm/mas_devops/roles/minio/templates/minio/minio-ns.yml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: "{{ minio_namespace }}" diff --git a/ibm/mas_devops/roles/minio/templates/minio/minio-pvc.yml.j2 b/ibm/mas_devops/roles/minio/templates/minio/minio-pvc.yml.j2 new file mode 100644 index 0000000000..b8ed15b65c --- /dev/null +++ b/ibm/mas_devops/roles/minio/templates/minio/minio-pvc.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc + namespace: "{{ minio_namespace }}" +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + storageClassName: "{{ primary_storage_class }}" + resources: + requests: + storage: 40Gi diff --git a/ibm/mas_devops/roles/minio/templates/minio/minio-route.yml.j2 b/ibm/mas_devops/roles/minio/templates/minio/minio-route.yml.j2 new file mode 100644 index 0000000000..af5815e587 --- /dev/null +++ b/ibm/mas_devops/roles/minio/templates/minio/minio-route.yml.j2 @@ -0,0 +1,16 @@ +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-route + namespace: {{ minio_namespace }} +spec: + to: + kind: Service + name: minio-service + weight: 100 + port: + targetPort: 9090 + tls: + termination: edge + insecureEdgeTerminationPolicy: None diff --git a/ibm/mas_devops/roles/minio/templates/minio/minio-service.yml.j2 b/ibm/mas_devops/roles/minio/templates/minio/minio-service.yml.j2 new file mode 100644 index 0000000000..a2e8173a02 --- /dev/null +++ b/ibm/mas_devops/roles/minio/templates/minio/minio-service.yml.j2 @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: minio-service + namespace: {{ minio_namespace }} +spec: + type: ClusterIP + ports: + - name: api + port: 9000 + targetPort: 9000 + protocol: TCP + - name: console + port: 9090 + targetPort: 9090 + protocol: TCP + selector: + app: {{ minio_instance_name }} diff --git a/ibm/mas_devops/roles/minio/vars/main.yml b/ibm/mas_devops/roles/minio/vars/main.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ibm/mas_devops/roles/minio/vars/main.yml @@ -0,0 +1 @@ +--- From fbb9e14c28ede83287be610eb37e5f8d548b479c Mon Sep 17 00:00:00 2001 From: karol-czarnecki <112193189+karol-czarnecki@users.noreply.github.com> Date: Thu, 20 Mar 2025 12:45:49 +0000 Subject: [PATCH 128/169] [patch] merge Masaib7501 (#1688) --- .../playbooks/oneclick_add_aibroker.yml | 2 +- ibm/mas_devops/roles/aibroker/tasks/main.yml | 9 +++++++++ .../roles/aibroker/tasks/mariadb/main.yml | 5 +++++ .../roles/aibroker/tasks/minio/main.yml | 5 +++++ .../roles/aibroker/tasks/watsonxai/main.yml | 5 +++++ .../templates/aibroker/aibrokerapp.yml.j2 | 12 ++++++++++++ .../aibroker/aibrokerworkspace.yml.j2 | 6 +++++- .../mariadb/mariadb-credentials.yml.j2 | 10 ++++++++++ .../templates/minio/minio-credentials.yml.j2 | 10 ++++++++++ .../watsonxai/watsonxai-credentials.yml.j2 | 9 +++++++++ ibm/mas_devops/roles/aibroker/vars/main.yml | 19 +++++++++++++++++++ ibm/mas_devops/roles/odh/tasks/main.yml | 6 +++--- 12 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 ibm/mas_devops/roles/aibroker/tasks/mariadb/main.yml create mode 100644 ibm/mas_devops/roles/aibroker/tasks/minio/main.yml create mode 100644 ibm/mas_devops/roles/aibroker/tasks/watsonxai/main.yml create mode 100644 ibm/mas_devops/roles/aibroker/templates/mariadb/mariadb-credentials.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker/templates/minio/minio-credentials.yml.j2 create mode 100644 ibm/mas_devops/roles/aibroker/templates/watsonxai/watsonxai-credentials.yml.j2 diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 6c6e9fbc99..4f0c6876cf 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -72,4 +72,4 @@ - ibm.mas_devops.odh - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker - - ibm.mas_devops.aibroker_tenant + # - ibm.mas_devops.aibroker_tenant diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 4ff9e365f9..1a7bde9e04 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -32,6 +32,15 @@ kind: Namespace name: "{{ aibroker_namespace }}" +# Create secret for minio +- include_tasks: tasks/minio/main.yml + +# Create secret for mariadb +- include_tasks: tasks/mariadb/main.yml + +# Create secret for watsonX Ai +- include_tasks: tasks/watsonxai/main.yml + # Create config for SLS - include_tasks: tasks/config_sls/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/mariadb/main.yml b/ibm/mas_devops/roles/aibroker/tasks/mariadb/main.yml new file mode 100644 index 0000000000..b1a7f456b2 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/tasks/mariadb/main.yml @@ -0,0 +1,5 @@ +--- +- name: "Create secret for mariadb credentials" + kubernetes.core.k8s: + apply: yes + template: "templates/mariadb/mariadb-credentials.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker/tasks/minio/main.yml b/ibm/mas_devops/roles/aibroker/tasks/minio/main.yml new file mode 100644 index 0000000000..7b8ef6ba80 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/tasks/minio/main.yml @@ -0,0 +1,5 @@ +--- +- name: "Create secret for minio credentials" + kubernetes.core.k8s: + apply: yes + template: "templates/minio/minio-credentials.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker/tasks/watsonxai/main.yml b/ibm/mas_devops/roles/aibroker/tasks/watsonxai/main.yml new file mode 100644 index 0000000000..ef64bd9bf8 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/tasks/watsonxai/main.yml @@ -0,0 +1,5 @@ +--- +- name: "Create secret for watsonxai credentials" + kubernetes.core.k8s: + apply: yes + template: "templates/watsonxai/watsonxai-credentials.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index c108a8e1ec..772fab691d 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -22,3 +22,15 @@ spec: ca: "{{ jdbccfg.ca | b64decode }}" cluster_domain: "{{ cluster_domain }}" in_saas_env: "{{ in_saas_env }}" + minio: + host: "{{ mas_aibroker_storage_host }}" + port: "{{ mas_aibroker_storage_port }}" + credentials_secret: "{{ mas_aibroker_minio_secret }}" + pipeline_bucket: "{{ mas_aibroker_storage_pipelines_bucket }}" + tenants_bucket: "{{ mas_aibroker_storage_tenants_bucket }}" + templates_bucket: "{{ mas_aibroker_storage_templates_bucket }}" + mariadb: + host: "{{ mas_aibroker_db_host }}" + port: "{{ mas_aibroker_db_port }}" + credentials_secret: "{{ mas_aibroker_mariadb_secret }}" + database_name: "{{ mas_aibroker_db_secret_name }}" diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 index cdbba6ff91..462f903dca 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 @@ -29,4 +29,8 @@ spec: subscription_id: "{{ mas_aibroker_sls_subscription_id }}" instance_id: "{{ mas_instance_id }}" cluster_domain: "{{ cluster_domain }}" - in_saas_env: "{{ in_saas_env }}" \ No newline at end of file + in_saas_env: "{{ in_saas_env }}" + watsonxai: + url: "{{ mas_aibroker_watsonxai_url }}" + api_key: "{{ mas_aibroker_watsonx_secret }}" + project_id: "{{ mas_aibroker_watsonxai_project_id }}" diff --git a/ibm/mas_devops/roles/aibroker/templates/mariadb/mariadb-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/mariadb/mariadb-credentials.yml.j2 new file mode 100644 index 0000000000..8db0de61f8 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/templates/mariadb/mariadb-credentials.yml.j2 @@ -0,0 +1,10 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: "{{ mas_aibroker_mariadb_secret }}" + namespace: mas-{{ mas_instance_id }}-aibroker +data: + username: "{{ mas_aibroker_db_user | b64encode }}" + password: "{{ mas_aibroker_db_secret_value | b64encode }}" +type: Opaque \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/templates/minio/minio-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/minio/minio-credentials.yml.j2 new file mode 100644 index 0000000000..461db7576a --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/templates/minio/minio-credentials.yml.j2 @@ -0,0 +1,10 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: "{{ mas_aibroker_minio_secret }}" + namespace: mas-{{ mas_instance_id }}-aibroker +data: + username: "{{ mas_aibroker_storage_accesskey | b64encode }}" + password: "{{ mas_aibroker_storage_secretkey | b64encode }}" +type: Opaque \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/templates/watsonxai/watsonxai-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/watsonxai/watsonxai-credentials.yml.j2 new file mode 100644 index 0000000000..3e20d32ef9 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker/templates/watsonxai/watsonxai-credentials.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: "{{ mas_aibroker_watsonx_secret }}" + namespace: mas-{{ mas_instance_id }}-aibroker +data: + api_key: "{{ mas_aibroker_storage_accesskey | b64encode }}" +type: Opaque \ No newline at end of file diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index c17fa30044..f60ce425b7 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -29,6 +29,7 @@ mas_aibroker_watsonxai_apikey: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY' mas_aibroker_watsonxai_url: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') }}" mas_aibroker_watsonxai_project_id: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') }}" mas_aibroker_watsonx_action: "{{ lookup('env', 'MAS_AIBROKER_WATSONX_ACTION') | default('install', true) }}" +mas_aibroker_watsonx_secret: "{{ mas_app_id }}-watsonxcfg" # S3 mas_aibroker_storage_host: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_HOST') | default('', true) }}" @@ -38,6 +39,11 @@ mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | mas_aibroker_s3_action: "{{ lookup('env', 'MAS_AIBROKER_S3_ACTION') | default('install', true) }}" mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" +# Bucket names +mas_aibroker_storage_pipelines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') }}" +mas_aibroker_storage_tenants_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TENANTS_BUCKET') }}" +mas_aibroker_storage_templates_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET') }}" + # SAAS mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" @@ -59,3 +65,16 @@ mas_aibroker_jdbc_secret: "{{ mas_app_id }}-jdbccfg" # SLS mas_aibroker_sls_registration_key_secret: "sls-registration-key" mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" + +# Minio +mas_aibroker_minio_secret: "{{ mas_app_id }}-miniocfg" + +# MariaDB +mas_aibroker_mariadb_secret: "{{ mas_app_id }}-mariadbcfg" +mas_aibroker_db_host: "{{ lookup('env', 'MAS_AIBROKER_DB_HOST') | default('', true) }}" +mas_aibroker_db_port: "{{ lookup('env', 'MAS_AIBROKER_DB_PORT') | default('', true) }}" +mas_aibroker_db_user: "{{ lookup('env', 'MAS_AIBROKER_DB_USER') | default('', true) }}" +mas_aibroker_db_database: "{{ lookup('env', 'MAS_AIBROKER_DB_DATABASE') | default('', true) }}" +mas_aibroker_db_secret_name: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') | default('', true) }}" +mas_aibroker_db_secret_key: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_KEY') | default('password', true) }}" +mas_aibroker_db_secret_value: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') | default('', true) }}" diff --git a/ibm/mas_devops/roles/odh/tasks/main.yml b/ibm/mas_devops/roles/odh/tasks/main.yml index bda0b97936..0be38235c3 100644 --- a/ibm/mas_devops/roles/odh/tasks/main.yml +++ b/ibm/mas_devops/roles/odh/tasks/main.yml @@ -54,7 +54,7 @@ - dsci_lookup.resources[0].status.phase is defined - dsci_lookup.resources[0].status.phase == 'Ready' retries: 10 - delay: 30 + delay: 60 # Create Data Science Cluster - name: "Create Data Science Cluster" @@ -74,7 +74,7 @@ - dsc_lookup.resources[0].status.phase is defined - dsc_lookup.resources[0].status.phase == 'Ready' retries: 10 - delay: 30 + delay: 60 # Create Data Science Pipelines Applications - name: "Create Data Science Pipelines Applications CR" @@ -94,7 +94,7 @@ - dspa_lookup.resources | length > 0 - dspa_lookup.resources | json_query('[*].status.conditions[?type==`Ready`][].status') | select ('match','True') | list | length == 1 retries: 10 - delay: 30 + delay: 60 # Wait until the pipeline-runner-instance service account is available # ----------------------------------------------------------------------------- From 40e9eb24af6c02cfbdca02869b5b2af56a2fafd9 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 20 Mar 2025 13:35:52 +0000 Subject: [PATCH 129/169] [patch] fixed type on CRD --- .../roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 | 2 +- .../aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 index 462f903dca..10a18e40c9 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 @@ -22,7 +22,7 @@ spec: ca: "{{ jdbccfg.ca }}" sls: url: "{{ slscfg.url }}" - sls_regitration_key_secret: "{{ slscfg.registration_key }}" + sls_registration_key_secret: "{{ slscfg.registration_key }}" ca: "{{ slscfg.ca }}" tenant: tenant_id: "{{ tenantNamespace }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 0cb94a3c28..acbf45a73d 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -22,7 +22,7 @@ spec: ca: "{{ jdbccfg.ca }}" sls: url: "{{ slscfg.url }}" - sls_regitration_key_secret: "{{ slscfg.registration_key }}" + sls_registration_key_secret: "{{ slscfg.registration_key }}" ca: "{{ slscfg.ca }}" tenant: tenant_id: "{{ tenantNamespace }}" From a88b5c326b3d29cdf7b80bfff0b08dfb1c349a73 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Fri, 21 Mar 2025 13:05:56 +0530 Subject: [PATCH 130/169] [patch] updated aibroker_tenant role --- .../aibroker_tenant/tasks/config_db2/main.yml | 8 ++--- .../aibroker_tenant/tasks/config_dro/main.yml | 8 ++--- .../aibroker_tenant/tasks/config_sls/main.yml | 8 ++--- .../tasks/tenant/install/main.yml | 36 ++++++++++++++----- 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml index 69ed491313..9f16dec5bb 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml @@ -22,7 +22,7 @@ - "JDBC url .................. {{ jdbccfg.url }}" - "JDBC cert ca .............. {{ jdbccfg.ca }}" -- name: "Create secret for JDBC admin credentials" - kubernetes.core.k8s: - apply: yes - template: "templates/jdbc/jdbc-admin-credentials.yml.j2" +# - name: "Create secret for JDBC admin credentials" +# kubernetes.core.k8s: +# apply: yes +# template: "templates/jdbc/jdbc-admin-credentials.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml index 7a2a60a520..cad63b7662 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml @@ -20,7 +20,7 @@ - "dro url ................. {{ drocfg.url }}" - "dro cert ca ............. {{ drocfg.ca }}" -- name: "Create secret for Dro registration key" - kubernetes.core.k8s: - apply: yes - template: "templates/dro/dro-token.yml.j2" +# - name: "Create secret for Dro registration key" +# kubernetes.core.k8s: +# apply: yes +# template: "templates/dro/dro-token.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml index f32195ca35..c167d5fdbf 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml @@ -20,7 +20,7 @@ - "sls url ................. {{ slscfg.url }}" - "sls cert ca ............. {{ slscfg.ca }}" -- name: "Create secret for SLS registration key" - kubernetes.core.k8s: - apply: yes - template: "templates/sls/sls-tenant-details.yml.j2" +# - name: "Create secret for SLS registration key" +# kubernetes.core.k8s: +# apply: yes +# template: "templates/sls/sls-tenant-details.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index 41ec997e4e..3169ddcaaf 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -67,15 +67,33 @@ namespace: "{{ aibroker_namespace }}" register: sls_secret_info -- name: Create SLS secret - script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} {{ mas_aibroker_path_ca_crt }}" - when: +# If sls secret does not exist then create one. +- when: - sls_secret_info.resources | length == 0 - register: sls_secret_output - -- name: Debug Create sls secret for AI Broker - debug: - msg: "{{ sls_secret_output }}" + - mas_aibroker_saas == true + block: + - name: "Create cert location {{ mas_aibroker_path_ca_crt }}" + shell: mkdir -p '{{ mas_aibroker_path_ca_crt }}' + + - name: "Insert SLS cert ca to file located {{ mas_aibroker_path_ca_crt }}" + shell: "echo '{{ slscfg.ca }}' > '{{ mas_aibroker_path_ca_crt }}/sls_ca.crt' " + + - name: debug #TODO: remove before final commit + debug: + msg: + - "tenantNamespace ..... {{ tenantNamespace }} " + - "mas_instance_id ..... {{ mas_instance_id }} " + - "mas_aibroker_sls_url ..... {{ slscfg.url }} " + - "mas_aibroker_sls_registration_key ..... {{ slscfg.registration_key }} " + - "mas_aibroker_path_ca_crt ..... {{ mas_aibroker_path_ca_crt }} " + + - name: Create SLS secret + script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" + register: sls_secret_output + + - name: Debug Create sls secret for AI Broker + debug: + msg: "{{ sls_secret_output }}" - name: Delete DRO secret command: oc delete secret "{{ tenantNamespace }}----dro-secret" -n {{ aibroker_namespace }} @@ -84,7 +102,7 @@ failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - name: Create DRO token - script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_dro_token }}" + script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ drocfg.registration_key }}" register: dro_token_output - name: Debug Create DRO token From 5ed1ed30cd905cd6437294ab2b8b99772dfb6c70 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Fri, 21 Mar 2025 13:06:17 +0530 Subject: [PATCH 131/169] [patch] enabled aibroker_tenant role --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 4f0c6876cf..6c6e9fbc99 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -72,4 +72,4 @@ - ibm.mas_devops.odh - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker - # - ibm.mas_devops.aibroker_tenant + - ibm.mas_devops.aibroker_tenant From c4940d5f318adaa6188414269d6998cf8d5affa1 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Fri, 21 Mar 2025 13:06:43 +0530 Subject: [PATCH 132/169] [patch] Reverted changes --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 6c6e9fbc99..4f0c6876cf 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -72,4 +72,4 @@ - ibm.mas_devops.odh - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker - - ibm.mas_devops.aibroker_tenant + # - ibm.mas_devops.aibroker_tenant From a1915b431cb4ac8c12b116b7c822b2eeda297efc Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Tue, 25 Mar 2025 18:58:53 +0530 Subject: [PATCH 133/169] [patch] Fix aibroker_tenant role (#1692) --- .../playbooks/oneclick_add_aibroker.yml | 2 +- .../tasks/tenant/install/main.yml | 26 ++++++++++++++++--- .../roles/aibroker_tenant/vars/main.yml | 3 +++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 4f0c6876cf..6c6e9fbc99 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -72,4 +72,4 @@ - ibm.mas_devops.odh - ibm.mas_devops.kmodels - ibm.mas_devops.aibroker - # - ibm.mas_devops.aibroker_tenant + - ibm.mas_devops.aibroker_tenant diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index 3169ddcaaf..0975f9b196 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -134,9 +134,27 @@ retries: 60 delay: 30 +- name: Lookup the aibroker route + kubernetes.core.k8s_info: + api_version: route.openshift.io/v1 + kind: Route + name: "{{ aibroker.name }}" + namespace: "{{ aibroker_namespace }}" + wait: yes + wait_timeout: 30 + register: aibrokerRoute + +- name: Set aibroker route host + ansible.builtin.set_fact: + aibrokerHostName: "{{ aibrokerRoute.resources[0].spec.host }}" + +- name: Debug aibrokerRoute + debug: + msg: "{{ aibrokerHostName }}" + - name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri uri: - url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" + url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenant" validate_certs: no method: POST headers: @@ -156,7 +174,7 @@ - name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri uri: - url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" validate_certs: no method: POST headers: @@ -176,7 +194,7 @@ - name: Create tenant {{ tenantNamespace }} and add to db2 for pcc - using uri uri: - url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" validate_certs: no method: POST headers: @@ -196,7 +214,7 @@ - name: Create tenant {{ tenantNamespace }} and add to db2 for mcc - using uri uri: - url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" + url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" validate_certs: no method: POST headers: diff --git a/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml b/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml index defda35239..9cdcb78c0d 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml @@ -1,5 +1,8 @@ --- # aibroker vars +aibroker: + name: "aibroker" + aibroker_namespace: 'mas-{{ mas_instance_id }}-aibroker' mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" From 775f00c9f1254351aa8ecf8612d4c4eef9921d0f Mon Sep 17 00:00:00 2001 From: shauryasandoo <39964277+shauryasandoo@users.noreply.github.com> Date: Thu, 27 Mar 2025 19:51:33 +0530 Subject: [PATCH 134/169] [patch] added SSL (#1699) Co-authored-by: Shaurya Sandoo --- .../roles/aibroker/files/alm_sample_sls_use.py | 14 +++++++------- .../aibroker_tenant/files/alm_sample_sls_use.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py index 666dc461ef..295d28ade7 100644 --- a/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py +++ b/ibm/mas_devops/roles/aibroker/files/alm_sample_sls_use.py @@ -101,7 +101,7 @@ def generate_api_key(): headers = {'X-Registration-Key': slsRegistrationKey} # TODO: Rawa - Swagger docs don't document the need for this critical header #print ("Making initial registration request for client "+slsClientId) - response = requests.post(slsUrl+"/api/registrations",verify=False,json=data, headers=headers ) + response = requests.post(slsUrl+"/api/registrations",verify=caCertificateForSSL,json=data, headers=headers ) response.raise_for_status() registrationId = response.json()['registrationId'] @@ -113,7 +113,7 @@ def generate_api_key(): provisioningWaiting = True while provisioningWaiting: headers = {'X-Registration-Key': slsRegistrationKey} - response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=False,headers=headers ) + response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=caCertificateForSSL,headers=headers ) response.raise_for_status() status = response.json()['state'] # print("Waiting for client provisioning to be completed. Current status: %s" %status) @@ -136,7 +136,7 @@ def generate_api_key(): # The next step is to confirm the client. Poll status until complete. confirmationWaiting = True while confirmationWaiting: - response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=False, cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL, cert=(clientTlsCrtPath,clientTlsKeyPath)) #print (response) status = response.json()['state'] #print("Waiting for client registration to be confirmed. Current status: %s" %status) @@ -155,12 +155,12 @@ def generate_api_key(): def spendAppPoints(quantity): #print ("Let's spend "+str(quantity)+" AppPoints") data = {'validity': 86400, 'quantity': quantity} - response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() # String representation of AppPoint usage by your client. TODO: SLS will shortly be providing an update to correctly report usage for your client. At present, it returns the overall usage across all clients. def getUsage(): - response = requests.get(slsUrl+"/api/tokens/",verify=False,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner + response = requests.get(slsUrl+"/api/tokens/",verify=caCertificateForSSL,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner response.raise_for_status() j = json.loads(response.content.decode('utf-8')) return j[0]['used'] @@ -243,12 +243,12 @@ def getUsage(): # At deprovision, return your licenses and unregister the client print ("Preparing for deprovision: returning any AppPoints to the pool") - response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() print("Usage after deprovisioning: "+str(getUsage())) print ("Preparing for deprovision: unregistering the SLS client") - response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() #print ("Deleting redundant client certificates") os.remove(clientTlsCrtPath) diff --git a/ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py index 666dc461ef..295d28ade7 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py +++ b/ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py @@ -101,7 +101,7 @@ def generate_api_key(): headers = {'X-Registration-Key': slsRegistrationKey} # TODO: Rawa - Swagger docs don't document the need for this critical header #print ("Making initial registration request for client "+slsClientId) - response = requests.post(slsUrl+"/api/registrations",verify=False,json=data, headers=headers ) + response = requests.post(slsUrl+"/api/registrations",verify=caCertificateForSSL,json=data, headers=headers ) response.raise_for_status() registrationId = response.json()['registrationId'] @@ -113,7 +113,7 @@ def generate_api_key(): provisioningWaiting = True while provisioningWaiting: headers = {'X-Registration-Key': slsRegistrationKey} - response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=False,headers=headers ) + response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=caCertificateForSSL,headers=headers ) response.raise_for_status() status = response.json()['state'] # print("Waiting for client provisioning to be completed. Current status: %s" %status) @@ -136,7 +136,7 @@ def generate_api_key(): # The next step is to confirm the client. Poll status until complete. confirmationWaiting = True while confirmationWaiting: - response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=False, cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL, cert=(clientTlsCrtPath,clientTlsKeyPath)) #print (response) status = response.json()['state'] #print("Waiting for client registration to be confirmed. Current status: %s" %status) @@ -155,12 +155,12 @@ def generate_api_key(): def spendAppPoints(quantity): #print ("Let's spend "+str(quantity)+" AppPoints") data = {'validity': 86400, 'quantity': quantity} - response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() # String representation of AppPoint usage by your client. TODO: SLS will shortly be providing an update to correctly report usage for your client. At present, it returns the overall usage across all clients. def getUsage(): - response = requests.get(slsUrl+"/api/tokens/",verify=False,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner + response = requests.get(slsUrl+"/api/tokens/",verify=caCertificateForSSL,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner response.raise_for_status() j = json.loads(response.content.decode('utf-8')) return j[0]['used'] @@ -243,12 +243,12 @@ def getUsage(): # At deprovision, return your licenses and unregister the client print ("Preparing for deprovision: returning any AppPoints to the pool") - response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() print("Usage after deprovisioning: "+str(getUsage())) print ("Preparing for deprovision: unregistering the SLS client") - response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=False,cert=(clientTlsCrtPath,clientTlsKeyPath)) + response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) response.raise_for_status() #print ("Deleting redundant client certificates") os.remove(clientTlsCrtPath) From ed0775eec9d0ef7b5cdc67a216cddcc9ad7d08c3 Mon Sep 17 00:00:00 2001 From: shauryasandoo <39964277+shauryasandoo@users.noreply.github.com> Date: Thu, 3 Apr 2025 15:45:52 +0530 Subject: [PATCH 135/169] [patch] adding ssl cert fix (#1704) Co-authored-by: Shaurya Sandoo --- .../aibroker/tasks/tenant/install/main.yml | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml index 41ec997e4e..1fd9cff464 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml @@ -67,11 +67,33 @@ namespace: "{{ aibroker_namespace }}" register: sls_secret_info -- name: Create SLS secret - script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_sls_url }} {{ mas_aibroker_sls_registration_key }} {{ mas_aibroker_path_ca_crt }}" - when: +# If sls secret does not exist then create one. +- when: - sls_secret_info.resources | length == 0 - register: sls_secret_output + - mas_aibroker_saas == true + block: + - name: "Create cert location {{ mas_aibroker_path_ca_crt }}" + shell: mkdir -p '{{ mas_aibroker_path_ca_crt }}' + + - name: "Insert SLS cert ca to file located {{ mas_aibroker_path_ca_crt }}" + shell: "echo '{{ slscfg.ca }}' > '{{ mas_aibroker_path_ca_crt }}/sls_ca.crt' " + + - name: debug #TODO: remove before final commit + debug: + msg: + - "tenantNamespace ..... {{ tenantNamespace }} " + - "mas_instance_id ..... {{ mas_instance_id }} " + - "mas_aibroker_sls_url ..... {{ slscfg.url }} " + - "mas_aibroker_sls_registration_key ..... {{ slscfg.registration_key }} " + - "mas_aibroker_path_ca_crt ..... {{ mas_aibroker_path_ca_crt }} " + + - name: Create SLS secret + script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" + register: sls_secret_output + + - name: Debug Create sls secret for AI Broker + debug: + msg: "{{ sls_secret_output }}" - name: Debug Create sls secret for AI Broker debug: From 910b439300916b2acb4be0dc2e5ba8b590bae58b Mon Sep 17 00:00:00 2001 From: Joe Harte Date: Fri, 4 Apr 2025 16:35:10 +0100 Subject: [PATCH 136/169] [minor] Create workspace CR task (#1685) --- .gitignore | 7 +- .../playbooks/oneclick_add_aibroker.yml | 19 ++ .../roles/aibroker/tasks/aibroker/main.yml | 47 ---- ibm/mas_devops/roles/aibroker/tasks/main.yml | 24 -- .../aibroker/tasks/tenant/install/main.yml | 219 ------------------ .../roles/aibroker/tasks/tenant/main.yml | 14 -- .../aibroker/tasks/tenant/remove/main.yml | 50 ---- .../aibroker/aibrokerworkspace.yml.j2 | 36 --- .../files/create_sls_secret.sh | 11 +- .../aibroker_tenant/tasks/aibroker/main.yml | 128 ---------- .../aibroker_tenant/tasks/config_db2/main.yml | 28 --- .../roles/aibroker_tenant/tasks/main.yml | 15 -- .../tasks/tenant/install/main.yml | 54 +++-- .../templates/aibroker/aibrokerapp.yml.j2 | 22 -- .../aibroker/aibrokerworkspace.yml.j2 | 14 +- .../jdbc/jdbc-admin-credentials.yml.j2 | 10 - .../roles/aibroker_tenant/vars/main.yml | 1 + 17 files changed, 75 insertions(+), 624 deletions(-) delete mode 100644 ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml delete mode 100644 ibm/mas_devops/roles/aibroker/tasks/tenant/main.yml delete mode 100644 ibm/mas_devops/roles/aibroker/tasks/tenant/remove/main.yml delete mode 100644 ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/aibroker/main.yml delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerapp.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/jdbc/jdbc-admin-credentials.yml.j2 diff --git a/.gitignore b/.gitignore index 067e4d4ecd..916da35f48 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ build/bin/downloads/*.tgz .pyenv cpd-cli-workspace/* /tmp -/node_modules -package-lock.json -package.json +/node_modules +package-lock.json +package.json +ibm/mas_devops/playbooks/certs/ diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 6c6e9fbc99..547d7b5543 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -24,6 +24,25 @@ db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" pre_tasks: + - name: "Get cluster domain" + kubernetes.core.k8s_info: + api_version: config.openshift.io/v1 + kind: DNS + name: cluster + register: _cluster_dns + + - name: "Set fact: cluster domain" + set_fact: + cluster_domain: "{{ _cluster_dns.resources[0].spec.baseDomain }}" + + - name: "Debug: cluster domain" + debug: + msg: "Cluster domain ........................ {{ cluster_domain }}" + + - name: "Check if SaaS environment" + set_fact: + in_saas_env: "{{ lookup('env', 'INSAASENV') | default('false', true) | bool }}" + # For the full set of supported environment variables refer to the playbook documentation - name: Check for required common environment variables assert: diff --git a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml index ebe99c6e35..6e9a05138f 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml @@ -79,50 +79,3 @@ - aibrokerapp_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 retries: 45 # approx 45 minutes before we give up delay: 60 # 1 minute - -# IBM Maximo AI Broker workspace -# ----------------------------------------------------------------------------- -- name: Create AI Broker workspace CR - vars: - annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}" - kubernetes.core.k8s: - state: present - namespace: "{{ aibroker_namespace }}" - template: templates/aibroker/aibrokerworkspace.yml.j2 - register: aibrokerWorkspaceResult - -# Wait IBM Maximo AI Broker custom resource to be complete -# ----------------------------------------------------------------------------- -- name: "Wait for AiBrokerWorkspace custom resource to be ready (60s delay)" - kubernetes.core.k8s_info: - api_version: apps.mas.ibm.com/v1 - name: "{{ mas_instance_id }}" - namespace: "{{ aibroker_namespace }}" - kind: AiBrokerWorkspace - register: aibrokerworkspace_cr_result - until: - - aibrokerworkspace_cr_result.resources is defined and aibrokerworkspace_cr_result.resources | length == 1 - - aibrokerworkspace_cr_result.resources[0].status is defined - - aibrokerworkspace_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 - retries: 45 # approx 45 minutes before we give up - delay: 60 # 1 minute - -- name: Await to aibroker-api pod created - pause: - minutes: 3 - prompt: "Waiting for aibroker-api pod" - -- name: "Wait for aibroker-api pod" - shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} - register: _is_not_exists - until: _is_not_exists.stdout|int == 0 - retries: 60 - delay: 30 - -- name: "Wait for aibroker-api pod to be ready (30s delay)" - shell: > - oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l - register: _is_not_ready - until: _is_not_ready.stdout|int == 0 - retries: 60 - delay: 30 diff --git a/ibm/mas_devops/roles/aibroker/tasks/main.yml b/ibm/mas_devops/roles/aibroker/tasks/main.yml index 1a7bde9e04..0408aa4da2 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/main.yml @@ -2,27 +2,6 @@ - name: Load variables (main) include_vars: "vars/main.yml" -# Get cluster domain -# ----------------------------------------------------------------------------- -- name: "Get cluster domain" - kubernetes.core.k8s_info: - api_version: config.openshift.io/v1 - kind: DNS - name: cluster - register: _cluster_dns - -- name: "Set fact: cluster domain" - set_fact: - cluster_domain: "{{ _cluster_dns.resources[0].spec.baseDomain }}" - -- name: "Debug: cluster domain" - debug: - msg: "Cluster domain ........................ {{ cluster_domain }}" - -- name: "Set fact: cluster domain" - set_fact: - in_saas_env: "{{ lookup('env', 'INSAASENV') | default('false', true) | bool }}" - # Create IBM Maximo IBM Maximo AI Broker # ----------------------------------------------------------------------------- - name: "Create IBM Maximo AI Broker namespace" @@ -57,9 +36,6 @@ # - include_tasks: tasks/saas/main.yml # when: mas_aibroker_saas == true -# # create AI Broker tenant -# - include_tasks: tasks/tenant/main.yml - # # create AI Broker api key # - include_tasks: tasks/apikey/main.yml diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml deleted file mode 100644 index 1fd9cff464..0000000000 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/install/main.yml +++ /dev/null @@ -1,219 +0,0 @@ ---- -- name: "Check if tenant namespace: {{ tenantNamespace }} exists" - kubernetes.core.k8s_info: - api_version: v1 - kind: Namespace - name: "{{ tenantNamespace }}" - register: namespace_info - -# Create tenant namespace -- name: "Create a tenant namespace: {{ tenantNamespace }}" - kubernetes.core.k8s: - name: "{{ tenantNamespace }}" - api_version: v1 - kind: Namespace - when: - - namespace_info.resources | length == 0 - -# Copy secrets to tenant namespace -- name: "Copy secrets to namespace: {{ tenantNamespace }}" - shell: 'oc get secret {{ item }} -n mas-{{ mas_instance_id }}-aibroker -o yaml | sed "s/namespace: .*/namespace: {{ tenantNamespace }}/" | oc apply --force -f -' - with_items: - - km-s3-secret - - "{{ pullSecretName }}" - -# Adding inference server runtimes -- name: "Create config map for connector config" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/connector-configmap.yml.j2') }}" - -- name: "Create SA for server runtimes" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/km-s3-sa.yml.j2') }}" - -- name: "Create huggingfaceserver server runtime" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-huggingfaceserver.yml.j2') }}" - -- name: "Create lgbserver server runtime" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-lgbserver.yml.j2') }}" - -- name: "Create sklearnserver server runtime" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-sklearnserver.yml.j2') }}" - -- name: "Create xgbserver server runtime" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-xgbserver.yml.j2') }}" - -# Applay RBAC roles -- name: "Apply RBAC for tenant" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/rbac.yml.j2') }}" - -- name: "Check if SLS key secret exists in namespace: {{ aibroker_namespace }}" - k8s_info: - api_version: v1 - kind: Secret - name: "{{ tenantNamespace }}----sls-secret" - namespace: "{{ aibroker_namespace }}" - register: sls_secret_info - -# If sls secret does not exist then create one. -- when: - - sls_secret_info.resources | length == 0 - - mas_aibroker_saas == true - block: - - name: "Create cert location {{ mas_aibroker_path_ca_crt }}" - shell: mkdir -p '{{ mas_aibroker_path_ca_crt }}' - - - name: "Insert SLS cert ca to file located {{ mas_aibroker_path_ca_crt }}" - shell: "echo '{{ slscfg.ca }}' > '{{ mas_aibroker_path_ca_crt }}/sls_ca.crt' " - - - name: debug #TODO: remove before final commit - debug: - msg: - - "tenantNamespace ..... {{ tenantNamespace }} " - - "mas_instance_id ..... {{ mas_instance_id }} " - - "mas_aibroker_sls_url ..... {{ slscfg.url }} " - - "mas_aibroker_sls_registration_key ..... {{ slscfg.registration_key }} " - - "mas_aibroker_path_ca_crt ..... {{ mas_aibroker_path_ca_crt }} " - - - name: Create SLS secret - script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" - register: sls_secret_output - - - name: Debug Create sls secret for AI Broker - debug: - msg: "{{ sls_secret_output }}" - -- name: Debug Create sls secret for AI Broker - debug: - msg: "{{ sls_secret_output }}" - -- name: Delete DRO secret - command: oc delete secret "{{ tenantNamespace }}----dro-secret" -n {{ aibroker_namespace }} - register: delete_result - ignore_errors: yes - failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - -- name: Create DRO token - script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ mas_aibroker_dro_token }}" - register: dro_token_output - -- name: Debug Create DRO token - debug: - msg: "{{ dro_token_output }}" - -# - name: Wait for service /service/rest/api/v1/tenant #need more accurate way to check if service working -# pause: -# minutes: 8 -# prompt: "Waiting for for service /service/rest/api/v1/tenant" - -- name: Await to aibroker-api pod created - pause: - minutes: 3 - prompt: "Waiting for aibroker-api pod" - -- name: "Wait for aibroker-api pod" - shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} - register: _is_not_exists - until: _is_not_exists.stdout|int == 0 - retries: 60 - delay: 30 - -- name: "Wait for aibroker-api pod to be ready (30s delay)" - shell: > - oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l - register: _is_not_ready - until: _is_not_ready.stdout|int == 0 - retries: 60 - delay: 30 - -- name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri - uri: - url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenant" - validate_certs: no - method: POST - headers: - apikey: "{{ provision_tenant_api_key }}" - accept: "*/*" - Content-Type: "application/json" - tenantid: "{{ mas_aibroker_provision_tenant }}" - body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"sls_url\": \"{{ mas_aibroker_sls_url }}\", \"dro_url\": \"{{ mas_aibroker_dro_url }}\"}" - status_code: [201] - timeout: 30 - register: add_tenant_to_db2 - -- name: Debug tenant #TODO: remove on final commit - debug: - msg: "{{ add_tenant_to_db2 }}" - -- name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri - uri: - url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" - validate_certs: no - method: POST - headers: - apikey: "{{ provision_tenant_api_key }}" - accept: "*/*" - Content-Type: "application/json" - tenantid: "{{ mas_aibroker_provision_tenant }}" - body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"similarity\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" - status_code: [201] - timeout: 30 - register: add_tenant_entitlements_to_db2 - -- name: Debug tenant entitlement #TODO: remove on final commit - debug: - msg: "{{ add_tenant_entitlements_to_db2 }}" - -- name: Create tenant {{ tenantNamespace }} and add to db2 for pcc - using uri - uri: - url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" - validate_certs: no - method: POST - headers: - apikey: "{{ provision_tenant_api_key }}" - accept: "*/*" - Content-Type: "application/json" - tenantid: "{{ mas_aibroker_provision_tenant }}" - body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"pcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" - status_code: [201] - timeout: 30 - register: add_tenant_entitlements_to_db2_pcc - -- name: Debug tenant entitlement #TODO: remove on final commit - debug: - msg: "{{ add_tenant_entitlements_to_db2_pcc }}" - -- name: Create tenant {{ tenantNamespace }} and add to db2 for mcc - using uri - uri: - url: "https://aibroker.{{ mas_instance_id }}.{{ mas_aibroker_domain }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" - validate_certs: no - method: POST - headers: - apikey: "{{ provision_tenant_api_key }}" - accept: "*/*" - Content-Type: "application/json" - tenantid: "{{ mas_aibroker_provision_tenant }}" - body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"mcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" - status_code: [201] - timeout: 30 - register: add_tenant_entitlements_to_db2_mcc - -- name: Debug tenant entitlement #TODO: remove on final commit - debug: - msg: "{{ add_tenant_entitlements_to_db2_mcc }}" diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/main.yml deleted file mode 100644 index 2161a6a02b..0000000000 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: "Tenant: Debug properties" - debug: - msg: - - "Tenant action ........................ {{ tenant_action }}" - -# 1. Perform the selected action -# ----------------------------------------------------------------------------- -# Two actions are supported: -# - install - -- include_tasks: "tenant/{{ tenant_action }}/main.yml" - when: - - tenant_action != "none" diff --git a/ibm/mas_devops/roles/aibroker/tasks/tenant/remove/main.yml b/ibm/mas_devops/roles/aibroker/tasks/tenant/remove/main.yml deleted file mode 100644 index db88c99c5c..0000000000 --- a/ibm/mas_devops/roles/aibroker/tasks/tenant/remove/main.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -# Tasks to remove tenant resources -- name: Check if models exist before deleting kmodels - shell: oc get isvc,ig -n {{ tenantNamespace }} -o name - register: result - -- name: Check if any models exist - fail: - msg: "Please remove all models before deleting kmodels" - when: result.rc == 0 and result.stdout != '' - -- name: Delete monitor-config ConfigMap - shell: oc delete cm monitor-config -n {{ tenantNamespace }} - ignore_errors: true - register: delete_result - failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - -- name: Delete connector-config ConfigMap - shell: oc delete cm connector-config -n {{ tenantNamespace }} - ignore_errors: true - register: delete_result - failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - -- name: Delete km-s3-sa ServiceAccount - shell: oc delete sa km-s3-sa -n {{ tenantNamespace }} - ignore_errors: true - register: delete_result - failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - -- name: Delete km-s3-secret Secret - shell: oc delete secret km-s3-secret -n {{ tenantNamespace }} - ignore_errors: true - register: delete_result - failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - -- name: Delete regcred Secret - shell: oc delete secret regcred -n {{ tenantNamespace }} - ignore_errors: true - register: delete_result - failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - -- name: Delete tenant namespace - shell: oc delete ns {{ tenantNamespace }} - ignore_errors: true - register: delete_result - failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - -- name: Tenant status - debug: - msg: "Tenant was deleted" diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 deleted file mode 100644 index 10a18e40c9..0000000000 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerworkspace.yml.j2 +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: apps.mas.ibm.com/v1 -kind: AiBrokerWorkspace -metadata: - name: "{{ mas_instance_id }}" - namespace: "{{ aibroker_namespace }}" - labels: - mas.ibm.com/applicationId: aibroker - mas.ibm.com/instanceId: "{{ mas_instance_id }}" -spec: - settings: - icr: - cp: "{{ mas_icr_cp }}" - cpopen: "{{ mas_icr_cpopen }}" - dro: - url: "{{ drocfg.url }}" - dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" - ca: "{{ drocfg.ca }}" - jdbc: - url: "{{ jdbccfg.url }}" - credentials_secret: "{{ mas_aibroker_jdbc_secret }}" - ca: "{{ jdbccfg.ca }}" - sls: - url: "{{ slscfg.url }}" - sls_registration_key_secret: "{{ slscfg.registration_key }}" - ca: "{{ slscfg.ca }}" - tenant: - tenant_id: "{{ tenantNamespace }}" - subscription_id: "{{ mas_aibroker_sls_subscription_id }}" - instance_id: "{{ mas_instance_id }}" - cluster_domain: "{{ cluster_domain }}" - in_saas_env: "{{ in_saas_env }}" - watsonxai: - url: "{{ mas_aibroker_watsonxai_url }}" - api_key: "{{ mas_aibroker_watsonx_secret }}" - project_id: "{{ mas_aibroker_watsonxai_project_id }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh index f942f3e026..e8adca7e44 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh +++ b/ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh @@ -1,7 +1,10 @@ #!/bin/bash -TENANT=$1 -instance_id=$2 +set -e + +ROLE_PATH="${1}" +TENANT=$2 +instance_id=$3 if [ -z ${TENANT} ]; then #echo "❌ Missing tenant name" @@ -19,10 +22,10 @@ mkdir -p certs echo "creating SLS registration, please wait....." # instanceIdentifier=`python3 {{ role_path }}/files/alm_sample_sls_use.py $3 $4` -instanceIdentifier=$(python3 ../roles/aibroker/files/alm_sample_sls_use.py $3 $4 $5) +instanceIdentifier=$(python3 "${ROLE_PATH}/files/alm_sample_sls_use.py" $4 $5 $6) # echo "SLS registration is created successfully." -registrationKey=$4 +registrationKey=$5 # echo "----------" # echo $instanceIdentifier diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/aibroker/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/aibroker/main.yml deleted file mode 100644 index ebe99c6e35..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/aibroker/main.yml +++ /dev/null @@ -1,128 +0,0 @@ ---- -# Provide Debug information -# ----------------------------------------------------------------------------- -- name: "Debug information - IBM Maximo AI Broker" - debug: - msg: - - "Namespace ...................... {{ aibroker_namespace }}" - - "Channel ........................ {{ mas_app_channel }}" - - "MAS Instance Id ................ {{ mas_instance_id }}" - -# # Create IBM Maximo IBM Maximo AI Broker -# # ----------------------------------------------------------------------------- -# - name: "Create IBM Maximo AI Broker namespace" -# kubernetes.core.k8s: -# state: present -# api_version: v1 -# kind: Namespace -# name: "{{ aibroker_namespace }}" - - -# Install the operator & create entitlement secret -# ----------------------------------------------------------------------------- -- name: "Create IBM Entitlement Key" - ibm.mas_devops.update_ibm_entitlement: - namespace: "{{ aibroker_namespace }}" - icr_username: "{{ mas_entitlement_username }}" - icr_password: "{{ mas_entitlement_key }}" - artifactory_username: "{{ artifactory_username }}" - artifactory_password: "{{ artifactory_token }}" - -# - name: "Create ibm-mas-aibroker Subscription" -# ibm.mas_devops.apply_subscription: -# namespace: "{{ aibroker_namespace }}" -# package_name: ibm-mas-aibroker -# package_channel: "{{ aibroker_channel }}" -# catalog_source: "{{ mas_catalog_source }}" -# register: subscription - -- name: "Install AI Broker Operator Group" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/aibroker/operator-group.yml.j2') }}" - -- name: "Install AI Broker Operator" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/aibroker/subscription.yml.j2') }}" - -# Wait until the IBM Maximo AI Broker CRD is available -# ----------------------------------------------------------------------------- -- name: "Wait until the IBM Maximo AI Broker Operator CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: aibrokerapps.apps.mas.ibm.com - -# IBM Maximo AI Broker installation -# ----------------------------------------------------------------------------- -- name: Create AI Broker CR - vars: - annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}" - kubernetes.core.k8s: - state: present - namespace: "{{ aibroker_namespace }}" - template: templates/aibroker/aibrokerapp.yml.j2 - register: aibrokerAppResult - -# Wait IBM Maximo AI Broker custom resource to be complete -# ----------------------------------------------------------------------------- -- name: "Wait for AiBrokerApp custom resource to be ready (60s delay)" - kubernetes.core.k8s_info: - api_version: apps.mas.ibm.com/v1 - name: "{{ mas_instance_id }}" - namespace: "{{ aibroker_namespace }}" - kind: AiBrokerApp - register: aibrokerapp_cr_result - until: - - aibrokerapp_cr_result.resources is defined and aibrokerapp_cr_result.resources | length == 1 - - aibrokerapp_cr_result.resources[0].status is defined - - aibrokerapp_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 - retries: 45 # approx 45 minutes before we give up - delay: 60 # 1 minute - -# IBM Maximo AI Broker workspace -# ----------------------------------------------------------------------------- -- name: Create AI Broker workspace CR - vars: - annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}" - kubernetes.core.k8s: - state: present - namespace: "{{ aibroker_namespace }}" - template: templates/aibroker/aibrokerworkspace.yml.j2 - register: aibrokerWorkspaceResult - -# Wait IBM Maximo AI Broker custom resource to be complete -# ----------------------------------------------------------------------------- -- name: "Wait for AiBrokerWorkspace custom resource to be ready (60s delay)" - kubernetes.core.k8s_info: - api_version: apps.mas.ibm.com/v1 - name: "{{ mas_instance_id }}" - namespace: "{{ aibroker_namespace }}" - kind: AiBrokerWorkspace - register: aibrokerworkspace_cr_result - until: - - aibrokerworkspace_cr_result.resources is defined and aibrokerworkspace_cr_result.resources | length == 1 - - aibrokerworkspace_cr_result.resources[0].status is defined - - aibrokerworkspace_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 - retries: 45 # approx 45 minutes before we give up - delay: 60 # 1 minute - -- name: Await to aibroker-api pod created - pause: - minutes: 3 - prompt: "Waiting for aibroker-api pod" - -- name: "Wait for aibroker-api pod" - shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} - register: _is_not_exists - until: _is_not_exists.stdout|int == 0 - retries: 60 - delay: 30 - -- name: "Wait for aibroker-api pod to be ready (30s delay)" - shell: > - oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l - register: _is_not_ready - until: _is_not_ready.stdout|int == 0 - retries: 60 - delay: 30 diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml deleted file mode 100644 index 9f16dec5bb..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_db2/main.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: "Read jdbc config file" - ansible.builtin.set_fact: - jdbccfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_app_id }}-db2u.yml') | from_yaml_all }}" - -- name: "Set dro mandatory vars" - ansible.builtin.set_fact: - jdbccfg: - # username: "{{ jdbccfg[0].data.username | b64decode}}" - # password: "{{ jdbccfg[0].data.password | b64decode}}" - username: "{{ jdbccfg[0].data.username }}" - password: "{{ jdbccfg[0].data.password }}" - url: "{{ jdbccfg[1].spec.config.url }}" - # ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') }}" - ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" - -- name: "Debug: jdbc information" #TODO: remove before PR - debug: - msg: - - "JDBC username ............. {{ jdbccfg.username }}" - - "JDBC password ............. {{ jdbccfg.password }}" - - "JDBC url .................. {{ jdbccfg.url }}" - - "JDBC cert ca .............. {{ jdbccfg.ca }}" - -# - name: "Create secret for JDBC admin credentials" -# kubernetes.core.k8s: -# apply: yes -# template: "templates/jdbc/jdbc-admin-credentials.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml index 9abf73d078..7f8a652234 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml @@ -2,27 +2,12 @@ - name: Load variables (main) include_vars: "vars/main.yml" -# # Create IBM Maximo IBM Maximo AI Broker -# # ----------------------------------------------------------------------------- -# - name: "Create IBM Maximo AI Broker namespace" -# kubernetes.core.k8s: -# state: present -# api_version: v1 -# kind: Namespace -# name: "{{ aibroker_namespace }}" - # Create config for SLS - include_tasks: tasks/config_sls/main.yml # Create config for DRO - include_tasks: tasks/config_dro/main.yml -# Create config for DB2 -- include_tasks: tasks/config_db2/main.yml - -# # install AI Broker api -# - include_tasks: tasks/aibroker/main.yml - # run api calls for aibroker saas - include_tasks: tasks/saas/main.yml when: mas_aibroker_saas == true diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index 0975f9b196..fa775e7e26 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -15,6 +15,28 @@ when: - namespace_info.resources | length == 0 +- name: "Create tenant CR" + vars: + annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}" + kubernetes.core.k8s: + state: present + namespace: "{{ aibroker_namespace }}" + template: templates/aibroker/aibrokerworkspace.yml.j2 + +- name: "Wait for tenant CR to be ready" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + name: "{{ tenantNamespace }}" + namespace: "{{ aibroker_namespace }}" + kind: AiBrokerWorkspace + register: aibrokerworkspace_cr_result + until: + - aibrokerworkspace_cr_result.resources is defined and aibrokerworkspace_cr_result.resources | length == 1 + - aibrokerworkspace_cr_result.resources[0].status is defined + - aibrokerworkspace_cr_result.resources | json_query('[*].status.conditions[?type==`Successful`][].status') | select ('match','True') | list | length == 1 + retries: 45 # approx 45 minutes before we give up + delay: 60 # 1 minute + # Copy secrets to tenant namespace - name: "Copy secrets to namespace: {{ tenantNamespace }}" shell: 'oc get secret {{ item }} -n mas-{{ mas_instance_id }}-aibroker -o yaml | sed "s/namespace: .*/namespace: {{ tenantNamespace }}/" | oc apply --force -f -' @@ -88,7 +110,7 @@ - "mas_aibroker_path_ca_crt ..... {{ mas_aibroker_path_ca_crt }} " - name: Create SLS secret - script: "{{ role_path }}/files/create_sls_secret.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" + script: "{{ role_path }}/files/create_sls_secret.sh {{ role_path }} {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" register: sls_secret_output - name: Debug Create sls secret for AI Broker @@ -152,7 +174,7 @@ debug: msg: "{{ aibrokerHostName }}" -- name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri +- name: Register DRO and SLS with AI Broker for tenant "{{ tenantNamespace }}" uri: url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenant" validate_certs: no @@ -163,16 +185,20 @@ Content-Type: "application/json" tenantid: "{{ mas_aibroker_provision_tenant }}" body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"sls_url\": \"{{ mas_aibroker_sls_url }}\", \"dro_url\": \"{{ mas_aibroker_dro_url }}\"}" - status_code: [201] + body: "{\"tenant_name\": \"{{ tenantNamespace }}\", + \"sls_url\": \"{{ aibrokerworkspace_cr_result.resources[0].spec.settings.sls.url }}\", + \"dro_url\": \"{{ aibrokerworkspace_cr_result.resources[0].spec.settings.dro.url }}\", + \"subscription_id\": \"{{ mas_aibroker_sls_subscription_id }}\", \"dro_tenant_id\": \"{{ mas_aibroker_dro_tenant_id }}\"}" + status_code: [204, 409] timeout: 30 - register: add_tenant_to_db2 + register: register_tenant -- name: Debug tenant #TODO: remove on final commit +- name: Warn if tenant exists + when: register_tenant.status == 409 debug: - msg: "{{ add_tenant_to_db2 }}" + msg: Warning - tenant {{ tenantNamespace }} is already registered with AI Broker -- name: Create tenant {{ tenantNamespace }} and add to db2 for saas - using uri +- name: Register entitlement to Similarity model for tenant "{{ tenantNamespace }}" uri: url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" validate_certs: no @@ -188,11 +214,7 @@ timeout: 30 register: add_tenant_entitlements_to_db2 -- name: Debug tenant entitlement #TODO: remove on final commit - debug: - msg: "{{ add_tenant_entitlements_to_db2 }}" - -- name: Create tenant {{ tenantNamespace }} and add to db2 for pcc - using uri +- name: Register entitlement to PCC model for tenant "{{ tenantNamespace }}" uri: url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" validate_certs: no @@ -208,11 +230,7 @@ timeout: 30 register: add_tenant_entitlements_to_db2_pcc -- name: Debug tenant entitlement #TODO: remove on final commit - debug: - msg: "{{ add_tenant_entitlements_to_db2_pcc }}" - -- name: Create tenant {{ tenantNamespace }} and add to db2 for mcc - using uri +- name: Register entitlement to MCC model for tenant "{{ tenantNamespace }}" uri: url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" validate_certs: no diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerapp.yml.j2 deleted file mode 100644 index 889ab3929e..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerapp.yml.j2 +++ /dev/null @@ -1,22 +0,0 @@ ---- -apiVersion: apps.mas.ibm.com/v1 -kind: AiBrokerApp -metadata: - name: "{{ mas_instance_id }}" - namespace: "{{ aibroker_namespace }}" - labels: - mas.ibm.com/applicationId: aibroker - mas.ibm.com/instanceId: "{{ mas_instance_id }}" -spec: - settings: - icr: - cp: "{{ mas_icr_cp }}" - cpopen: "{{ mas_icr_cpopen }}" - dro: - url: "{{ drocfg.url }}" - dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" - ca: "{{ drocfg.ca }}" - jdbc: - url: "{{ jdbccfg.url }}" - credentials_secret: "{{ mas_aibroker_jdbc_secret }}" - ca: "{{ jdbccfg.ca }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index acbf45a73d..185b5a1ef2 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -2,13 +2,15 @@ apiVersion: apps.mas.ibm.com/v1 kind: AiBrokerWorkspace metadata: - name: "{{ mas_instance_id }}" + name: "{{ tenantNamespace }}" namespace: "{{ aibroker_namespace }}" labels: mas.ibm.com/applicationId: aibroker mas.ibm.com/instanceId: "{{ mas_instance_id }}" spec: settings: + cluster_domain: "{{ cluster_domain }}" + in_saas_env: "{{ in_saas_env }}" icr: cp: "{{ mas_icr_cp }}" cpopen: "{{ mas_icr_cpopen }}" @@ -16,15 +18,15 @@ spec: url: "{{ drocfg.url }}" dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" ca: "{{ drocfg.ca }}" - jdbc: - url: "{{ jdbccfg.url }}" - credentials_secret: "{{ mas_aibroker_jdbc_secret }}" - ca: "{{ jdbccfg.ca }}" sls: url: "{{ slscfg.url }}" sls_registration_key_secret: "{{ slscfg.registration_key }}" ca: "{{ slscfg.ca }}" + watsonxai: + url: "{{ mas_aibroker_watsonxai_url }}" + api_key: "{{ mas_aibroker_watsonxai_apikey }}" + project_id: "{{ mas_aibroker_watsonxai_project_id }}" tenant: tenant_id: "{{ tenantNamespace }}" subscription_id: "{{ mas_aibroker_sls_subscription_id }}" - instance_id: "{{ mas_instance_id }}" \ No newline at end of file + instance_id: "{{ mas_instance_id }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/jdbc/jdbc-admin-credentials.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/jdbc/jdbc-admin-credentials.yml.j2 deleted file mode 100644 index 7e0ac3bf5f..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/jdbc/jdbc-admin-credentials.yml.j2 +++ /dev/null @@ -1,10 +0,0 @@ ---- -kind: Secret -apiVersion: v1 -metadata: - name: {{ mas_aibroker_jdbc_secret }} - namespace: mas-{{ mas_instance_id }}-aibroker -data: - username: "{{ jdbccfg.username }}" - password: "{{ jdbccfg.password | b64encode }}" -type: Opaque diff --git a/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml b/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml index 9cdcb78c0d..a6699e8356 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml @@ -54,6 +54,7 @@ mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT' # DRO mas_aibroker_dro_token_secret: "dro-token" +mas_aibroker_dro_tenant_id: "{{ lookup('env', 'MAS_AIBROKER_DRO_TENANT_ID') | default('001', true) }}" # JDBC mas_aibroker_jdbc_secret: "jdbc-admin-credentials" From c0354794beead8e508e2f3c6d067d77ef617ad3e Mon Sep 17 00:00:00 2001 From: Joe Harte Date: Mon, 7 Apr 2025 14:19:53 +0100 Subject: [PATCH 137/169] [minor] Improve usage of variables in AI Broker tenant role (#1708) --- .../roles/aibroker_tenant/defaults/main.yml | 68 ++++++++++++------- .../aibroker_tenant/tasks/config_dro/main.yml | 48 ++++++++++--- .../aibroker_tenant/tasks/config_sls/main.yml | 48 ++++++++++--- .../roles/aibroker_tenant/tasks/main.yml | 2 - .../tasks/tenant/install/main.yml | 4 +- .../aibroker/aibrokerworkspace.yml.j2 | 5 ++ .../templates/aibroker/operator-group.yml.j2 | 9 --- .../templates/aibroker/subscription.yml.j2 | 12 ---- .../roles/aibroker_tenant/vars/main.yml | 64 ----------------- 9 files changed, 132 insertions(+), 128 deletions(-) delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/operator-group.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/subscription.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/vars/main.yml diff --git a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml index fc4c738430..d4d4c672ea 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml @@ -1,29 +1,51 @@ --- mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" -aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" +mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" +mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" +mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" -# Custom Labels -# ----------------------------------------------------------------------------- +# Custom labels for resources custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string | ibm.mas_devops.string2dict() }}" -# Source container registry -# ----------------------------------------------------------------------------- -# mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" -# mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" -icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" -icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" - -# MAS Entitlement -# ----------------------------------------------------------------------------- -mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}" -ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" -mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" - -# Development Registry Entitlement -artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" -artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" - -# MAS Annotation block -# ----------------------------------------------------------------------------- -mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" +aibroker_name: "aibroker" +aibroker_namespace: "{{ lookup('env', 'MAS_AIBROKER_NAMESPACE') | default('mas-{}-aibroker'.format(mas_instance_id), true) }}" +aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" +aibroker_pull_secret_name: 'ibm-entitlement' +mas_aibroker_path_ca_crt: './certs' + +# Tenant +mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" +tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" +tenantNamespace: 'aibroker-{{ mas_aibroker_tenant_name }}' + +# API Key +mas_aibroker_apikey_action: "{{ lookup('env', 'MAS_AIBROKER_APIKEY_ACTION') | default('install', true) }}" + +# WatsonX +mas_aibroker_watsonxai_apikey: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY') }}" +mas_aibroker_watsonxai_url: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') }}" +mas_aibroker_watsonxai_project_id: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') }}" +mas_aibroker_watsonx_action: "{{ lookup('env', 'MAS_AIBROKER_WATSONX_ACTION') | default('install', true) }}" + +# S3 +mas_aibroker_storage_host: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_HOST') | default('', true) }}" +mas_aibroker_storage_accesskey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') | default('', true) }}" +mas_aibroker_storage_secretkey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') | default('', true) }}" +mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | default('', true) }}" +mas_aibroker_s3_action: "{{ lookup('env', 'MAS_AIBROKER_S3_ACTION') | default('install', true) }}" +mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" +mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" + +# SAAS +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" +mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" + +# DRO +mas_aibroker_dro_token_secret: "dro-token" +mas_aibroker_dro_tenant_id: "{{ lookup('env', 'MAS_AIBROKER_DRO_TENANT_ID') | default('001', true) }}" + +# SLS +mas_aibroker_sls_registration_key_secret: "sls-registration-key" +mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml index cad63b7662..c2f463a402 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml @@ -1,16 +1,28 @@ --- -- name: "Read dro config file" +- name: Check if DRO config file exists + stat: + path: "{{ mas_config_dir }}/dro.yml" + register: dro_file_result + +- name: Read DRO config from file + vars: + dro_file_cfg: "{{ lookup('file', '{{ mas_config_dir }}/dro.yml') | from_yaml_all }}" ansible.builtin.set_fact: - drocfg: "{{ lookup('file', '{{ mas_config_dir }}/dro.yml') | from_yaml_all }}" + drocfg: + secret_name: "{{ dro_file_cfg[0].metadata.name }}" + registration_key: "{{ dro_file_cfg[0].stringData.api_key }}" + url: "{{ dro_file_cfg[1].spec.config.url }}" + ca: "{{ dro_file_cfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" + when: dro_file_result.stat.exists -- name: "Set dro mandatory vars" +- name: Read DRO config from environment ansible.builtin.set_fact: drocfg: - secret_name: "{{ drocfg[0].metadata.name }}" - registration_key: "{{ drocfg[0].stringData.api_key }}" - url: "{{ drocfg[1].spec.config.url }}" - ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" - # ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') }}" + secret_name: "{{ lookup('env', 'MAS_AIBROKER_DRO_SECRET_NAME') }}" + registration_key: "{{ lookup('env', 'MAS_AIBROKER_DRO_API_KEY') }}" + url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') }}" + ca: "{{ lookup('env', 'MAS_AIBROKER_DRO_CA_CERT') }}" + when: not dro_file_result.stat.exists - name: "Debug: dro information" #TODO: remove before PR debug: @@ -20,6 +32,26 @@ - "dro url ................. {{ drocfg.url }}" - "dro cert ca ............. {{ drocfg.ca }}" +- name: "Validate DRO configuration" + when: drocfg.secret_name | length == 0 + fail: + msg: "drocfg.secret_name must not empty" + +- name: "Validate DRO configuration" + when: drocfg.registration_key | length == 0 + fail: + msg: "drocfg.registration_key must not empty" + +- name: "Validate DRO configuration" + when: drocfg.url | length == 0 + fail: + msg: "drocfg.url must not empty" + +- name: "Validate DRO configuration" + when: drocfg.ca | length == 0 + fail: + msg: "drocfg.ca must not empty" + # - name: "Create secret for Dro registration key" # kubernetes.core.k8s: # apply: yes diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml index c167d5fdbf..d0205d0a9e 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml @@ -1,16 +1,28 @@ --- -- name: "Read sls config file" +- name: Check if SLS config file exists + stat: + path: "{{ mas_config_dir }}/sls.yml" + register: sls_file_result + +- name: Read SLS config from file + vars: + sls_file_cfg: "{{ lookup('file', '{{ mas_config_dir }}/sls.yml') | from_yaml_all }}" ansible.builtin.set_fact: - slscfg: "{{ lookup('file', '{{ mas_config_dir }}/sls.yml') | from_yaml_all }}" + slscfg: + secret_name: "{{ sls_file_cfg[0].metadata.name }}" + registration_key: "{{ sls_file_cfg[0].stringData.registrationKey }}" + url: "{{ sls_file_cfg[1].spec.config.url }}" + ca: "{{ sls_file_cfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" + when: sls_file_result.stat.exists -- name: "Set sls mandatory vars" +- name: Read SLS config from environment ansible.builtin.set_fact: slscfg: - secret_name: "{{ slscfg[0].metadata.name }}" - registration_key: "{{ slscfg[0].stringData.registrationKey }}" - url: "{{ slscfg[1].spec.config.url }}" - ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" - # ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') }}" + secret_name: "{{ lookup('env', 'MAS_AIBROKER_SLS_SECRET_NAME') }}" + registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') }}" + url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') }}" + ca: "{{ lookup('env', 'MAS_AIBROKER_SLS_CA_CERT') }}" + when: not sls_file_result.stat.exists - name: "Debug: sls information" #TODO: remove before PR debug: @@ -20,6 +32,26 @@ - "sls url ................. {{ slscfg.url }}" - "sls cert ca ............. {{ slscfg.ca }}" +- name: "Validate SLS configuration" + when: slscfg.secret_name | length == 0 + fail: + msg: "slscfg.secret_name must not empty" + +- name: "Validate SLS configuration" + when: slscfg.registration_key | length == 0 + fail: + msg: "slscfg.registration_key must not empty" + +- name: "Validate SLS configuration" + when: slscfg.url | length == 0 + fail: + msg: "slscfg.url must not empty" + +- name: "Validate SLS configuration" + when: slscfg.ca | length == 0 + fail: + msg: "slscfg.ca must not empty" + # - name: "Create secret for SLS registration key" # kubernetes.core.k8s: # apply: yes diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml index 7f8a652234..c517f994a7 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/main.yml @@ -1,6 +1,4 @@ --- -- name: Load variables (main) - include_vars: "vars/main.yml" # Create config for SLS - include_tasks: tasks/config_sls/main.yml diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index fa775e7e26..52bdab76a7 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -42,7 +42,7 @@ shell: 'oc get secret {{ item }} -n mas-{{ mas_instance_id }}-aibroker -o yaml | sed "s/namespace: .*/namespace: {{ tenantNamespace }}/" | oc apply --force -f -' with_items: - km-s3-secret - - "{{ pullSecretName }}" + - "{{ aibroker_pull_secret_name }}" # Adding inference server runtimes - name: "Create config map for connector config" @@ -160,7 +160,7 @@ kubernetes.core.k8s_info: api_version: route.openshift.io/v1 kind: Route - name: "{{ aibroker.name }}" + name: "{{ aibroker_name }}" namespace: "{{ aibroker_namespace }}" wait: yes wait_timeout: 30 diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 185b5a1ef2..095408e4db 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -7,6 +7,11 @@ metadata: labels: mas.ibm.com/applicationId: aibroker mas.ibm.com/instanceId: "{{ mas_instance_id }}" +{% if custom_labels is defined and custom_labels.items() %} +{% for key, value in custom_labels.items() %} + "{{ key }}": "{{ value }}" +{% endfor %} +{% endif %} spec: settings: cluster_domain: "{{ cluster_domain }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/operator-group.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/operator-group.yml.j2 deleted file mode 100644 index 5aab10febd..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/operator-group.yml.j2 +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: mas-{{ mas_instance_id }}-aibroker - namespace: "{{ aibroker_namespace }}" -spec: - targetNamespaces: - - "{{ aibroker_namespace }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/subscription.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/subscription.yml.j2 deleted file mode 100644 index dc7a429078..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/subscription.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: ibm-mas-aibroker - namespace: "{{ aibroker_namespace }}" -spec: - channel: "{{ mas_app_channel }}" - installPlanApproval: Automatic - name: ibm-mas-aibroker - source: "{{ mas_catalog_source }}" - sourceNamespace: openshift-marketplace diff --git a/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml b/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml deleted file mode 100644 index a6699e8356..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/vars/main.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -# aibroker vars -aibroker: - name: "aibroker" - -aibroker_namespace: 'mas-{{ mas_instance_id }}-aibroker' -mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" -aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" -mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" -pullSecretName: 'ibm-entitlement' - -# Whether to perform an install or remove tenant -tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" - -# Create Api Key vars -mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" -tenantNamespace: 'aibroker-{{ mas_aibroker_tenant_name }}' -# app_domain: "{{ lookup('env', 'APP_DOMAIN') }}" -mas_aibroker_apikey_action: "{{ lookup('env', 'MAS_AIBROKER_APIKEY_ACTION') | default('install', true) }}" - -# Main variables -ibm_entitlement_username: "{{ lookup('env','IBM_ENTITLEMENT_USERNAME') }}" -ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" - -# Source container registry -# ----------------------------------------------------------------------------- -mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" -mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" - -# WatsonX -mas_aibroker_watsonxai_apikey: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY') }}" -mas_aibroker_watsonxai_url: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') }}" -mas_aibroker_watsonxai_project_id: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') }}" -mas_aibroker_watsonx_action: "{{ lookup('env', 'MAS_AIBROKER_WATSONX_ACTION') | default('install', true) }}" - -# S3 -mas_aibroker_storage_host: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_HOST') | default('', true) }}" -mas_aibroker_storage_accesskey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') | default('', true) }}" -mas_aibroker_storage_secretkey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') | default('', true) }}" -mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | default('', true) }}" -mas_aibroker_s3_action: "{{ lookup('env', 'MAS_AIBROKER_S3_ACTION') | default('install', true) }}" -mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" -mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" - -# SAAS -mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" -mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" -mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" -mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" -mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" -mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" -mas_aibroker_path_ca_crt: './certs' -mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" - -# DRO -mas_aibroker_dro_token_secret: "dro-token" -mas_aibroker_dro_tenant_id: "{{ lookup('env', 'MAS_AIBROKER_DRO_TENANT_ID') | default('001', true) }}" - -# JDBC -mas_aibroker_jdbc_secret: "jdbc-admin-credentials" - -# SLS -mas_aibroker_sls_registration_key_secret: "sls-registration-key" -mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" From fc29e71853677260010dccd1e938e8a5d902112c Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 10 Apr 2025 14:53:53 +0100 Subject: [PATCH 138/169] [patch] removed wait for aibroker-api, and route tasks which are not needed on this role --- .../tasks/tenant/install/main.yml | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index 52bdab76a7..1f61dc35f8 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -136,43 +136,43 @@ # minutes: 8 # prompt: "Waiting for for service /service/rest/api/v1/tenant" -- name: Await to aibroker-api pod created - pause: - minutes: 3 - prompt: "Waiting for aibroker-api pod" - -- name: "Wait for aibroker-api pod" - shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} - register: _is_not_exists - until: _is_not_exists.stdout|int == 0 - retries: 60 - delay: 30 - -- name: "Wait for aibroker-api pod to be ready (30s delay)" - shell: > - oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l - register: _is_not_ready - until: _is_not_ready.stdout|int == 0 - retries: 60 - delay: 30 - -- name: Lookup the aibroker route - kubernetes.core.k8s_info: - api_version: route.openshift.io/v1 - kind: Route - name: "{{ aibroker_name }}" - namespace: "{{ aibroker_namespace }}" - wait: yes - wait_timeout: 30 - register: aibrokerRoute - -- name: Set aibroker route host - ansible.builtin.set_fact: - aibrokerHostName: "{{ aibrokerRoute.resources[0].spec.host }}" - -- name: Debug aibrokerRoute - debug: - msg: "{{ aibrokerHostName }}" +# - name: Await to aibroker-api pod created +# pause: +# minutes: 3 +# prompt: "Waiting for aibroker-api pod" + +# - name: "Wait for aibroker-api pod" +# shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} +# register: _is_not_exists +# until: _is_not_exists.stdout|int == 0 +# retries: 60 +# delay: 30 + +# - name: "Wait for aibroker-api pod to be ready (30s delay)" +# shell: > +# oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l +# register: _is_not_ready +# until: _is_not_ready.stdout|int == 0 +# retries: 60 +# delay: 30 + +# - name: Lookup the aibroker route +# kubernetes.core.k8s_info: +# api_version: route.openshift.io/v1 +# kind: Route +# name: "{{ aibroker_name }}" +# namespace: "{{ aibroker_namespace }}" +# wait: yes +# wait_timeout: 30 +# register: aibrokerRoute + +# - name: Set aibroker route host +# ansible.builtin.set_fact: +# aibrokerHostName: "{{ aibrokerRoute.resources[0].spec.host }}" + +# - name: Debug aibrokerRoute +# debug: +# msg: "{{ aibrokerHostName }}" - name: Register DRO and SLS with AI Broker for tenant "{{ tenantNamespace }}" uri: From 6c1f36ab06b27723e4b181cd8e7958629b1b9ffd Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 10 Apr 2025 15:09:18 +0100 Subject: [PATCH 139/169] [patch] roll back route as it is needed for register DRO and SLS --- .../tasks/tenant/install/main.yml | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index 1f61dc35f8..fb32736d79 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -156,23 +156,23 @@ # retries: 60 # delay: 30 -# - name: Lookup the aibroker route -# kubernetes.core.k8s_info: -# api_version: route.openshift.io/v1 -# kind: Route -# name: "{{ aibroker_name }}" -# namespace: "{{ aibroker_namespace }}" -# wait: yes -# wait_timeout: 30 -# register: aibrokerRoute - -# - name: Set aibroker route host -# ansible.builtin.set_fact: -# aibrokerHostName: "{{ aibrokerRoute.resources[0].spec.host }}" - -# - name: Debug aibrokerRoute -# debug: -# msg: "{{ aibrokerHostName }}" +- name: Lookup the aibroker route + kubernetes.core.k8s_info: + api_version: route.openshift.io/v1 + kind: Route + name: "{{ aibroker_name }}" + namespace: "{{ aibroker_namespace }}" + wait: yes + wait_timeout: 30 + register: aibrokerRoute + +- name: Set aibroker route host + ansible.builtin.set_fact: + aibrokerHostName: "{{ aibrokerRoute.resources[0].spec.host }}" + +- name: Debug aibrokerRoute + debug: + msg: "{{ aibrokerHostName }}" - name: Register DRO and SLS with AI Broker for tenant "{{ tenantNamespace }}" uri: From cb81375bce3b8c9cff2066ad917828f12f311205 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 10 Apr 2025 15:44:27 +0100 Subject: [PATCH 140/169] [patch] update playbook docs for aibroker --- docs/playbooks/oneclick-aibroker.md | 139 +++++++++++++++------------- 1 file changed, 75 insertions(+), 64 deletions(-) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index 1f7804e864..038c57cdcc 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -6,12 +6,12 @@ You will need a RedHat OpenShift v4.14 or above. ### Dependencies: -* IBM Suite License Service installed on OCP cluster or external instance -* IBM Data Reporter Operator installed on OCP cluster or external instance +* IBM Suite License Service installed on OCP cluster or external instance or details from external instance +* IBM Data Reporter Operator installed on OCP cluster or external instance or details from external instance * Object Storage - + Minio (installed on the same cluster what aibroker) - + AWS S3 (if customer use AWS S3 bucket bucket) buckets needs to have unique names -* MariaDB database (installed in cluster where aibroker instance) or in AWS + + Minio (installed on the same cluster what aibroker) or external instance or details from external instance + + AWS S3 (if customer use AWS S3 bucket bucket) buckets needs to have unique names +* MariaDB database (installed in cluster where aibroker instance) or external instance or details from external instance ## Overview @@ -25,8 +25,8 @@ This playbook can be ran against any OCP cluster regardless of its type; whether + Install MariaDB (~5 minutes) **optional** + Install Minio (~5 minutes) **optional** * Install AI broker application (using playbook): - + Install application (~10 Minutes) - + Configure AI Broker (kmodels, tenant, etc) (~5 Minutes) + + Install application (~20 Minutes) + + Configure AI Broker (kmodels, tenant, etc) (~20 Minutes) All timings are estimates, see the individual pages for each of these playbooks for more information. Use this sample playbook as a starting point for installing application, just customize the application install and configure stages at the end of the playbook. @@ -100,48 +100,53 @@ source /tmp/venv/bin/activate python3 -m pip install boto3 ``` -#### Run playbooks for deploy AI Broker on SAAS +#### Run playbooks for deploy AI Broker ```bash +git checkout aib2op +git pull + export ARTIFACTORY_USERNAME="" export ARTIFACTORY_TOKEN="" -export MAS_ICR_CP="docker-na-public.artifactory.swg-devops.com/wiotp-docker-local" -export MAS_ICR_CPOPEN="docker-na-public.artifactory.swg-devops.com/wiotp-docker-local/cpopen" -export MAS_CATALOG_VERSION="v9-xxx-amd64" -export MAS_ENTITLEMENT_USERNAME="" -export MAS_ENTITLEMENT_KEY="" -export IBM_ENTITLEMENT_KEY=${MAS_ENTITLEMENT_KEY} -export MAS_INSTANCE_ID="" -export MAS_APP_CHANNEL="9.1.x" -export MAS_CONFIG_DIR="config_path_location" +export MAS_ENTITLEMENT_USERNAME="" +export MAS_ENTITLEMENT_KEY="" +export MAS_INSTANCE_ID="" +export MAS_APP_CHANNEL="" +export MAS_CATALOG_VERSION="" export IBM_ENTITLEMENT_KEY=${MAS_ENTITLEMENT_KEY} -# export MAS_AIBROKER_SAAS="true" -# export MAS_AIBROKER_DOMAIN="apps.domain" -# export MAS_AIBROKER_SLS_URL="https://sls.ibm-sls.ibm-sls."${MAS_AIBROKER_DOMAIN} -# export MAS_AIBROKER_SLS_REGISTRATION_KEY="xxxxxxx" -# export MAS_AIBROKER_DRO_URL="https://ibm-data-reporter-redhat-marketplace."${MAS_AIBROKER_DOMAIN} -# export MAS_AIBROKER_DRO_TOKEN="xxx" -# export DB2_INSTANCE_NAME="aibroker" -# export MAS_AIBROKER_STORAGE_ACCESSKEY="" -# export MAS_AIBROKER_STORAGE_SECRETKEY="" -# export MAS_AIBROKER_STORAGE_HOST="" -# export MAS_AIBROKER_STORAGE_SSL="true or false" -# export MAS_AIBROKER_STORAGE_REGION="" -# export MAS_AIBROKER_STORAGE_PROVIDER="" -# export MAS_AIBROKER_STORAGE_PORT="" -# export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" -# export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" -# export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" -# export MAS_AIBROKER_WATSONXAI_APIKEY="" -# export MAS_AIBROKER_WATSONXAI_URL="" -# export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" -# export MAS_AIBROKER_DB_HOST="" -# export MAS_AIBROKER_DB_PORT="" -# export MAS_AIBROKER_DB_USER="" -# export MAS_AIBROKER_DB_DATABASE="" -# export MAS_AIBROKER_DB_SECRET_NAME="" -# export MAS_AIBROKER_DB_SECRET_VALUE="" - +export MAS_CONFIG_DIR="/home/user/" +export DRO_CONTACT_EMAIL="xxx@ibm.com" +export DRO_CONTACT_FIRSTNAME="xxx" +export DRO_CONTACT_LASTNAME="xxx" +export SLS_MONGODB_CFG_FILE=${MAS_CONFIG_DIR}/mongo-mongoce.yml +export MINIO_ROOT_PASSWORD="" +export MAS_AIBROKER_STORAGE_ACCESSKEY="" +export MAS_AIBROKER_STORAGE_SECRETKEY="" +export MAS_AIBROKER_STORAGE_HOST="" +export MAS_AIBROKER_STORAGE_SSL="" +export MAS_AIBROKER_STORAGE_PROVIDER="" +export MAS_AIBROKER_STORAGE_PORT="" +export MAS_AIBROKER_STORAGE_REGION="" +export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" +export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" +export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" +export MARIADB_PASSWORD="" +export MAS_AIBROKER_DB_HOST="" +export MAS_AIBROKER_DB_PORT="" +export MAS_AIBROKER_DB_USER="" +export MAS_AIBROKER_DB_DATABASE="" +export MAS_AIBROKER_DB_SECRET_NAME="" +export MAS_AIBROKER_DB_SECRET_VALUE="" +export MAS_AIBROKER_WATSONXAI_APIKEY="" +export MAS_AIBROKER_WATSONXAI_URL="" +export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" +export MAS_AIBROKER_SAAS="true" +export INSAASENV="True" +export MAS_AIBROKER_SUBSCRIPTION_ID="" +export MAS_AIBROKER_DRO_TENANT_ID="" +oc create namespace openshift-serverless +oc create namespace cert-manager-operator +oc create namespace ibm-sls oc login --token=xxxx --server=https://myocpserver ansible-playbook playbooks/oneclick_add_aibroker.yml ``` @@ -150,7 +155,17 @@ ansible-playbook playbooks/oneclick_add_aibroker.yml * `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` * in `AWS` for `MAS_AIBROKER_STORAGE_PIPELINES_BUCKET`, `MAS_AIBROKER_STORAGE_TENANTS_BUCKET`, `MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET` user need to create S3 buckets with unique name -## NOTICE: playbook oneclick_add_aibroker.yml will run three roles: +## NOTICE: playbook oneclick_add_aibroker.yml will run roles: + +### Roles: * optional + - ibm.mas_devops.ibm_catalogs + - ibm.mas_devops.cert_manager + - ibm.mas_devops.mongodb + - ibm.mas_devops.sls + - ibm.mas_devops.dro + - ibm.mas_devops.db2 + - ibm.mas_devops.minio + - ibm.mas_devops.mariadb ### Role: odh @@ -172,6 +187,9 @@ ansible-playbook playbooks/oneclick_add_aibroker.yml ### Role: aibroker * Install AI Broker api application + +### Role: aibroker + * Create AI Broker tenant * Create, delete AI Broker API Key * Create, delete AWS S3 API Key @@ -284,29 +302,22 @@ ansible-playbook playbooks/run_role.yml # create tanant SAAS ```bash -export MAS_AIBROKER_TENANT_NAME="tenant1" +export MAS_AIBROKER_TENANT_NAME="user7" +export MAS_AIBROKER_SLS_SUBSCRIPTION_ID="007" export TENANT_ACTION="install" -export MAS_INSTANCE_ID="aibdev" -export MAS_AIBROKER_DOMAIN="apps.aibrokerdev.cp.fyre.ibm.com" +export ROLE_NAME="aibroker_tenant" +export MAS_AIBROKER_SAAS="true" +export MAS_AIBROKER_DOMAIN="" export MAS_AIBROKER_SLS_URL="https://sls.ibm-sls.ibm-sls."${MAS_AIBROKER_DOMAIN} -export MAS_AIBROKER_DRO_URL="https://ibm-data-reporter-redhat-marketplace."${MAS_AIBROKER_DOMAIN} export MAS_AIBROKER_SLS_REGISTRATION_KEY="" +export MAS_AIBROKER_DRO_URL="https://ibm-data-reporter-redhat-marketplace."${MAS_AIBROKER_DOMAIN} export MAS_AIBROKER_DRO_TOKEN="" -export MAS_AIBROKER_SAAS="true" -export MAS_AIBROKER_STORAGE_ACCESSKEY="" -export MAS_AIBROKER_STORAGE_SECRETKEY="" -export MAS_AIBROKER_STORAGE_HOST="" -export MAS_AIBROKER_STORAGE_SSL="true or false" -export MAS_AIBROKER_STORAGE_REGION="" -export MAS_AIBROKER_STORAGE_PROVIDER="" -export MAS_AIBROKER_STORAGE_PORT="" -export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" -export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" -export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" -export MAS_AIBROKER_WATSONXAI_APIKEY="" -export MAS_AIBROKER_WATSONXAI_URL="" -export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" -export ROLE_NAME="aibroker" +export MAS_AIBROKER_SLS_CACERT="" +export MAS_AIBROKER_DRO_CACERT="" +export MAS_AIBROKER_WATSONXAI_APIKEY="" +export MAS_AIBROKER_WATSONXAI_URL="" +export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" +oc login --token=xxxx --server=https://myocpserver ansible-playbook playbooks/run_role.yml ``` From 858dc7ec7308e674e82e1b59810861bc7d78ac50 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Fri, 11 Apr 2025 16:58:38 +0100 Subject: [PATCH 141/169] [patch] added to create tenant part needed vars for storage --- docs/playbooks/oneclick-aibroker.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index 038c57cdcc..0f410f6fef 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -317,6 +317,16 @@ export MAS_AIBROKER_DRO_CACERT="" export MAS_AIBROKER_WATSONXAI_APIKEY="" export MAS_AIBROKER_WATSONXAI_URL="" export MAS_AIBROKER_WATSONXAI_PROJECT_ID="" +export MAS_AIBROKER_STORAGE_ACCESSKEY="" +export MAS_AIBROKER_STORAGE_SECRETKEY="" +export MAS_AIBROKER_STORAGE_HOST="" +export MAS_AIBROKER_STORAGE_SSL="" +export MAS_AIBROKER_STORAGE_PROVIDER="" +export MAS_AIBROKER_STORAGE_PORT="" +export MAS_AIBROKER_STORAGE_REGION="" +export MAS_AIBROKER_STORAGE_PIPELINES_BUCKET="" +export MAS_AIBROKER_STORAGE_TENANTS_BUCKET="" +export MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET="" oc login --token=xxxx --server=https://myocpserver ansible-playbook playbooks/run_role.yml ``` From 5dd02381ef7d7712c6058804c4d8ff5bd23fa3a5 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 14 Apr 2025 15:32:37 +0100 Subject: [PATCH 142/169] [patch] update kmodels to be inline versions for SAAS release --- ibm/mas_devops/roles/kmodels/vars/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ibm/mas_devops/roles/kmodels/vars/main.yml b/ibm/mas_devops/roles/kmodels/vars/main.yml index 7a4f767474..a710672834 100644 --- a/ibm/mas_devops/roles/kmodels/vars/main.yml +++ b/ibm/mas_devops/roles/kmodels/vars/main.yml @@ -2,17 +2,17 @@ # vars file for kmodels pullSecretName: "ibm-entitlement" # controller -mas_aibroker_controller_tag: 1.0.2 +mas_aibroker_controller_tag: 1.0.7 controller: "aibroker/controller:{{ mas_aibroker_controller_tag }}" # store -mas_aibroker_store_tag: 1.0.2 +mas_aibroker_store_tag: 1.0.3 store: "aibroker/store:{{ mas_aibroker_store_tag }}" # watcher -mas_aibroker_watcher_tag: 1.0.2-pre.dev10x +mas_aibroker_watcher_tag: 1.0.4 watcher: "aibroker/watcher:{{ mas_aibroker_watcher_tag }}" -mas_aibroker_connector_tag: 1.0.0 -mas_aibroker_pipeline_steps_tag: 1.0.0 +mas_aibroker_connector_tag: 1.0.1 +mas_aibroker_pipeline_steps_tag: 1.0.1 # Bucket names mas_aibroker_storage_pipelines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') }}" From 9f6b251c8735fd9c3dd13416e2a29db4504eae05 Mon Sep 17 00:00:00 2001 From: Bhautiksinh Vala <94218990+Bhautik-Vala@users.noreply.github.com> Date: Mon, 14 Apr 2025 20:41:01 +0530 Subject: [PATCH 143/169] [patch] change var IN_SAAS_ENV to MAS_AIBROKER_SAAS - remove vars duplicacy (#1712) Co-authored-by: Bhautik Vala --- ibm/mas_devops/playbooks/oneclick_add_aibroker.yml | 6 +++--- .../roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 | 2 +- .../templates/aibroker/aibrokerworkspace.yml.j2 | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 547d7b5543..71076d4638 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -39,9 +39,9 @@ debug: msg: "Cluster domain ........................ {{ cluster_domain }}" - - name: "Check if SaaS environment" - set_fact: - in_saas_env: "{{ lookup('env', 'INSAASENV') | default('false', true) | bool }}" + # - name: "Check if SaaS environment" + # set_fact: + # in_saas_env: "{{ lookup('env', 'INSAASENV') | default('false', true) | bool }}" # For the full set of supported environment variables refer to the playbook documentation - name: Check for required common environment variables diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index 772fab691d..6850f2a608 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -21,7 +21,7 @@ spec: credentials_secret: "{{ mas_aibroker_jdbc_secret }}" ca: "{{ jdbccfg.ca | b64decode }}" cluster_domain: "{{ cluster_domain }}" - in_saas_env: "{{ in_saas_env }}" + in_saas_env: "{{ mas_aibroker_saas }}" minio: host: "{{ mas_aibroker_storage_host }}" port: "{{ mas_aibroker_storage_port }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 095408e4db..7b0e2165ac 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -15,7 +15,7 @@ metadata: spec: settings: cluster_domain: "{{ cluster_domain }}" - in_saas_env: "{{ in_saas_env }}" + in_saas_env: "{{ mas_aibroker_saas }}" icr: cp: "{{ mas_icr_cp }}" cpopen: "{{ mas_icr_cpopen }}" From 1ab29780508f952b05f00a21599c82d71fc81b2d Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Tue, 22 Apr 2025 19:40:45 +0530 Subject: [PATCH 144/169] [patch] Adding conditional check for mongodb, sls, dro, db2, mariadb,minio (#1717) --- .../playbooks/oneclick_add_aibroker.yml | 61 ++++++++++++++---- .../roles/aibroker/tasks/config_db2/main.yml | 62 +++++++++++++++---- .../roles/aibroker/tasks/config_dro/main.yml | 51 ++++++++++++--- .../roles/aibroker/tasks/config_sls/main.yml | 54 ++++++++++++---- 4 files changed, 185 insertions(+), 43 deletions(-) diff --git a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml index 71076d4638..108a92e06e 100644 --- a/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml +++ b/ibm/mas_devops/playbooks/oneclick_add_aibroker.yml @@ -23,6 +23,15 @@ # ----------------------------------------------------------------------------- db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') | default('aibroker', true) }}" + # Control dependencies + # ----------------------------------------------------------------------------- + install_db2: "{{ lookup('env', 'INSTALL_DB2') | default('False', true) | bool }}" + install_minio: "{{ lookup('env', 'INSTALL_MINIO') | default('False', true) | bool }}" + install_mariadb: "{{ lookup('env', 'INSTALL_MARIADB') | default('False', true) | bool }}" + install_mongo: "{{ lookup('env', 'INSTALL_MONGO') | default('False', true) | bool }}" + install_sls: "{{ lookup('env', 'INSTALL_SLS') | default('False', true) | bool }}" + install_dro: "{{ lookup('env', 'INSTALL_DRO') | default('False', true) | bool }}" + pre_tasks: - name: "Get cluster domain" kubernetes.core.k8s_info: @@ -79,16 +88,44 @@ # fail_msg: "One or more required environment variables are not defined" # when: mas_aibroker_saas == true + # - include_role: + # name: ibm.mas_devops.mongodb + # when: install_mongo == true + # - include_role: + # name: ibm.mas_devops.sls + # when: install_sls == true + # - include_role: + # name: ibm.mas_devops.dro + # when: install_dro == true + # - include_role: + # name: ibm.mas_devops.db2 + # when: install_db2 == true + # - include_role: + # name: ibm.mas_devops.suite_config + # when: install_db2 == true + # - include_role: + # name: ibm.mas_devops.minio + # when: install_minio == true + # - include_role: + # name: ibm.mas_devops.mariadb + # when: install_mariadb == true + roles: - - ibm.mas_devops.ibm_catalogs - - ibm.mas_devops.cert_manager - - ibm.mas_devops.mongodb - - ibm.mas_devops.sls - - ibm.mas_devops.dro - - ibm.mas_devops.db2 - - ibm.mas_devops.minio - - ibm.mas_devops.mariadb - - ibm.mas_devops.odh - - ibm.mas_devops.kmodels - - ibm.mas_devops.aibroker - - ibm.mas_devops.aibroker_tenant + - role: ibm.mas_devops.ibm_catalogs + - role: ibm.mas_devops.cert_manager + - role: ibm.mas_devops.mongodb + when: install_mongo == true + - role: ibm.mas_devops.sls + when: install_sls == true + - role: ibm.mas_devops.dro + when: install_dro == true + - role: ibm.mas_devops.db2 + when: install_db2 == true + - role: ibm.mas_devops.minio + when: install_minio == true + - role: ibm.mas_devops.mariadb + when: install_mariadb == true + - role: ibm.mas_devops.odh + - role: ibm.mas_devops.aibroker + - role: ibm.mas_devops.kmodels + - role: ibm.mas_devops.aibroker_tenant diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml index 88a8adf973..51526812c4 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_db2/main.yml @@ -1,25 +1,65 @@ --- -- name: "Read jdbc config file" +- name: Check if JDBC config file exists + ansible.builtin.stat: + path: "{{ mas_config_dir }}/jdbc-{{ mas_app_id }}-db2u.yml" + register: jdbc_config_file_result + +- name: Read JDBC config from file + vars: + jdbc_file_cfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_app_id }}-db2u.yml') | from_yaml_all }}" ansible.builtin.set_fact: - jdbccfg: "{{ lookup('file', '{{ mas_config_dir }}/jdbc-{{ mas_app_id }}-db2u.yml') | from_yaml_all }}" + jdbccfg: + username: "{{ jdbc_file_cfg[0].data.username }}" + password: "{{ jdbc_file_cfg[0].data.password }}" + url: "{{ jdbc_file_cfg[1].spec.config.url | b64encode }}" + sslenabled: "{{ jdbc_file_cfg[1].spec.config.sslEnabled | b64encode }}" + ca: "{{ jdbc_file_cfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') | b64encode }}" + when: jdbc_config_file_result.stat.exists -- name: "Set dro mandatory vars" +- name: Read JDBC config from environment ansible.builtin.set_fact: jdbccfg: - username: "{{ jdbccfg[0].data.username }}" - password: "{{ jdbccfg[0].data.password }}" - url: "{{ jdbccfg[1].spec.config.url | b64encode }}" - sslenabled: "{{ jdbccfg[1].spec.config.sslEnabled | b64encode }}" - ca: "{{ jdbccfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') | b64encode }}" - -- name: "Debug: jdbc information" #TODO: remove before PR - debug: + username: "{{ lookup('env', 'MAS_AIBROKER_DB2_USERNAME') }}" + password: "{{ lookup('env', 'MAS_AIBROKER_DB2_PASSWORD') }}" + url: "{{ lookup('env', 'MAS_AIBROKER_DB2_JDBC_URL') }}" + sslenabled: "{{ lookup('env', 'MAS_AIBROKER_DB2_SSL_ENABLED') | default('True', true) | bool }}" + ca: "{{ lookup('env', 'MAS_AIBROKER_DB2_CA_CERT') }}" + when: not jdbc_config_file_result.stat.exists + +- name: "Debug: jdbc information" # TODO: remove before PR + ansible.builtin.debug: msg: - "JDBC username ............. {{ jdbccfg.username }}" - "JDBC password ............. {{ jdbccfg.password }}" - "JDBC url .................. {{ jdbccfg.url }}" + - "JDBC sslenabled ........... {{ jdbccfg.sslenabled }}" - "JDBC cert ca .............. {{ jdbccfg.ca }}" +- name: "Validate JDBC configuration" + when: jdbccfg.username | length == 0 + ansible.builtin.fail: + msg: "jdbccfg.username must not be empty" + +- name: "Validate JDBC configuration" + when: jdbccfg.password | length == 0 + ansible.builtin.fail: + msg: "jdbccfg.password must not be empty" + +- name: "Validate JDBC configuration" + when: jdbccfg.url | length == 0 + ansible.builtin.fail: + msg: "jdbccfg.url must not be empty" + +- name: "Validate JDBC configuration" + when: jdbccfg.sslenabled | length == 0 + ansible.builtin.fail: + msg: "jdbccfg.sslenabled must not be empty" + +- name: "Validate JDBC configuration" + when: jdbccfg.ca | length == 0 + ansible.builtin.fail: + msg: "jdbccfg.ca must not be empty" + - name: "Create secret for JDBC admin credentials" kubernetes.core.k8s: apply: yes diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml index 5ac7e1d06e..ff30dc54b7 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_dro/main.yml @@ -1,24 +1,57 @@ --- -- name: "Read dro config file" +- name: Check if DRO config file exists + ansible.builtin.stat: + path: "{{ mas_config_dir }}/dro.yml" + register: dro_file_result + +- name: Read DRO config from file + vars: + dro_file_cfg: "{{ lookup('file', '{{ mas_config_dir }}/dro.yml') | from_yaml_all }}" ansible.builtin.set_fact: - drocfg: "{{ lookup('file', '{{ mas_config_dir }}/dro.yml') | from_yaml_all }}" + drocfg: + secret_name: "{{ dro_file_cfg[0].metadata.name }}" + registration_key: "{{ dro_file_cfg[0].stringData.api_key }}" + url: "{{ dro_file_cfg[1].spec.config.url }}" + ca: "{{ dro_file_cfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" + when: dro_file_result.stat.exists -- name: "Set dro mandatory vars" +- name: Read DRO config from environment ansible.builtin.set_fact: drocfg: - secret_name: "{{ drocfg[0].metadata.name }}" - registration_key: "{{ drocfg[0].stringData.api_key }}" - url: "{{ drocfg[1].spec.config.url }}" - ca: "{{ drocfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') | regex_replace('\\n\\s*\\n', '\\n') }}" + secret_name: "{{ lookup('env', 'MAS_AIBROKER_DRO_SECRET_NAME') }}" + registration_key: "{{ lookup('env', 'MAS_AIBROKER_DRO_API_KEY') }}" + url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') }}" + ca: "{{ lookup('env', 'MAS_AIBROKER_DRO_CA_CERT') }}" + when: not dro_file_result.stat.exists -- name: "Debug: dro information" #TODO: remove before PR - debug: +- name: "Debug: dro information" # TODO: remove before PR + ansible.builtin.debug: msg: - "dro secret name ......... {{ drocfg.secret_name }}" - "dro api key ............. {{ drocfg.registration_key }}" - "dro url ................. {{ drocfg.url }}" - "dro cert ca ............. {{ drocfg.ca }}" +- name: "Validate DRO configuration" + when: drocfg.secret_name | length == 0 + ansible.builtin.fail: + msg: "drocfg.secret_name must not empty" + +- name: "Validate DRO configuration" + when: drocfg.registration_key | length == 0 + ansible.builtin.fail: + msg: "drocfg.registration_key must not empty" + +- name: "Validate DRO configuration" + when: drocfg.url | length == 0 + ansible.builtin.fail: + msg: "drocfg.url must not empty" + +- name: "Validate DRO configuration" + when: drocfg.ca | length == 0 + ansible.builtin.fail: + msg: "drocfg.ca must not empty" + - name: "Create secret for Dro registration key" kubernetes.core.k8s: apply: yes diff --git a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml index f32195ca35..96672410b4 100644 --- a/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker/tasks/config_sls/main.yml @@ -1,25 +1,57 @@ --- -- name: "Read sls config file" +- name: Check if SLS config file exists + ansible.builtin.stat: + path: "{{ mas_config_dir }}/sls.yml" + register: sls_file_result + +- name: Read SLS config from file + vars: + sls_file_cfg: "{{ lookup('file', '{{ mas_config_dir }}/sls.yml') | from_yaml_all }}" ansible.builtin.set_fact: - slscfg: "{{ lookup('file', '{{ mas_config_dir }}/sls.yml') | from_yaml_all }}" + slscfg: + secret_name: "{{ sls_file_cfg[0].metadata.name }}" + registration_key: "{{ sls_file_cfg[0].stringData.registrationKey }}" + url: "{{ sls_file_cfg[1].spec.config.url }}" + ca: "{{ sls_file_cfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" + when: sls_file_result.stat.exists -- name: "Set sls mandatory vars" +- name: Read SLS config from environment ansible.builtin.set_fact: slscfg: - secret_name: "{{ slscfg[0].metadata.name }}" - registration_key: "{{ slscfg[0].stringData.registrationKey }}" - url: "{{ slscfg[1].spec.config.url }}" - ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') | join(',') | replace(',','\n') }}" - # ca: "{{ slscfg[1].spec.certificates | map(attribute='crt') }}" - -- name: "Debug: sls information" #TODO: remove before PR - debug: + secret_name: "{{ lookup('env', 'MAS_AIBROKER_SLS_SECRET_NAME') }}" + registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') }}" + url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') }}" + ca: "{{ lookup('env', 'MAS_AIBROKER_SLS_CA_CERT') }}" + when: not sls_file_result.stat.exists + +- name: "Debug: sls information" # TODO: remove before PR + ansible.builtin.debug: msg: - "sls secret name ......... {{ slscfg.secret_name }}" - "sls registration key .... {{ slscfg.registration_key }}" - "sls url ................. {{ slscfg.url }}" - "sls cert ca ............. {{ slscfg.ca }}" +- name: "Validate SLS configuration" + when: slscfg.secret_name | length == 0 + ansible.builtin.fail: + msg: "slscfg.secret_name must not be empty" + +- name: "Validate SLS configuration" + when: slscfg.registration_key | length == 0 + ansible.builtin.fail: + msg: "slscfg.registration_key must not be empty" + +- name: "Validate SLS configuration" + when: slscfg.url | length == 0 + ansible.builtin.fail: + msg: "slscfg.url must not be empty" + +- name: "Validate SLS configuration" + when: slscfg.ca | length == 0 + ansible.builtin.fail: + msg: "slscfg.ca must not be empty" + - name: "Create secret for SLS registration key" kubernetes.core.k8s: apply: yes From c7da559378f1a2b7e0352db28ea0bc4c905d2299 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 23 Apr 2025 15:59:25 +0100 Subject: [PATCH 145/169] [patch] update odh to 2.19.0 --- ibm/mas_devops/roles/odh/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/odh/vars/main.yml b/ibm/mas_devops/roles/odh/vars/main.yml index 015b701aea..6a3a815101 100644 --- a/ibm/mas_devops/roles/odh/vars/main.yml +++ b/ibm/mas_devops/roles/odh/vars/main.yml @@ -21,7 +21,7 @@ authorino_catalog_source: "{{ lookup('env', 'AUTHORINO_CATALOG_SOURCE') | defaul # ODH operator vars odh_channel: "{{ lookup('env', 'ODH_CHANNEL') | default('fast', true) }}" odh_catalog_source: "{{ lookup('env', 'ODH_CATALOG_SOURCE') | default('community-operators', true) }}" -odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.11.1', true) }}" +odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.19.0', true) }}" # Storage vars mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" From 658e9805ec266b932fd7196c58087a734416f015 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 23 Apr 2025 16:56:33 +0100 Subject: [PATCH 146/169] [patch] update docs with new dependencies --- docs/playbooks/oneclick-aibroker.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/playbooks/oneclick-aibroker.md b/docs/playbooks/oneclick-aibroker.md index 0f410f6fef..53d461dac1 100644 --- a/docs/playbooks/oneclick-aibroker.md +++ b/docs/playbooks/oneclick-aibroker.md @@ -84,7 +84,12 @@ AI Broker supports **AWS** and **Minio** storage providers. * `MAS_AIBROKER_APIKEY_ACTION` Whether to install or remove or update apikey (default value is: install) * `MAS_AIBROKER_WATSONX_ACTION` Whether to install or remove watsonx secret (default value is: install) * `MAS_AIBROKER_S3_ACTION` Whether to install or remove s3 (default value is: install) - +* `INSTALL_DB2` Whether to install DB2 (default value is: false) +* `INSTALL_MINIO` Whether to install minio (default value is: false) +* `INSTALL_MARIADB` Whether to install mariadb (default value is: false) +* `INSTALL_SLS` Whether to install IBM Suite License Service (default value is: false) +* `INSTALL_DRO` Whether to install IBM Data Reporter Operator (default value is: false) + ## Usage @@ -144,6 +149,12 @@ export MAS_AIBROKER_SAAS="true" export INSAASENV="True" export MAS_AIBROKER_SUBSCRIPTION_ID="" export MAS_AIBROKER_DRO_TENANT_ID="" +export INSTALL_DB2="" +export INSTALL_MINIO="" +export INSTALL_MARIADB="" +export INSTALL_MONGO="" +export INSTALL_SLS="" +export INSTALL_DRO="" oc create namespace openshift-serverless oc create namespace cert-manager-operator oc create namespace ibm-sls @@ -153,11 +164,12 @@ ansible-playbook playbooks/oneclick_add_aibroker.yml * `MAS_AIBROKER_SLS_REGISTRATION_KEY` - value can be found in `ibm-sls` namespace, in pod `sls-api-licensing-85699fb57-9lmrq` please look in environments tab, then value `REGISTRATION_KEY` * `MAS_AIBROKER_DRO_TOKEN` - go to `mas-instance_id-core` namespace and in secrets find `dro-apikey` -* in `AWS` for `MAS_AIBROKER_STORAGE_PIPELINES_BUCKET`, `MAS_AIBROKER_STORAGE_TENANTS_BUCKET`, `MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET` user need to create S3 buckets with unique name +* in `AWS` for `MAS_AIBROKER_STORAGE_PIPELINES_BUCKET`, `MAS_AIBROKER_STORAGE_TENANTS_BUCKET`, `MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET` user need to create S3 buckets with unique name ## NOTICE: playbook oneclick_add_aibroker.yml will run roles: ### Roles: * optional + - ibm.mas_devops.ibm_catalogs - ibm.mas_devops.cert_manager - ibm.mas_devops.mongodb From dbbfe032d558e58efe8a73a0acd1d7fe1517c156 Mon Sep 17 00:00:00 2001 From: Joe Harte Date: Fri, 25 Apr 2025 13:21:41 +0100 Subject: [PATCH 147/169] [minor] simplify AI Broker tenant setup (#1721) --- .../roles/aibroker_tenant/tasks/saas/main.yml | 12 -- .../tasks/tenant/install/main.yml | 166 ------------------ .../tenant/connector-configmap.yml.j2 | 7 - .../templates/tenant/km-s3-sa.yml.j2 | 8 - .../tenant/kserve-huggingfaceserver.yml.j2 | 33 ---- .../templates/tenant/kserve-lgbserver.yml.j2 | 33 ---- .../tenant/kserve-sklearnserver.yml.j2 | 32 ---- .../templates/tenant/kserve-xgbserver.yml.j2 | 33 ---- .../templates/tenant/rbac.yml.j2 | 104 ----------- 9 files changed, 428 deletions(-) delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/connector-configmap.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/km-s3-sa.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-huggingfaceserver.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-lgbserver.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-sklearnserver.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-xgbserver.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/rbac.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/saas/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/saas/main.yml index 49b80c6dc6..00b4a2dc12 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/saas/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/saas/main.yml @@ -11,15 +11,3 @@ script: "{{ role_path }}/files/create_apikey.sh {{ mas_aibroker_provision_tenant }} {{ aibroker_namespace }}" when: - apikey_secret_info.resources | length == 0 - -- name: "Lookup tenant {{ mas_aibroker_provision_tenant }} api key" - shell: oc get secret {{ mas_aibroker_provision_tenant }}----apikey-secret -n {{ aibroker_namespace }} -o json | jq -r '.data.AIBROKER_APIKEY' | base64 -d - register: provision_tenant_api_key_output - -- name: "Set aibroker api key variable" - set_fact: - provision_tenant_api_key: "{{ provision_tenant_api_key_output.stdout }}" - -- name: Debug {{ mas_aibroker_provision_tenant }} api key #TODO: remove on final commit - debug: - msg: "Provision tenant api key: {{ provision_tenant_api_key }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index fb32736d79..fcaa8e50e0 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -6,7 +6,6 @@ name: "{{ tenantNamespace }}" register: namespace_info -# Create tenant namespace - name: "Create a tenant namespace: {{ tenantNamespace }}" kubernetes.core.k8s: name: "{{ tenantNamespace }}" @@ -37,50 +36,12 @@ retries: 45 # approx 45 minutes before we give up delay: 60 # 1 minute -# Copy secrets to tenant namespace - name: "Copy secrets to namespace: {{ tenantNamespace }}" shell: 'oc get secret {{ item }} -n mas-{{ mas_instance_id }}-aibroker -o yaml | sed "s/namespace: .*/namespace: {{ tenantNamespace }}/" | oc apply --force -f -' with_items: - km-s3-secret - "{{ aibroker_pull_secret_name }}" -# Adding inference server runtimes -- name: "Create config map for connector config" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/connector-configmap.yml.j2') }}" - -- name: "Create SA for server runtimes" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/km-s3-sa.yml.j2') }}" - -- name: "Create huggingfaceserver server runtime" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-huggingfaceserver.yml.j2') }}" - -- name: "Create lgbserver server runtime" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-lgbserver.yml.j2') }}" - -- name: "Create sklearnserver server runtime" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-sklearnserver.yml.j2') }}" - -- name: "Create xgbserver server runtime" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/kserve-xgbserver.yml.j2') }}" - -# Applay RBAC roles -- name: "Apply RBAC for tenant" - kubernetes.core.k8s: - apply: yes - definition: "{{ lookup('template', 'templates/tenant/rbac.yml.j2') }}" - - name: "Check if SLS key secret exists in namespace: {{ aibroker_namespace }}" k8s_info: api_version: v1 @@ -113,10 +74,6 @@ script: "{{ role_path }}/files/create_sls_secret.sh {{ role_path }} {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" register: sls_secret_output - - name: Debug Create sls secret for AI Broker - debug: - msg: "{{ sls_secret_output }}" - - name: Delete DRO secret command: oc delete secret "{{ tenantNamespace }}----dro-secret" -n {{ aibroker_namespace }} register: delete_result @@ -126,126 +83,3 @@ - name: Create DRO token script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ drocfg.registration_key }}" register: dro_token_output - -- name: Debug Create DRO token - debug: - msg: "{{ dro_token_output }}" - -# - name: Wait for service /service/rest/api/v1/tenant #need more accurate way to check if service working -# pause: -# minutes: 8 -# prompt: "Waiting for for service /service/rest/api/v1/tenant" - -# - name: Await to aibroker-api pod created -# pause: -# minutes: 3 -# prompt: "Waiting for aibroker-api pod" - -# - name: "Wait for aibroker-api pod" -# shell: oc wait --for=condition=ready pod -l app=aibroker-api -n {{ aibroker_namespace }} -# register: _is_not_exists -# until: _is_not_exists.stdout|int == 0 -# retries: 60 -# delay: 30 - -# - name: "Wait for aibroker-api pod to be ready (30s delay)" -# shell: > -# oc get pod -l app=aibroker-api -n {{ aibroker_namespace }} --no-headers=true | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l -# register: _is_not_ready -# until: _is_not_ready.stdout|int == 0 -# retries: 60 -# delay: 30 - -- name: Lookup the aibroker route - kubernetes.core.k8s_info: - api_version: route.openshift.io/v1 - kind: Route - name: "{{ aibroker_name }}" - namespace: "{{ aibroker_namespace }}" - wait: yes - wait_timeout: 30 - register: aibrokerRoute - -- name: Set aibroker route host - ansible.builtin.set_fact: - aibrokerHostName: "{{ aibrokerRoute.resources[0].spec.host }}" - -- name: Debug aibrokerRoute - debug: - msg: "{{ aibrokerHostName }}" - -- name: Register DRO and SLS with AI Broker for tenant "{{ tenantNamespace }}" - uri: - url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenant" - validate_certs: no - method: POST - headers: - apikey: "{{ provision_tenant_api_key }}" - accept: "*/*" - Content-Type: "application/json" - tenantid: "{{ mas_aibroker_provision_tenant }}" - body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", - \"sls_url\": \"{{ aibrokerworkspace_cr_result.resources[0].spec.settings.sls.url }}\", - \"dro_url\": \"{{ aibrokerworkspace_cr_result.resources[0].spec.settings.dro.url }}\", - \"subscription_id\": \"{{ mas_aibroker_sls_subscription_id }}\", \"dro_tenant_id\": \"{{ mas_aibroker_dro_tenant_id }}\"}" - status_code: [204, 409] - timeout: 30 - register: register_tenant - -- name: Warn if tenant exists - when: register_tenant.status == 409 - debug: - msg: Warning - tenant {{ tenantNamespace }} is already registered with AI Broker - -- name: Register entitlement to Similarity model for tenant "{{ tenantNamespace }}" - uri: - url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" - validate_certs: no - method: POST - headers: - apikey: "{{ provision_tenant_api_key }}" - accept: "*/*" - Content-Type: "application/json" - tenantid: "{{ mas_aibroker_provision_tenant }}" - body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"similarity\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" - status_code: [201] - timeout: 30 - register: add_tenant_entitlements_to_db2 - -- name: Register entitlement to PCC model for tenant "{{ tenantNamespace }}" - uri: - url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" - validate_certs: no - method: POST - headers: - apikey: "{{ provision_tenant_api_key }}" - accept: "*/*" - Content-Type: "application/json" - tenantid: "{{ mas_aibroker_provision_tenant }}" - body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"pcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" - status_code: [201] - timeout: 30 - register: add_tenant_entitlements_to_db2_pcc - -- name: Register entitlement to MCC model for tenant "{{ tenantNamespace }}" - uri: - url: "https://{{ aibrokerHostName }}/ibm/aibroker/service/rest/api/v1/tenantEntitlements" - validate_certs: no - method: POST - headers: - apikey: "{{ provision_tenant_api_key }}" - accept: "*/*" - Content-Type: "application/json" - tenantid: "{{ mas_aibroker_provision_tenant }}" - body_format: "json" - body: "{\"tenant_name\": \"{{ tenantNamespace }}\", \"entitlement_type\": \"standard\", \"model_type\": \"mcc\", \"entitlement_start_date\": \"2025-01-01\", \"entitlement_end_date\": \"2026-01-01\"}" - status_code: [201] - timeout: 30 - register: add_tenant_entitlements_to_db2_mcc - -- name: Debug tenant entitlement #TODO: remove on final commit - debug: - msg: "{{ add_tenant_entitlements_to_db2_mcc }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/connector-configmap.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/connector-configmap.yml.j2 deleted file mode 100644 index a4fdee577d..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/connector-configmap.yml.j2 +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: connector-config - namespace: mas-{{ mas_instance_id }}-aibroker -data: {} diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/km-s3-sa.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/km-s3-sa.yml.j2 deleted file mode 100644 index db2d7a4edf..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/km-s3-sa.yml.j2 +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: km-s3-sa - namespace: "{{ tenantNamespace }}" -secrets: - - name: km-s3-secret diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-huggingfaceserver.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-huggingfaceserver.yml.j2 deleted file mode 100644 index 1ddc436b8e..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-huggingfaceserver.yml.j2 +++ /dev/null @@ -1,33 +0,0 @@ ---- -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - name: kserve-huggingfaceserver - namespace: "{{ tenantNamespace }}" -spec: - annotations: - prometheus.kserve.io/port: "8080" - prometheus.kserve.io/path: "/metrics" - supportedModelFormats: - - name: huggingface - version: "1" - autoSelect: true - priority: 1 - protocolVersions: - - v2 - - v1 - containers: - - name: kserve-container - image: kserve/huggingfaceserver:latest - args: - - '--model_name={{".Name"}}' - env: - - name: HUGGINGFACE_HUB_CACHE - value: /tmp/huggingface/hub - resources: - requests: - cpu: "1" - memory: 2Gi - limits: - cpu: "1" - memory: 2Gi diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-lgbserver.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-lgbserver.yml.j2 deleted file mode 100644 index a79769f1c5..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-lgbserver.yml.j2 +++ /dev/null @@ -1,33 +0,0 @@ ---- -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - name: kserve-lgbserver - namespace: "{{ tenantNamespace }}" -spec: - annotations: - prometheus.kserve.io/port: "8080" - prometheus.kserve.io/path: "/metrics" - supportedModelFormats: - - name: lightgbm - version: "3" - autoSelect: true - priority: 1 - protocolVersions: - - v1 - - v2 - containers: - - name: kserve-container - image: kserve/lgbserver:latest - args: - - '--model_name={{".Name"}}' - - --model_dir=/mnt/models - - --http_port=8080 - - --nthread=1 - resources: - requests: - cpu: "1" - memory: 2Gi - limits: - cpu: "1" - memory: 2Gi diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-sklearnserver.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-sklearnserver.yml.j2 deleted file mode 100644 index 2a05ee1fc8..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-sklearnserver.yml.j2 +++ /dev/null @@ -1,32 +0,0 @@ ---- -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - name: kserve-sklearnserver - namespace: "{{ tenantNamespace }}" -spec: - annotations: - prometheus.kserve.io/port: "8080" - prometheus.kserve.io/path: "/metrics" - supportedModelFormats: - - name: sklearn - version: "1" - autoSelect: true - priority: 1 - protocolVersions: - - v1 - - v2 - containers: - - name: kserve-container - image: kserve/sklearnserver:latest - args: - - '--model_name={{".Name"}}' - - --model_dir=/mnt/models - - --http_port=8080 - resources: - requests: - cpu: "1" - memory: 2Gi - limits: - cpu: "1" - memory: 2Gi diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-xgbserver.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-xgbserver.yml.j2 deleted file mode 100644 index daec36ac28..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/kserve-xgbserver.yml.j2 +++ /dev/null @@ -1,33 +0,0 @@ ---- -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - name: kserve-xgbserver - namespace: "{{ tenantNamespace }}" -spec: - annotations: - prometheus.kserve.io/port: "8080" - prometheus.kserve.io/path: "/metrics" - supportedModelFormats: - - name: xgboost - version: "1" - autoSelect: true - priority: 1 - protocolVersions: - - v1 - - v2 - containers: - - name: kserve-container - image: kserve/xgbserver:latest - args: - - '--model_name={{".Name"}}' - - --model_dir=/mnt/models - - --http_port=8080 - - --nthread=1 - resources: - requests: - cpu: "1" - memory: 2Gi - limits: - cpu: "1" - memory: 2Gi diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/rbac.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/rbac.yml.j2 deleted file mode 100644 index fa28085ab0..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/rbac.yml.j2 +++ /dev/null @@ -1,104 +0,0 @@ ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: km-controller-role - namespace: {{ tenantNamespace }} -rules: - - verbs: - - list - apiGroups: - - '' - resources: - - pods - - configmaps - - secrets - - verbs: - - delete - apiGroups: - - '' - resources: - - pods - - verbs: - - list - apiGroups: - - metrics.k8s.io - resources: - - nodes - - pods - - verbs: - - list - apiGroups: - - apps - resources: - - deployments - - verbs: - - list - - create - - delete - apiGroups: - - batch - resources: - - jobs - - cronjobs - - verbs: - - create - - list - - delete - - get - - patch - apiGroups: - - serving.kserve.io - resources: - - inferenceservices ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: km-controller-binding - namespace: {{ tenantNamespace }} -subjects: - - kind: ServiceAccount - name: km-controller - namespace: mas-{{ mas_instance_id }}-aibroker -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: km-controller-role ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: km-watcher-role - namespace: {{ tenantNamespace }} -rules: - - verbs: - - list - - watch - apiGroups: - - '' - resources: - - events - - pods - - verbs: - - list - - watch - apiGroups: - - batch - resources: - - cronjobs - - jobs ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: km-watcher-binding - namespace: {{ tenantNamespace }} -subjects: - - kind: ServiceAccount - name: km-watcher - namespace: mas-{{ mas_instance_id }}-aibroker -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: km-watcher-role From 4a2bd389767e53bf122ffa6805269a55d94d68ab Mon Sep 17 00:00:00 2001 From: Joe Harte Date: Fri, 25 Apr 2025 14:17:56 +0100 Subject: [PATCH 148/169] [patch] remove unused CR fields (#1722) --- .../aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 7b0e2165ac..6375de80af 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -14,8 +14,6 @@ metadata: {% endif %} spec: settings: - cluster_domain: "{{ cluster_domain }}" - in_saas_env: "{{ mas_aibroker_saas }}" icr: cp: "{{ mas_icr_cp }}" cpopen: "{{ mas_icr_cpopen }}" From 9958ec17c39c948bebd86f9fe9759de12e4bebaa Mon Sep 17 00:00:00 2001 From: Joe Harte Date: Mon, 28 Apr 2025 14:17:14 +0100 Subject: [PATCH 149/169] [major] AI Service entitlement in tenant CR (#1723) --- ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml | 3 +++ .../templates/aibroker/aibrokerworkspace.yml.j2 | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml index d4d4c672ea..97d5b897b4 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml @@ -19,6 +19,9 @@ mas_aibroker_path_ca_crt: './certs' mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" tenantNamespace: 'aibroker-{{ mas_aibroker_tenant_name }}' +tenant_entitlement_type: "{{ lookup('env', 'MAS_AIBROKER_TENANT_ENTITLEMENT_TYPE') }}" +tenant_entitlement_start_date: "{{ lookup('env', 'MAS_AIBROKER_TENANT_ENTITLEMENT_START_DATE') }}" +tenant_entitlement_end_date: "{{ lookup('env', 'MAS_AIBROKER_TENANT_ENTITLEMENT_END_DATE') }}" # API Key mas_aibroker_apikey_action: "{{ lookup('env', 'MAS_AIBROKER_APIKEY_ACTION') | default('install', true) }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 6375de80af..33f47bc527 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -33,3 +33,7 @@ spec: tenant_id: "{{ tenantNamespace }}" subscription_id: "{{ mas_aibroker_sls_subscription_id }}" instance_id: "{{ mas_instance_id }}" + entitlement: + type: {{ tenant_entitlement_type }} + startDate: {{ tenant_entitlement_start_date }} + endDate: {{ tenant_entitlement_end_date }} From 492f1b7f05120750529f0e0380aef93f5a40258e Mon Sep 17 00:00:00 2001 From: karol-czarnecki <112193189+karol-czarnecki@users.noreply.github.com> Date: Tue, 29 Apr 2025 16:05:39 +0100 Subject: [PATCH 150/169] [patch] Sync ansible-devops playbook Saas with June release (#1726) --- .../roles/aibroker/defaults/main.yml | 77 ++++++++++++++++++ ibm/mas_devops/roles/aibroker/vars/main.yml | 79 ------------------- .../roles/kmodels/defaults/main.yml | 32 ++++++++ .../kmodels-controller/configmap.yml.j2 | 4 + ibm/mas_devops/roles/kmodels/vars/main.yml | 32 -------- ibm/mas_devops/roles/odh/defaults/main.yml | 50 ++++++++++++ ibm/mas_devops/roles/odh/tasks/main.yml | 3 + .../roles/odh/tasks/odh-operator.yml | 17 ---- .../roles/odh/tasks/serverless-operator.yml | 18 ----- ibm/mas_devops/roles/odh/vars/main.yml | 50 ------------ 10 files changed, 166 insertions(+), 196 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/defaults/main.yml b/ibm/mas_devops/roles/aibroker/defaults/main.yml index fc4c738430..064e7e35b9 100644 --- a/ibm/mas_devops/roles/aibroker/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker/defaults/main.yml @@ -27,3 +27,80 @@ artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" # MAS Annotation block # ----------------------------------------------------------------------------- mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" + +# aibroker vars +mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" +aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" +pullSecretName: 'ibm-entitlement' + +# Whether to perform an install or remove tenant +tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" + +# Create Api Key vars +mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" +tenantNamespace: 'aibroker-{{ mas_aibroker_tenant_name }}' +# app_domain: "{{ lookup('env', 'APP_DOMAIN') }}" +mas_aibroker_apikey_action: "{{ lookup('env', 'MAS_AIBROKER_APIKEY_ACTION') | default('install', true) }}" + +# Main variables +ibm_entitlement_username: "{{ lookup('env','IBM_ENTITLEMENT_USERNAME') }}" + +# Source container registry +# ----------------------------------------------------------------------------- +mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" +mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" + +# WatsonX +mas_aibroker_watsonxai_apikey: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY') }}" +mas_aibroker_watsonxai_url: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') }}" +mas_aibroker_watsonxai_project_id: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') }}" +mas_aibroker_watsonx_action: "{{ lookup('env', 'MAS_AIBROKER_WATSONX_ACTION') | default('install', true) }}" +mas_aibroker_watsonx_secret: "{{ mas_app_id }}-watsonxcfg" + +# S3 +mas_aibroker_storage_host: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_HOST') | default('', true) }}" +mas_aibroker_storage_accesskey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') | default('', true) }}" +mas_aibroker_storage_secretkey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') | default('', true) }}" +mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | default('', true) }}" +mas_aibroker_s3_action: "{{ lookup('env', 'MAS_AIBROKER_S3_ACTION') | default('install', true) }}" +mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" +mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" +# Bucket names +mas_aibroker_storage_pipelines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') }}" +mas_aibroker_storage_tenants_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TENANTS_BUCKET') }}" +mas_aibroker_storage_templates_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET') }}" + + +# SAAS +mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" +mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" +mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" +mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" +mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" +mas_aibroker_path_ca_crt: './certs' +mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" + +# DRO +mas_aibroker_dro_token_secret: "dro-token" +mas_aibroker_dro_cacert_secret: "dro-certificates" + +# JDBC +mas_aibroker_jdbc_secret: "{{ mas_app_id }}-jdbccfg" + +# SLS +mas_aibroker_sls_registration_key_secret: "sls-registration-key" +mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" + +# Minio +mas_aibroker_minio_secret: "{{ mas_app_id }}-miniocfg" + +# MariaDB +mas_aibroker_mariadb_secret: "{{ mas_app_id }}-mariadbcfg" +mas_aibroker_db_host: "{{ lookup('env', 'MAS_AIBROKER_DB_HOST') | default('', true) }}" +mas_aibroker_db_port: "{{ lookup('env', 'MAS_AIBROKER_DB_PORT') | default('', true) }}" +mas_aibroker_db_user: "{{ lookup('env', 'MAS_AIBROKER_DB_USER') | default('', true) }}" +mas_aibroker_db_database: "{{ lookup('env', 'MAS_AIBROKER_DB_DATABASE') | default('', true) }}" +mas_aibroker_db_secret_name: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') | default('', true) }}" +mas_aibroker_db_secret_key: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_KEY') | default('password', true) }}" +mas_aibroker_db_secret_value: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') | default('', true) }}" diff --git a/ibm/mas_devops/roles/aibroker/vars/main.yml b/ibm/mas_devops/roles/aibroker/vars/main.yml index f60ce425b7..ed97d539c0 100644 --- a/ibm/mas_devops/roles/aibroker/vars/main.yml +++ b/ibm/mas_devops/roles/aibroker/vars/main.yml @@ -1,80 +1 @@ --- -# aibroker vars -aibroker_namespace: 'mas-{{ mas_instance_id }}-aibroker' -mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" -aibroker_channel: "{{ lookup('env', 'MAS_AIBROKER_CHANNEL') }}" -mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" -pullSecretName: 'ibm-entitlement' - -# Whether to perform an install or remove tenant -tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" - -# Create Api Key vars -mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" -tenantNamespace: 'aibroker-{{ mas_aibroker_tenant_name }}' -# app_domain: "{{ lookup('env', 'APP_DOMAIN') }}" -mas_aibroker_apikey_action: "{{ lookup('env', 'MAS_AIBROKER_APIKEY_ACTION') | default('install', true) }}" - -# Main variables -ibm_entitlement_username: "{{ lookup('env','IBM_ENTITLEMENT_USERNAME') }}" -ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" - -# Source container registry -# ----------------------------------------------------------------------------- -mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" -mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" - -# WatsonX -mas_aibroker_watsonxai_apikey: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY') }}" -mas_aibroker_watsonxai_url: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') }}" -mas_aibroker_watsonxai_project_id: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') }}" -mas_aibroker_watsonx_action: "{{ lookup('env', 'MAS_AIBROKER_WATSONX_ACTION') | default('install', true) }}" -mas_aibroker_watsonx_secret: "{{ mas_app_id }}-watsonxcfg" - -# S3 -mas_aibroker_storage_host: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_HOST') | default('', true) }}" -mas_aibroker_storage_accesskey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') | default('', true) }}" -mas_aibroker_storage_secretkey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') | default('', true) }}" -mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | default('', true) }}" -mas_aibroker_s3_action: "{{ lookup('env', 'MAS_AIBROKER_S3_ACTION') | default('install', true) }}" -mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" -mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" -# Bucket names -mas_aibroker_storage_pipelines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') }}" -mas_aibroker_storage_tenants_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TENANTS_BUCKET') }}" -mas_aibroker_storage_templates_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET') }}" - - -# SAAS -mas_aibroker_domain: "{{ lookup('env', 'MAS_AIBROKER_DOMAIN') }}" -mas_aibroker_sls_url: "{{ lookup('env', 'MAS_AIBROKER_SLS_URL') | default('', true) }}" -mas_aibroker_dro_url: "{{ lookup('env', 'MAS_AIBROKER_DRO_URL') | default('', true) }}" -mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" -mas_aibroker_sls_registration_key: "{{ lookup('env', 'MAS_AIBROKER_SLS_REGISTRATION_KEY') | default('', false) }}" -mas_aibroker_dro_token: "{{ lookup('env', 'MAS_AIBROKER_DRO_TOKEN') | default('', false) }}" -mas_aibroker_path_ca_crt: './certs' -mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" - -# DRO -mas_aibroker_dro_token_secret: "dro-token" -mas_aibroker_dro_cacert_secret: "dro-certificates" - -# JDBC -mas_aibroker_jdbc_secret: "{{ mas_app_id }}-jdbccfg" - -# SLS -mas_aibroker_sls_registration_key_secret: "sls-registration-key" -mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" - -# Minio -mas_aibroker_minio_secret: "{{ mas_app_id }}-miniocfg" - -# MariaDB -mas_aibroker_mariadb_secret: "{{ mas_app_id }}-mariadbcfg" -mas_aibroker_db_host: "{{ lookup('env', 'MAS_AIBROKER_DB_HOST') | default('', true) }}" -mas_aibroker_db_port: "{{ lookup('env', 'MAS_AIBROKER_DB_PORT') | default('', true) }}" -mas_aibroker_db_user: "{{ lookup('env', 'MAS_AIBROKER_DB_USER') | default('', true) }}" -mas_aibroker_db_database: "{{ lookup('env', 'MAS_AIBROKER_DB_DATABASE') | default('', true) }}" -mas_aibroker_db_secret_name: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') | default('', true) }}" -mas_aibroker_db_secret_key: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_KEY') | default('password', true) }}" -mas_aibroker_db_secret_value: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') | default('', true) }}" diff --git a/ibm/mas_devops/roles/kmodels/defaults/main.yml b/ibm/mas_devops/roles/kmodels/defaults/main.yml index c1c468d5b6..84b0f81b32 100644 --- a/ibm/mas_devops/roles/kmodels/defaults/main.yml +++ b/ibm/mas_devops/roles/kmodels/defaults/main.yml @@ -24,3 +24,35 @@ aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" # storage class primary_storage_class: "{{ lookup('env', 'PRIMARY_STORAGE_CLASS') }}" + +# vars file for kmodels +pullSecretName: "ibm-entitlement" +# controller +mas_aibroker_controller_tag: 1.0.8 +controller: "aibroker/controller:{{ mas_aibroker_controller_tag }}" +# store +mas_aibroker_store_tag: 1.0.5 +store: "aibroker/store:{{ mas_aibroker_store_tag }}" +# watcher +mas_aibroker_watcher_tag: 1.0.5 +watcher: "aibroker/watcher:{{ mas_aibroker_watcher_tag }}" + +mas_aibroker_connector_tag: 1.0.1 +mas_aibroker_pipeline_steps_tag: 1.0.2 + +# model +model_id_unique_length: "{{ lookup('env', 'MODEL_ID_UNIQUE_LENGTH') | default('8', true) }}" +model_id_prefix: "{{ lookup('env', 'MODEL_ID_PREFIX') | default('m', true) }}" + +# Bucket names +mas_aibroker_storage_pipelines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') }}" +mas_aibroker_storage_tenants_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TENANTS_BUCKET') }}" +mas_aibroker_storage_templates_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET') }}" + +# Kmodels tenant +mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" +tenantNamespace: "aibroker-{{ mas_aibroker_tenant_name }}" + +# Saas +mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" +mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 index 3a42e984ce..31531cea58 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 @@ -21,3 +21,7 @@ data: DEFAULT_TENANTS: {{ tenantNamespace }} {% endif %} PLATFORM: openshift + PVC_STORAGE_CLASS_NAME: "{{ primary_storage_class }}" + LOG_FORMAT: "json" + MODEL_ID_UNIQUE_LENGTH: "{{ model_id_unique_length }}" + MODEL_ID_PREFIX: "{{ model_id_prefix }}" diff --git a/ibm/mas_devops/roles/kmodels/vars/main.yml b/ibm/mas_devops/roles/kmodels/vars/main.yml index a710672834..ed97d539c0 100644 --- a/ibm/mas_devops/roles/kmodels/vars/main.yml +++ b/ibm/mas_devops/roles/kmodels/vars/main.yml @@ -1,33 +1 @@ --- -# vars file for kmodels -pullSecretName: "ibm-entitlement" -# controller -mas_aibroker_controller_tag: 1.0.7 -controller: "aibroker/controller:{{ mas_aibroker_controller_tag }}" -# store -mas_aibroker_store_tag: 1.0.3 -store: "aibroker/store:{{ mas_aibroker_store_tag }}" -# watcher -mas_aibroker_watcher_tag: 1.0.4 -watcher: "aibroker/watcher:{{ mas_aibroker_watcher_tag }}" - -mas_aibroker_connector_tag: 1.0.1 -mas_aibroker_pipeline_steps_tag: 1.0.1 - -# Bucket names -mas_aibroker_storage_pipelines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') }}" -mas_aibroker_storage_tenants_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TENANTS_BUCKET') }}" -mas_aibroker_storage_templates_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET') }}" - -# Source container registry -# ----------------------------------------------------------------------------- -mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}" -mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}" - -# Kmodels tenant -mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" -tenantNamespace: "aibroker-{{ mas_aibroker_tenant_name }}" - -# Saas -mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" -mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" diff --git a/ibm/mas_devops/roles/odh/defaults/main.yml b/ibm/mas_devops/roles/odh/defaults/main.yml index 3c81e38f34..c84ad1dfce 100644 --- a/ibm/mas_devops/roles/odh/defaults/main.yml +++ b/ibm/mas_devops/roles/odh/defaults/main.yml @@ -17,3 +17,53 @@ mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_enti # MAS Annotation block # ----------------------------------------------------------------------------- mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}" + +# main vars +mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" +aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" +pullSecretName: "ibm-entitlement" + +# Openshift Serverless vars +openshift_namespace: openshift-operators +serverless_namespace: openshift-serverless +serverless_catalog_source: "{{ lookup('env', 'SERVERLESS_CATALOG_SOURCE') | default('redhat-operators', true) }}" +serverless_channel: "{{ lookup('env', 'SERVERLESS_CHANNEL') | default('stable', true) }}" + +# Openshift Service Mesh vars +service_mesh_namespace: openshift-operators +service_mesh_channel: "{{ lookup('env', 'SERVICEMESH_CHANNEL') | default('stable', true) }}" +service_mesh_catalog_source: "{{ lookup('env', 'SERVICEMESH_CATALOG_SOURCE') | default('redhat-operators', true) }}" + +# Authorino operator vars +authorino_catalog_source: "{{ lookup('env', 'AUTHORINO_CATALOG_SOURCE') | default('community-operators', true) }}" + +# ODH operator vars +odh_channel: "{{ lookup('env', 'ODH_CHANNEL') | default('fast', true) }}" +odh_catalog_source: "{{ lookup('env', 'ODH_CATALOG_SOURCE') | default('community-operators', true) }}" +odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.19.0', true) }}" + +# Storage vars +mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" +mas_aibroker_storage_accesskey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') | default('', true) }}" +mas_aibroker_storage_secretkey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') | default('', true) }}" +mas_aibroker_storage_host: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_HOST') | default('', true) }}" +mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" +mas_aibroker_storage_ssl: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SSL') | default('true', true) }}" +mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | default('', true) }}" +mas_aibroker_storage_pipelines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') }}" + +# Pipeline operator +pipeline_channel: "{{ lookup('env', 'PIPELINE_CHANNEL') | default('latest', true) }}" +pipeline_catalog_source: "{{ lookup('env', 'PIPELINE_CATALOG_SOURCE') | default('redhat-operators', true) }}" + +# MariaDB +mas_aibroker_db_host: "{{ lookup('env', 'MAS_AIBROKER_DB_HOST') | default('', true) }}" +mas_aibroker_db_port: "{{ lookup('env', 'MAS_AIBROKER_DB_PORT') | default('', true) }}" +mas_aibroker_db_user: "{{ lookup('env', 'MAS_AIBROKER_DB_USER') | default('', true) }}" +mas_aibroker_db_database: "{{ lookup('env', 'MAS_AIBROKER_DB_DATABASE') | default('', true) }}" +mas_aibroker_db_secret_name: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') | default('', true) }}" +mas_aibroker_db_secret_key: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_KEY') | default('password', true) }}" +mas_aibroker_db_secret_value: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') | default('', true) }}" + +# DRO deps +mas_aibroker_dro_cacert: "{{ lookup('env', 'MAS_AIBROKER_DRO_CACERT') | default('', true) }}" diff --git a/ibm/mas_devops/roles/odh/tasks/main.yml b/ibm/mas_devops/roles/odh/tasks/main.yml index 0be38235c3..835d1595f5 100644 --- a/ibm/mas_devops/roles/odh/tasks/main.yml +++ b/ibm/mas_devops/roles/odh/tasks/main.yml @@ -122,3 +122,6 @@ namespace: "{{ aibroker_namespace }}" imagePullSecrets: - name: "{{ pullSecretName }}" + +- name: Patch config-features in knative-serving + shell: kubectl patch --namespace knative-serving configmap/config-features --type merge --patch '{"data":{"kubernetes.podspec-persistent-volume-claim":"enabled", "kubernetes.podspec-persistent-volume-write":"enabled"}}' diff --git a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml index 003dc5e8b3..11e3a93f79 100644 --- a/ibm/mas_devops/roles/odh/tasks/odh-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/odh-operator.yml @@ -1,21 +1,4 @@ --- -# Install Operator & create entitlement openshift-odh -# ----------------------------------------------------------------------------- -# - name: "Install Openshift odh Operator" -# ibm.mas_devops.apply_subscription: -# namespace: "{{ openshift_namespace }}" -# package_name: "opendatahub-operator" -# package_channel: "{{ odh_channel }}" -# catalog_source: "{{ odh_catalog_source }}" -# register: subscription - -# # Wait until the Opendata Hub CRD is available -# # ----------------------------------------------------------------------------- -# - name: "Wait until the Opendata Hub CRD is available" -# include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" -# vars: -# crd_name: knativeservings.operator.knative.dev - # enable following role temporary until find a way to use latest ODH versions and switch to apply_subscription role - name: "Install Openshift odh Operator specific version" kubernetes.core.k8s: diff --git a/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml b/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml index c048848367..e4230e2ac4 100644 --- a/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml +++ b/ibm/mas_devops/roles/odh/tasks/serverless-operator.yml @@ -1,22 +1,4 @@ --- -# # 1. Install Operator & create entitlement openshift-serverless -# # ----------------------------------------------------------------------------- -# - name: "Install Openshift Serverless Operator" -# ibm.mas_devops.apply_subscription: -# namespace: "{{ serverless_namespace }}" -# package_name: "serverless-operator" -# package_channel: "{{ serverless_channel }}" -# catalog_source: "{{ serverless_catalog_source }}" -# register: subscription - - -# # 2. Wait until the Serverless CRD is available -# # ----------------------------------------------------------------------------- -# - name: "Wait until the Serverless CRD is available" -# include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" -# vars: -# crd_name: knativeservings.operator.knative.dev - # enable following role temporary until find a way to use openshift-serverless and switch to apply_subscription role - name: "Install Openshift Serverless Operator" kubernetes.core.k8s: diff --git a/ibm/mas_devops/roles/odh/vars/main.yml b/ibm/mas_devops/roles/odh/vars/main.yml index 6a3a815101..ed97d539c0 100644 --- a/ibm/mas_devops/roles/odh/vars/main.yml +++ b/ibm/mas_devops/roles/odh/vars/main.yml @@ -1,51 +1 @@ --- -# main vars -mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" -aibroker_namespace: "mas-{{ mas_instance_id }}-aibroker" -pullSecretName: "ibm-entitlement" - -# Openshift Serverless vars -openshift_namespace: openshift-operators -serverless_namespace: openshift-serverless -serverless_catalog_source: "{{ lookup('env', 'SERVERLESS_CATALOG_SOURCE') | default('redhat-operators', true) }}" -serverless_channel: "{{ lookup('env', 'SERVERLESS_CHANNEL') | default('stable', true) }}" - -# Openshift Service Mesh vars -service_mesh_namespace: openshift-operators -service_mesh_channel: "{{ lookup('env', 'SERVICEMESH_CHANNEL') | default('stable', true) }}" -service_mesh_catalog_source: "{{ lookup('env', 'SERVICEMESH_CATALOG_SOURCE') | default('redhat-operators', true) }}" - -# Authorino operator vars -authorino_catalog_source: "{{ lookup('env', 'AUTHORINO_CATALOG_SOURCE') | default('community-operators', true) }}" - -# ODH operator vars -odh_channel: "{{ lookup('env', 'ODH_CHANNEL') | default('fast', true) }}" -odh_catalog_source: "{{ lookup('env', 'ODH_CATALOG_SOURCE') | default('community-operators', true) }}" -odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.19.0', true) }}" - -# Storage vars -mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" -mas_aibroker_storage_accesskey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_ACCESSKEY') | default('', true) }}" -mas_aibroker_storage_secretkey: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SECRETKEY') | default('', true) }}" -mas_aibroker_storage_host: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_HOST') | default('', true) }}" -mas_aibroker_storage_port: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PORT') | default('', true) }}" -mas_aibroker_storage_ssl: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SSL') | default('true', true) }}" -mas_aibroker_storage_region: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_REGION') | default('', true) }}" -#mas_aibroker_storage_piplines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPLINES_BUCKET') }}" -mas_aibroker_storage_pipelines_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PIPELINES_BUCKET') }}" -# mas_aibroker_storage_tenants_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TENANTS_BUCKET') }}" -# mas_aibroker_storage_templates_bucket: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_TEMPLATES_BUCKET') }}" - - -# Pipeline operator -pipeline_channel: "{{ lookup('env', 'PIPELINE_CHANNEL') | default('latest', true) }}" -pipeline_catalog_source: "{{ lookup('env', 'PIPELINE_CATALOG_SOURCE') | default('redhat-operators', true) }}" - -# MariaDB -mas_aibroker_db_host: "{{ lookup('env', 'MAS_AIBROKER_DB_HOST') | default('', true) }}" -mas_aibroker_db_port: "{{ lookup('env', 'MAS_AIBROKER_DB_PORT') | default('', true) }}" -mas_aibroker_db_user: "{{ lookup('env', 'MAS_AIBROKER_DB_USER') | default('', true) }}" -mas_aibroker_db_database: "{{ lookup('env', 'MAS_AIBROKER_DB_DATABASE') | default('', true) }}" -mas_aibroker_db_secret_name: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_NAME') | default('', true) }}" -mas_aibroker_db_secret_key: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_KEY') | default('password', true) }}" -mas_aibroker_db_secret_value: "{{ lookup('env', 'MAS_AIBROKER_DB_SECRET_VALUE') | default('', true) }}" From f365098405290d6c545d7b13621ad18e772b09ba Mon Sep 17 00:00:00 2001 From: roshi-dubey Date: Thu, 1 May 2025 19:51:03 +0530 Subject: [PATCH 151/169] [patch] update dro and db certs from CR (#1728) --- .../roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 index 6850f2a608..14b1fcec23 100644 --- a/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 +++ b/ibm/mas_devops/roles/aibroker/templates/aibroker/aibrokerapp.yml.j2 @@ -15,11 +15,12 @@ spec: dro: url: "{{ drocfg.url }}" dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" - ca: "{{ drocfg.ca }}" + ca: "{{ drocfg.ca | b64encode }}" jdbc: url: "{{ jdbccfg.url | b64decode }}" credentials_secret: "{{ mas_aibroker_jdbc_secret }}" - ca: "{{ jdbccfg.ca | b64decode }}" + ca: | + {{ jdbccfg.ca | indent(8) }} cluster_domain: "{{ cluster_domain }}" in_saas_env: "{{ mas_aibroker_saas }}" minio: @@ -33,4 +34,4 @@ spec: host: "{{ mas_aibroker_db_host }}" port: "{{ mas_aibroker_db_port }}" credentials_secret: "{{ mas_aibroker_mariadb_secret }}" - database_name: "{{ mas_aibroker_db_secret_name }}" + database_name: "{{ mas_aibroker_db_secret_name }}" \ No newline at end of file From 1dfe78b4c6200574e02ad57fd03a6bc62c2050c8 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Tue, 6 May 2025 09:26:37 +0100 Subject: [PATCH 152/169] [patch] update connector step and pipline steps versions --- ibm/mas_devops/roles/kmodels/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/kmodels/defaults/main.yml b/ibm/mas_devops/roles/kmodels/defaults/main.yml index 84b0f81b32..83b1cf269c 100644 --- a/ibm/mas_devops/roles/kmodels/defaults/main.yml +++ b/ibm/mas_devops/roles/kmodels/defaults/main.yml @@ -37,8 +37,8 @@ store: "aibroker/store:{{ mas_aibroker_store_tag }}" mas_aibroker_watcher_tag: 1.0.5 watcher: "aibroker/watcher:{{ mas_aibroker_watcher_tag }}" -mas_aibroker_connector_tag: 1.0.1 -mas_aibroker_pipeline_steps_tag: 1.0.2 +mas_aibroker_connector_tag: 1.0.2 +mas_aibroker_pipeline_steps_tag: 1.0.3 # model model_id_unique_length: "{{ lookup('env', 'MODEL_ID_UNIQUE_LENGTH') | default('8', true) }}" From b3e4502a4128037356bbbd5bd2e037858189add7 Mon Sep 17 00:00:00 2001 From: Joe Harte Date: Wed, 7 May 2025 09:25:29 +0100 Subject: [PATCH 153/169] [patch] fix DRO secret creation (#1734) --- .../roles/aibroker_tenant/defaults/main.yml | 2 +- .../aibroker_tenant/files/create_dro_token.sh | 40 ------------------- .../aibroker_tenant/tasks/config_dro/main.yml | 9 +++-- .../tasks/tenant/install/main.yml | 10 ----- .../aibroker/aibrokerworkspace.yml.j2 | 2 +- .../templates/dro/dro-secret.yml.j2 | 17 ++++++++ .../templates/dro/dro-token.yml.j2 | 9 ----- 7 files changed, 24 insertions(+), 65 deletions(-) delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/files/create_dro_token.sh create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-secret.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-token.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml index 97d5b897b4..d7a3c60e83 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml @@ -46,7 +46,7 @@ mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', tru mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" # DRO -mas_aibroker_dro_token_secret: "dro-token" +mas_aibroker_dro_token_secret: "{{ mas_aibroker_tenant_name }}----dro-secret" mas_aibroker_dro_tenant_id: "{{ lookup('env', 'MAS_AIBROKER_DRO_TENANT_ID') | default('001', true) }}" # SLS diff --git a/ibm/mas_devops/roles/aibroker_tenant/files/create_dro_token.sh b/ibm/mas_devops/roles/aibroker_tenant/files/create_dro_token.sh deleted file mode 100644 index 4850440c7c..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/files/create_dro_token.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -###### -# -# create DRO token in k8s secret -# -####### - -echo "Usage ./create_dro_token.sh tenant_name_in_lower_case instance_id token" - -NAMESPACE='aibroker' - -TENANT=$1 -instance_id=$2 -token=$3 - -if [[ -z ${instance_id} ]]; then - #echo "❌ Missing instance_id " - #exit 1 - echo "using default namespace aibroker" - NAMESPACE='aibroker' - -else - NAMESPACE=mas-${instance_id}-aibroker -fi - -echo $NAMESPACE - -echo $token - -if [ -z ${TENANT} ]; then - echo "using default tenant name=aibroker-user" - TENANT='aibroker-user' - #echo "Usage ./create_apikey.sh tenant_name_in_lower_case" - #exit 1 -fi - -echo "Creating DRO token in k8s secret" -oc create secret generic ${TENANT}----dro-secret -n ${NAMESPACE} \ - --from-literal=DRO_TOKEN=${token} diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml index c2f463a402..a065b813cb 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_dro/main.yml @@ -52,7 +52,8 @@ fail: msg: "drocfg.ca must not empty" -# - name: "Create secret for Dro registration key" -# kubernetes.core.k8s: -# apply: yes -# template: "templates/dro/dro-token.yml.j2" +- name: Create DRO secret + kubernetes.core.k8s: + state: present + namespace: "{{ aibroker_namespace }}" + template: "templates/dro/dro-secret.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index fcaa8e50e0..65eac50e12 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -73,13 +73,3 @@ - name: Create SLS secret script: "{{ role_path }}/files/create_sls_secret.sh {{ role_path }} {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" register: sls_secret_output - -- name: Delete DRO secret - command: oc delete secret "{{ tenantNamespace }}----dro-secret" -n {{ aibroker_namespace }} - register: delete_result - ignore_errors: yes - failed_when: delete_result.rc != 0 and "NotFound" not in delete_result.stderr - -- name: Create DRO token - script: "{{ role_path }}/files/create_dro_token.sh {{ tenantNamespace }} {{ mas_instance_id }} {{ drocfg.registration_key }}" - register: dro_token_output diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 33f47bc527..2dd91ca22c 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -30,7 +30,7 @@ spec: api_key: "{{ mas_aibroker_watsonxai_apikey }}" project_id: "{{ mas_aibroker_watsonxai_project_id }}" tenant: - tenant_id: "{{ tenantNamespace }}" + tenant_id: "{{ mas_aibroker_tenant_name }}" subscription_id: "{{ mas_aibroker_sls_subscription_id }}" instance_id: "{{ mas_instance_id }}" entitlement: diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-secret.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-secret.yml.j2 new file mode 100644 index 0000000000..49473690e3 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-secret.yml.j2 @@ -0,0 +1,17 @@ +--- +kind: Secret +apiVersion: v1 +type: Opaque +metadata: + name: {{ mas_aibroker_dro_token_secret }} + namespace: {{ aibroker_namespace }} + labels: + mas.ibm.com/applicationId: aibroker + mas.ibm.com/instanceId: "{{ mas_instance_id }}" +{% if custom_labels is defined and custom_labels.items() %} +{% for key, value in custom_labels.items() %} + "{{ key }}": "{{ value }}" +{% endfor %} +{% endif %} +data: + DRO_TOKEN: "{{ drocfg.registration_key | b64encode }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-token.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-token.yml.j2 deleted file mode 100644 index e6684c1714..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/dro/dro-token.yml.j2 +++ /dev/null @@ -1,9 +0,0 @@ ---- -kind: Secret -apiVersion: v1 -metadata: - name: {{ mas_aibroker_dro_token_secret }} - namespace: mas-{{ mas_instance_id }}-aibroker -data: - DRO_TOKEN: "{{ drocfg.registration_key | b64encode }}" -type: Opaque \ No newline at end of file From 76e3b2e36334e4d8f75e0eb262263c63ac33ffa2 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 7 May 2025 10:22:58 +0100 Subject: [PATCH 154/169] [patch] update default version of ODH to 2.28.0 --- ibm/mas_devops/roles/odh/defaults/main.yml | 3 ++- ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ibm/mas_devops/roles/odh/defaults/main.yml b/ibm/mas_devops/roles/odh/defaults/main.yml index c84ad1dfce..8ff5042e69 100644 --- a/ibm/mas_devops/roles/odh/defaults/main.yml +++ b/ibm/mas_devops/roles/odh/defaults/main.yml @@ -40,7 +40,8 @@ authorino_catalog_source: "{{ lookup('env', 'AUTHORINO_CATALOG_SOURCE') | defaul # ODH operator vars odh_channel: "{{ lookup('env', 'ODH_CHANNEL') | default('fast', true) }}" odh_catalog_source: "{{ lookup('env', 'ODH_CATALOG_SOURCE') | default('community-operators', true) }}" -odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.19.0', true) }}" +#odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.19.0', true) }}" +odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.28.0', true) }}" # Storage vars mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" diff --git a/ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 b/ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 index 867e73ae71..4d960b15b4 100644 --- a/ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 +++ b/ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 @@ -17,13 +17,13 @@ stringData: port: "{{ mas_aibroker_storage_port }}" ssl: "{{ mas_aibroker_storage_ssl }}" --- -apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1 +apiVersion: datasciencepipelinesapplications.opendatahub.io/v1 kind: DataSciencePipelinesApplication metadata: name: instance namespace: mas-{{ mas_instance_id }}-aibroker spec: - dspVersion: v1 + dspVersion: v2 database: disableHealthCheck: true customExtraParams: '{"tls":"false"}' From 79d837104a7e5b0642bbb9e6aba1df8a2ea72ee9 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 7 May 2025 10:24:37 +0100 Subject: [PATCH 155/169] [patch] update dspa CR for v2 version --- ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 b/ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 index f56357f782..95f0ceccc1 100644 --- a/ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 +++ b/ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 @@ -17,13 +17,13 @@ stringData: region: "{{ mas_aibroker_storage_region }}" ssl: "{{ mas_aibroker_storage_ssl }}" --- -apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1 +apiVersion: datasciencepipelinesapplications.opendatahub.io/v1 kind: DataSciencePipelinesApplication metadata: name: instance namespace: mas-{{ mas_instance_id }}-aibroker spec: - dspVersion: v1 + dspVersion: v2 database: disableHealthCheck: true customExtraParams: '{"tls":"false"}' From 1f36da3e1f3a84f8118335918a7b7fce6a997275 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 7 May 2025 14:35:57 +0100 Subject: [PATCH 156/169] [patch] added var mas_aibroker_storage_ssl as role kmodels need this --- ibm/mas_devops/roles/kmodels/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ibm/mas_devops/roles/kmodels/defaults/main.yml b/ibm/mas_devops/roles/kmodels/defaults/main.yml index 83b1cf269c..85ad2e6561 100644 --- a/ibm/mas_devops/roles/kmodels/defaults/main.yml +++ b/ibm/mas_devops/roles/kmodels/defaults/main.yml @@ -56,3 +56,6 @@ tenantNamespace: "aibroker-{{ mas_aibroker_tenant_name }}" # Saas mas_aibroker_saas: "{{ lookup('env', 'MAS_AIBROKER_SAAS') | default('false', true) | bool }}" mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT') | default('provision-tenant', true) }}" + +# S3 +mas_aibroker_storage_ssl: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_SSL') | default('true', true) }}" From 54f537fe6243074667ec6ada283a01411deed5fa Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 7 May 2025 14:38:32 +0100 Subject: [PATCH 157/169] [patch] update controller to 1.0.9 --- ibm/mas_devops/roles/kmodels/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/kmodels/defaults/main.yml b/ibm/mas_devops/roles/kmodels/defaults/main.yml index 85ad2e6561..bd4e9b00c0 100644 --- a/ibm/mas_devops/roles/kmodels/defaults/main.yml +++ b/ibm/mas_devops/roles/kmodels/defaults/main.yml @@ -28,7 +28,7 @@ primary_storage_class: "{{ lookup('env', 'PRIMARY_STORAGE_CLASS') }}" # vars file for kmodels pullSecretName: "ibm-entitlement" # controller -mas_aibroker_controller_tag: 1.0.8 +mas_aibroker_controller_tag: 1.0.9 controller: "aibroker/controller:{{ mas_aibroker_controller_tag }}" # store mas_aibroker_store_tag: 1.0.5 From 559da4974f2b3706c8ac9841f06768fa34b9d42c Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 7 May 2025 16:52:21 +0100 Subject: [PATCH 158/169] [patch] update controller config map for ODH 228 --- .../kmodels/templates/kmodels-controller/configmap.yml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 index 31531cea58..c2284c6d5c 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 @@ -7,7 +7,8 @@ metadata: labels: app: km-controller data: - DOCKER_SERVER: {{ mas_icr_cp }}/aibroker + CONTAINER_REGISTRY: {{ mas_icr_cp }}/aibroker + # DOCKER_SERVER: {{ mas_icr_cp }}/aibroker PIPELINES_BUCKET: "{{ mas_aibroker_storage_pipelines_bucket }}" TENANTS_BUCKET: "{{ mas_aibroker_storage_tenants_bucket }}" DOCKER_SECRETS_NAME: "ibm-entitlement" From 61708b3276721a5b4a82b8423fe62d9c764b1d81 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Wed, 7 May 2025 17:16:51 +0100 Subject: [PATCH 159/169] [patch] update config map for controller --- .../roles/kmodels/templates/kmodels-controller/configmap.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 index c2284c6d5c..a0971888e1 100644 --- a/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 +++ b/ibm/mas_devops/roles/kmodels/templates/kmodels-controller/configmap.yml.j2 @@ -26,3 +26,4 @@ data: LOG_FORMAT: "json" MODEL_ID_UNIQUE_LENGTH: "{{ model_id_unique_length }}" MODEL_ID_PREFIX: "{{ model_id_prefix }}" + AFFINITY_ENABLED: "false" From 8d0b6a657b79e54f579b77cc39b5d33b00190899 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Thu, 8 May 2025 09:27:03 +0100 Subject: [PATCH 160/169] [patch] add NP for predictor --- .../aibroker_tenant/tasks/tenant/install/main.yml | 5 +++++ .../templates/tenant/ingress-allow-isvc.yml.j2 | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/tenant/ingress-allow-isvc.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index 65eac50e12..f3fd2fd6a1 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -73,3 +73,8 @@ - name: Create SLS secret script: "{{ role_path }}/files/create_sls_secret.sh {{ role_path }} {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" register: sls_secret_output + +- name: "Create network policy for predictor" + kubernetes.core.k8s: + apply: yes + definition: "{{ lookup('template', 'templates/tenant/ingress-allow-isvc.yml.j2') }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/ingress-allow-isvc.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/ingress-allow-isvc.yml.j2 new file mode 100644 index 0000000000..b764d2a9dd --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/tenant/ingress-allow-isvc.yml.j2 @@ -0,0 +1,14 @@ +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-to-isvc + namespace: "{{ tenantNamespace }}" +spec: + podSelector: + matchLabels: + component: predictor + ingress: + - {} + policyTypes: + - Ingress From f1c205cc5eac7e89907ceed5bf426586f4cea063 Mon Sep 17 00:00:00 2001 From: Joe Harte Date: Thu, 8 May 2025 14:21:55 +0100 Subject: [PATCH 161/169] [patch] remove SLS registration (#1736) --- .../roles/aibroker_tenant/defaults/main.yml | 6 +- .../files/alm_sample_sls_use.py | 267 ------------------ .../files/create_sls_secret.sh | 50 ---- .../aibroker_tenant/tasks/config_sls/main.yml | 9 +- .../tasks/tenant/install/main.yml | 32 --- .../aibroker/aibrokerworkspace.yml.j2 | 2 +- .../templates/sls/sls-secret.yml.j2 | 17 ++ .../templates/sls/sls-tenant-details.yml.j2 | 9 - 8 files changed, 26 insertions(+), 366 deletions(-) delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-secret.yml.j2 delete mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-tenant-details.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml index d7a3c60e83..cc221a024f 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml @@ -16,9 +16,9 @@ aibroker_pull_secret_name: 'ibm-entitlement' mas_aibroker_path_ca_crt: './certs' # Tenant -mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('user', true) }}" +mas_aibroker_tenant_name: "{{ lookup('env', 'MAS_AIBROKER_TENANT_NAME') | default('aibroker-user', true) }}" tenant_action: "{{ lookup('env', 'TENANT_ACTION') | default('install', true) }}" -tenantNamespace: 'aibroker-{{ mas_aibroker_tenant_name }}' +tenantNamespace: '{{ mas_aibroker_tenant_name }}' tenant_entitlement_type: "{{ lookup('env', 'MAS_AIBROKER_TENANT_ENTITLEMENT_TYPE') }}" tenant_entitlement_start_date: "{{ lookup('env', 'MAS_AIBROKER_TENANT_ENTITLEMENT_START_DATE') }}" tenant_entitlement_end_date: "{{ lookup('env', 'MAS_AIBROKER_TENANT_ENTITLEMENT_END_DATE') }}" @@ -50,5 +50,5 @@ mas_aibroker_dro_token_secret: "{{ mas_aibroker_tenant_name }}----dro-secret" mas_aibroker_dro_tenant_id: "{{ lookup('env', 'MAS_AIBROKER_DRO_TENANT_ID') | default('001', true) }}" # SLS -mas_aibroker_sls_registration_key_secret: "sls-registration-key" +mas_aibroker_sls_secret: "{{ mas_aibroker_tenant_name }}----sls-secret" mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py b/ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py deleted file mode 100644 index 295d28ade7..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/files/alm_sample_sls_use.py +++ /dev/null @@ -1,267 +0,0 @@ -# ----------------------------------------------------------- -# Licensed Materials - Property of IBM -# 5737-M66, 5900-AAA -# (C) Copyright IBM Corp. 2024 All Rights Reserved. -# US Government Users Restricted Rights - Use, duplication, or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -# ----------------------------------------------------------- - -# This python script shows a simple model through which Sustainability offerings can interact in the SaaS pattern with a common ALM AppPoint pool. We'll -# see registration (provisioning), usage and deprovisioning. For clarity and simplicity, we do not perform validation of the certificate presented by the SLS service. Of course, you will want to -# do this in your offering. - -import os, base64, time, urllib3, requests, json, sys , secrets , string , random -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -slsUrl = sys.argv[1] -slsRegistrationKey = sys.argv[2] - -caCertificateForSSL = sys.argv[3] -# print('slsUrl='+slsUrl) -# print(slsRegistrationKey) -# print("caCertificateForSSL") -# print(caCertificateForSSL) -# -------------------------------------------------------------------------------------------------------------------------------- -# - -# - Configuration section - you need to complete steps 1-5 -# - -# -------------------------------------------------------------------------------------------------------------------------------- - - -def generate_api_key(): - alphabet = string.ascii_letters + string.digits - temp= ''.join(secrets.choice(alphabet) for i in range(12)) - return "88"+ temp.lower() - - -# 1: Provide a name for your offering - -offeringName = "aibroker" - -# 2: Provide am identifier for your instance. This should be pseudounique, so chances of collisions are low. For example, based on the cluster -# identity, or a pseudo unique UUID created at initial installation time - -instanceIdentifier = generate_api_key() #"xxxx" -#print("id") -print(instanceIdentifier) - -# 3: Provide the SLS instance URL (SRE will give you this) - -#slsUrl = "https://sls.ibm-sls.ibm-sls.apps.xxx.cp.fyre.ibm.com" - -# 4: Provide the SLS registration key (SRE will give you this) - -#slsRegistrationKey = "xxxx" - -# 5: Decide if you want the demonstration to tear down your client afterwards (as you would do in a deprovision). If not, the client -# will remain registered for you to experiment with. - -# set it to be False Kewei -#deprovision=True -deprovision=False - -# -------------------------------------------------------------------------------------------------------------------------------- -# - -# - Demo section: preparation -# - -# -------------------------------------------------------------------------------------------------------------------------------- - -# Prepare the SLS clientId to use, and prepare for storing client certificates that are generated at registration time. -slsClientId=offeringName+"-"+instanceIdentifier -certs_dir=f"{os.getcwd()}/certs" -# print("xxxxxxx") -# print(certs_dir) -clientCaCrtPath=certs_dir+"/"+slsClientId+"-ca.crt" -clientTlsCrtPath=certs_dir+"/"+slsClientId+"-tls.crt" -clientTlsKeyPath=certs_dir+"/"+slsClientId+"-tls.key" - -#print ("Using clientId %s"%slsClientId) -#print ("Client certificates for interacting with SLS will be stored in "+certs_dir+"/"+slsClientId) - -# -------------------------------------------------------------------------------------------------------------------------------- -# - -# - Demo section: registration (provision) or reuse -# - -# -------------------------------------------------------------------------------------------------------------------------------- - -# Let's check to see if client certificates already exist - if so, we can immediately move on to working with AppPoints. If not, -# we'll need to register ourselves with SLS to obtain the client certificates. - -if os.path.isfile(clientCaCrtPath): - # A client certificate appears to exist already. So let's use it. - #print ("Discovered client certificates for SLS client "+slsClientId+". Let's use them.") - pass -else: - #print ("This appears to be a new SLS client. Let's attempt to register and obtain some client certificates.") - pass - - # Make initial registration request - # print("will use certificate for client provisioning ") - data = {'clientId': slsClientId, 'description': 'ALM sample client registration', 'role': 'manage'} # TODO: Rawa - Swagger docs incorrectly state 'id' rather than 'clientId' - headers = {'X-Registration-Key': slsRegistrationKey} # TODO: Rawa - Swagger docs don't document the need for this critical header - - #print ("Making initial registration request for client "+slsClientId) - response = requests.post(slsUrl+"/api/registrations",verify=caCertificateForSSL,json=data, headers=headers ) - response.raise_for_status() - - registrationId = response.json()['registrationId'] - #print("Completing registration for registration request "+registrationId) - - # - # Poll registration status until it's complete - # - provisioningWaiting = True - while provisioningWaiting: - headers = {'X-Registration-Key': slsRegistrationKey} - response = requests.get(slsUrl+"/api/registrations/"+registrationId,verify=caCertificateForSSL,headers=headers ) - response.raise_for_status() - status = response.json()['state'] - # print("Waiting for client provisioning to be completed. Current status: %s" %status) - if status=="AWAITING_CONFIRMATION": - # We are now ready to proceed; client certs are available - # print ("Extracting client certificates") - # print(clientTlsCrtPath) - clientTlskey= base64.b64decode(response.json()['certs']['tls.key']) - clientTlscrt= base64.b64decode(response.json()['certs']['tls.crt']) - clientCacrt= base64.b64decode(response.json()['certs']['ca.crt']) - with open(f'{clientTlsCrtPath}', 'wb') as f: - f.write(clientTlscrt) - with open(f'{clientTlsKeyPath}', 'wb') as f: - f.write(clientTlskey) - with open(f'{clientCaCrtPath}', 'wb') as f: - f.write(clientCacrt) - provisioningWaiting = False - time.sleep(1) - - # The next step is to confirm the client. Poll status until complete. - confirmationWaiting = True - while confirmationWaiting: - response = requests.get(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL, cert=(clientTlsCrtPath,clientTlsKeyPath)) - #print (response) - status = response.json()['state'] - #print("Waiting for client registration to be confirmed. Current status: %s" %status) - if status=="REGISTERED": - # print ("Client registration confirmed.") - confirmationWaiting = False - time.sleep(1) - -# -------------------------------------------------------------------------------------------------------------------------------- -# - -# - Demo section: Define helper functions to perform SLS actions: mirror AppPoint usage to SLS and obtain AppPoint usage from SLS -# - -# -------------------------------------------------------------------------------------------------------------------------------- - -# Update 'quantity' AppPoints used in SLS. After 24 hours, the AppPoints will be automatically returned (ensuring no orphaned AppPoints if your service instance disappears) -def spendAppPoints(quantity): - #print ("Let's spend "+str(quantity)+" AppPoints") - data = {'validity': 86400, 'quantity': quantity} - response = requests.put(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,json=data,headers = {'Content-type': 'application/json','X-Product-Version': "1.0" },cert=(clientTlsCrtPath,clientTlsKeyPath)) - response.raise_for_status() - -# String representation of AppPoint usage by your client. TODO: SLS will shortly be providing an update to correctly report usage for your client. At present, it returns the overall usage across all clients. -def getUsage(): - response = requests.get(slsUrl+"/api/tokens/",verify=caCertificateForSSL,params={"owner":slsClientId},cert=(clientTlsCrtPath,clientTlsKeyPath)) # This will require an SLS update to return usage specific to your owner - response.raise_for_status() - j = json.loads(response.content.decode('utf-8')) - return j[0]['used'] - -# -------------------------------------------------------------------------------------------------------------------------------- -# - -# - Demo section: Interact with AppPoints -# - -# -------------------------------------------------------------------------------------------------------------------------------- - -# The ALM SaaS metering model is to perform the following actions every hour: -# - Locally calculate your AppPoint usage (based on your individual pricing model) -# - Check out the ValueMetric-Usage feature with (i) a licenseeId that is pseudo unique, (ii) an expiry of 86400 and (iii) a quantity matching the number of AppPoints -# -# Note that at each hour, you simply update your AppPoint usage: you do not need to explicitly return your previous hour's AppPoints -# -# As an example, let's suppose your AppPoints are charged on (i) users and (ii) IOPoints. -# 25 Users = 25 AppPoints -# 100 IOPoints = 10 AppPoints -# -# You can use one of two approaches to check out AppPoints. -# -# SIMPLE "single shot" -# -------------------- -# - Report AppPoint usage in one go. -# - This means a single, consistent licenseeId at the instance level - for example, licenseeId="eis-126745fvabv", i.e. reusing the SLS clientId is convenient. -# - Set the quantity to be 25+10 = 35 -# -# -# COMPLEX "breakdown by capability" -# --------------------------------- -# - Report AppPoint usage as a sequence of updates, per capability -# - Enables you to use SLS as a way to understand unequivocally breakdown of AppPoint usage by capability -# - Each capability has its own licenseeId - for example: -# - example: licenseeId="eis-126745fvabv-users",quantity=25 -# licenseeId="eis-126745fvabv-iopoints",quantity=10 -# -# In this code example, we'll do a single shot for simplicity and clarity - -# As described above, for simplicity in the single shot model, we set the licenseeId to the SLS clientId -licenseeId = slsClientId - -# Now let's define the license feature to check out. In SaaS ALM, this will be "MAS-ValueMetric-Usage" -productId = "MAS-ValueMetric-Usage" - -# -# Now let's start the day-to-day operation of the software - i.e., hourly updates of SLS of AppPoint usage, and hourly reporting of data (ready to send to DRO) -# - -#print ("Simulating a few hours' usage. Note that you will require an ALM-ready version of SLS, and an ALM-ready license, to report AppPoint usage correctly in shared environments.") - -# Hour 0: Check out 25+10=35 AppPoints -#print ("Hour 0") -#spendAppPoints(35) -# Get AppPoint usage from SLS (e.g., for reporting via DRO). -#print("Usage: "+str(getUsage())) - -# Hour 1: It's now 143 AppPoints overall -#print ("Hour 1") -#spendAppPoints(143) -#print("Usage: "+str(getUsage())) - -# Hour 2: It's now 133 AppPoints overall -#print ("Hour 2") -#spendAppPoints(133) -#print("Usage: "+str(getUsage())) - -# Hour 3: you get the picture :) -# -# ... - -# -------------------------------------------------------------------------------------------------------------------------------- -# - -# - Demo section: optional deprovision -# - -# -------------------------------------------------------------------------------------------------------------------------------- - -# Now suppose a request has been made to deprovision your service. How do we gracefully decouple from SLS? -if deprovision: - -# At deprovision, return your licenses and unregister the client - print ("Preparing for deprovision: returning any AppPoints to the pool") - response = requests.delete(slsUrl+"/api/products/"+productId+"/licensees/"+licenseeId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) - response.raise_for_status() - print("Usage after deprovisioning: "+str(getUsage())) - - print ("Preparing for deprovision: unregistering the SLS client") - response = requests.delete(slsUrl+"/api/clients/"+slsClientId,verify=caCertificateForSSL,cert=(clientTlsCrtPath,clientTlsKeyPath)) - response.raise_for_status() - #print ("Deleting redundant client certificates") - os.remove(clientTlsCrtPath) - os.remove(clientTlsKeyPath) - os.remove(clientCaCrtPath) - -else: - #print ("No deprovisioning will occur. Your client is still registered with SLS.") - pass - -# -------------------------------------------------------------------------------------------------------------------------------- -# - -# - Demo section: end -# - -# -------------------------------------------------------------------------------------------------------------------------------- -#print ("Demo over.") diff --git a/ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh b/ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh deleted file mode 100644 index e8adca7e44..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/files/create_sls_secret.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -e - -ROLE_PATH="${1}" -TENANT=$2 -instance_id=$3 - -if [ -z ${TENANT} ]; then - #echo "❌ Missing tenant name" - #exit 1 - echo "Usage ./create_sls_secret.sh tenant-name aibroker-instance-id-from-CustomResourceDefinitions sls-url-from-sls-routes slsRegistrationKey-from-sls-api-pod-Environment path_to_the_ca_crt" - echo "for example" - echo "./create_sls_secret.sh aibroker-user aibdev https://sls.ibm-sls.xxx.ibm.com xxxx-xxxx-xxxx-xxx" - exit 1 -fi - -cwd=$(pwd) -echo "location pwd in script create secret " - -mkdir -p certs - -echo "creating SLS registration, please wait....." -# instanceIdentifier=`python3 {{ role_path }}/files/alm_sample_sls_use.py $3 $4` -instanceIdentifier=$(python3 "${ROLE_PATH}/files/alm_sample_sls_use.py" $4 $5 $6) -# echo "SLS registration is created successfully." - -registrationKey=$5 - -# echo "----------" -# echo $instanceIdentifier -# echo "----------" -# Set variables -# Need to change -NAMESPACE=mas-${instance_id}-aibroker -slsClientId="aibroker"-$instanceIdentifier -# echo $slsClientId - -SECRET_NAME=${TENANT}----sls-secret -CA_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-ca.crt -TLS_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.crt -KEY_FILE_PATH=$cwd/certs/aibroker-${instanceIdentifier}-tls.key - -# Create the secret -oc create secret generic $SECRET_NAME -n $NAMESPACE \ - --from-file=$CA_FILE_PATH \ - --from-file=$TLS_FILE_PATH \ - --from-file=$KEY_FILE_PATH \ - --from-literal=SLS_CLIENT_ID=${slsClientId} \ - --from-literal=SLS_REGISTRATION_KEY=${registrationKey} diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml index d0205d0a9e..ef48f588c0 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/config_sls/main.yml @@ -52,7 +52,8 @@ fail: msg: "slscfg.ca must not empty" -# - name: "Create secret for SLS registration key" -# kubernetes.core.k8s: -# apply: yes -# template: "templates/sls/sls-tenant-details.yml.j2" +- name: Create SLS secret + kubernetes.core.k8s: + state: present + namespace: "{{ aibroker_namespace }}" + template: "templates/sls/sls-secret.yml.j2" diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml index f3fd2fd6a1..e661ad2618 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/tenant/install/main.yml @@ -42,38 +42,6 @@ - km-s3-secret - "{{ aibroker_pull_secret_name }}" -- name: "Check if SLS key secret exists in namespace: {{ aibroker_namespace }}" - k8s_info: - api_version: v1 - kind: Secret - name: "{{ tenantNamespace }}----sls-secret" - namespace: "{{ aibroker_namespace }}" - register: sls_secret_info - -# If sls secret does not exist then create one. -- when: - - sls_secret_info.resources | length == 0 - - mas_aibroker_saas == true - block: - - name: "Create cert location {{ mas_aibroker_path_ca_crt }}" - shell: mkdir -p '{{ mas_aibroker_path_ca_crt }}' - - - name: "Insert SLS cert ca to file located {{ mas_aibroker_path_ca_crt }}" - shell: "echo '{{ slscfg.ca }}' > '{{ mas_aibroker_path_ca_crt }}/sls_ca.crt' " - - - name: debug #TODO: remove before final commit - debug: - msg: - - "tenantNamespace ..... {{ tenantNamespace }} " - - "mas_instance_id ..... {{ mas_instance_id }} " - - "mas_aibroker_sls_url ..... {{ slscfg.url }} " - - "mas_aibroker_sls_registration_key ..... {{ slscfg.registration_key }} " - - "mas_aibroker_path_ca_crt ..... {{ mas_aibroker_path_ca_crt }} " - - - name: Create SLS secret - script: "{{ role_path }}/files/create_sls_secret.sh {{ role_path }} {{ tenantNamespace }} {{ mas_instance_id }} {{ slscfg.url }} {{ slscfg.registration_key }} {{ mas_aibroker_path_ca_crt }}/sls_ca.crt" - register: sls_secret_output - - name: "Create network policy for predictor" kubernetes.core.k8s: apply: yes diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 2dd91ca22c..cfef6e79af 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -23,7 +23,7 @@ spec: ca: "{{ drocfg.ca }}" sls: url: "{{ slscfg.url }}" - sls_registration_key_secret: "{{ slscfg.registration_key }}" + secretName: "{{ mas_aibroker_sls_secret }}" ca: "{{ slscfg.ca }}" watsonxai: url: "{{ mas_aibroker_watsonxai_url }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-secret.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-secret.yml.j2 new file mode 100644 index 0000000000..d35c789a69 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-secret.yml.j2 @@ -0,0 +1,17 @@ +--- +kind: Secret +apiVersion: v1 +type: Opaque +metadata: + name: {{ mas_aibroker_sls_secret }} + namespace: {{ aibroker_namespace }} + labels: + mas.ibm.com/applicationId: aibroker + mas.ibm.com/instanceId: "{{ mas_instance_id }}" +{% if custom_labels is defined and custom_labels.items() %} +{% for key, value in custom_labels.items() %} + "{{ key }}": "{{ value }}" +{% endfor %} +{% endif %} +data: + SLS_REGISTRATION_KEY: "{{ slscfg.registration_key | b64encode }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-tenant-details.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-tenant-details.yml.j2 deleted file mode 100644 index 35f6285706..0000000000 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/sls/sls-tenant-details.yml.j2 +++ /dev/null @@ -1,9 +0,0 @@ ---- -kind: Secret -apiVersion: v1 -metadata: - name: {{ mas_aibroker_sls_registration_key_secret }} - namespace: mas-{{ mas_instance_id }}-aibroker -data: - SLS_REGISTRATION_KEY: "{{ slscfg.registration_key | b64encode }}" -type: Opaque From 3670aa5a9238f0b96ec7455f93b68b9667a04ab3 Mon Sep 17 00:00:00 2001 From: Joe Harte Date: Fri, 9 May 2025 08:36:47 +0100 Subject: [PATCH 162/169] [patch] Update CR secret fields (#1738) --- ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml | 1 + .../templates/aibroker/aibrokerworkspace.yml.j2 | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml index cc221a024f..6bfad7acc9 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml @@ -27,6 +27,7 @@ tenant_entitlement_end_date: "{{ lookup('env', 'MAS_AIBROKER_TENANT_ENTITLEMENT_ mas_aibroker_apikey_action: "{{ lookup('env', 'MAS_AIBROKER_APIKEY_ACTION') | default('install', true) }}" # WatsonX +mas_aibroker_watsonxai_secret: "{{ mas_aibroker_tenant_name }}----wx-secret" mas_aibroker_watsonxai_apikey: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_APIKEY') }}" mas_aibroker_watsonxai_url: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_URL') }}" mas_aibroker_watsonxai_project_id: "{{ lookup('env', 'MAS_AIBROKER_WATSONXAI_PROJECT_ID') }}" diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index cfef6e79af..904125ef3c 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -19,7 +19,7 @@ spec: cpopen: "{{ mas_icr_cpopen }}" dro: url: "{{ drocfg.url }}" - dro_token_secret: "{{ mas_aibroker_dro_token_secret }}" + secretName: "{{ mas_aibroker_dro_token_secret }}" ca: "{{ drocfg.ca }}" sls: url: "{{ slscfg.url }}" @@ -27,7 +27,7 @@ spec: ca: "{{ slscfg.ca }}" watsonxai: url: "{{ mas_aibroker_watsonxai_url }}" - api_key: "{{ mas_aibroker_watsonxai_apikey }}" + secretName: "{{ mas_aibroker_watsonxai_secret }}" project_id: "{{ mas_aibroker_watsonxai_project_id }}" tenant: tenant_id: "{{ mas_aibroker_tenant_name }}" From ebaf495b6c1376428ab3e1561634422c1dd259eb Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 12 May 2025 10:20:15 +0100 Subject: [PATCH 163/169] [patch] restore odh 2.19.0 for dcut --- ibm/mas_devops/roles/odh/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/odh/defaults/main.yml b/ibm/mas_devops/roles/odh/defaults/main.yml index 8ff5042e69..7cb83889b8 100644 --- a/ibm/mas_devops/roles/odh/defaults/main.yml +++ b/ibm/mas_devops/roles/odh/defaults/main.yml @@ -40,8 +40,8 @@ authorino_catalog_source: "{{ lookup('env', 'AUTHORINO_CATALOG_SOURCE') | defaul # ODH operator vars odh_channel: "{{ lookup('env', 'ODH_CHANNEL') | default('fast', true) }}" odh_catalog_source: "{{ lookup('env', 'ODH_CATALOG_SOURCE') | default('community-operators', true) }}" -#odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.19.0', true) }}" -odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.28.0', true) }}" +odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.19.0', true) }}" +# odh_operator_version: "{{ lookup('env', 'ODH_OPERATOR_VERSION') | default('opendatahub-operator.v2.28.0', true) }}" # Storage vars mas_aibroker_storage_provider: "{{ lookup('env', 'MAS_AIBROKER_STORAGE_PROVIDER') | default('aws', true) }}" From 06a71ebb7c67ce9a85dbdbbbd8c17adec1d17a75 Mon Sep 17 00:00:00 2001 From: KAROL CZARNECKI Date: Mon, 12 May 2025 10:22:47 +0100 Subject: [PATCH 164/169] [patch] restore ODH 2.19.0 cr version --- ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 | 4 ++-- ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 b/ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 index 95f0ceccc1..f56357f782 100644 --- a/ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 +++ b/ibm/mas_devops/roles/odh/templates/odh/dspa-aws.yml.j2 @@ -17,13 +17,13 @@ stringData: region: "{{ mas_aibroker_storage_region }}" ssl: "{{ mas_aibroker_storage_ssl }}" --- -apiVersion: datasciencepipelinesapplications.opendatahub.io/v1 +apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1 kind: DataSciencePipelinesApplication metadata: name: instance namespace: mas-{{ mas_instance_id }}-aibroker spec: - dspVersion: v2 + dspVersion: v1 database: disableHealthCheck: true customExtraParams: '{"tls":"false"}' diff --git a/ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 b/ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 index 4d960b15b4..867e73ae71 100644 --- a/ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 +++ b/ibm/mas_devops/roles/odh/templates/odh/dspa-minio.yml.j2 @@ -17,13 +17,13 @@ stringData: port: "{{ mas_aibroker_storage_port }}" ssl: "{{ mas_aibroker_storage_ssl }}" --- -apiVersion: datasciencepipelinesapplications.opendatahub.io/v1 +apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1 kind: DataSciencePipelinesApplication metadata: name: instance namespace: mas-{{ mas_instance_id }}-aibroker spec: - dspVersion: v2 + dspVersion: v1 database: disableHealthCheck: true customExtraParams: '{"tls":"false"}' From 18d1b0e8bd1c6c04542681c1d0324f52b452d913 Mon Sep 17 00:00:00 2001 From: ravi makwana <62693371+sidd101010@users.noreply.github.com> Date: Mon, 12 May 2025 18:04:05 +0530 Subject: [PATCH 165/169] Added the rsl-secret --- ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml index 6bfad7acc9..22a2bc677b 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/defaults/main.yml @@ -50,6 +50,9 @@ mas_aibroker_provision_tenant: "{{ lookup('env', 'MAS_AIBROKER_PROVISION_TENANT' mas_aibroker_dro_token_secret: "{{ mas_aibroker_tenant_name }}----dro-secret" mas_aibroker_dro_tenant_id: "{{ lookup('env', 'MAS_AIBROKER_DRO_TENANT_ID') | default('001', true) }}" +# RSL +mas_aibroker_rsl_token_secret: "{{ mas_aibroker_tenant_name }}----rsl-secret" + # SLS mas_aibroker_sls_secret: "{{ mas_aibroker_tenant_name }}----sls-secret" mas_aibroker_sls_subscription_id: "{{ lookup('env', 'MAS_AIBROKER_SLS_SUBSCRIPTION_ID') | default('001', true) }}" From 2943b5275cbddc611e4a7fd842d6c208529a71b7 Mon Sep 17 00:00:00 2001 From: ravi makwana <62693371+sidd101010@users.noreply.github.com> Date: Mon, 12 May 2025 18:04:57 +0530 Subject: [PATCH 166/169] Read rsl config from environment --- .../roles/aibroker_tenant/tasks/rsl/main.yml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 ibm/mas_devops/roles/aibroker_tenant/tasks/rsl/main.yml diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/rsl/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/rsl/main.yml new file mode 100644 index 0000000000..9d7759104d --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/rsl/main.yml @@ -0,0 +1,49 @@ +- name: Read rsl config from environment + ansible.builtin.set_fact: + rslcfg: + authorization : "{{ lookup('env', 'RSL_AUTHORIZATION') }}" + url: "{{ lookup('env', 'RSL_URL') }}" + rsl_token: "{{ lookup('env', 'RSL_TOKEN') }}" + org_id : "{{ lookup('env', 'RSL_ORG_ID') }}" + secret_name: "{{ lookup('env', 'RSL_SECRET_NAME }}" + + +- name: "Debug: rsl information" #TODO: remove before PR + debug: + msg: + - "rsl secret name ......... {{ rslcfg.secret_name }}" + - "rsl authorization key ............. {{ rslcfg.authorization }}" + - "rsl url ................. {{ rslcfg.url }}" + - "rsl token ............. {{ rslcfg.rsl_token }}" + - "rsl org id .............. {{ rslcfg.org_id }}" + +- name: "Validate rsl configuration" + when: rslcfg.secret_name | length == 0 + fail: + msg: "rslcfg.secret_name must not empty" + +- name: "Validate rsl configuration" + when: rslcfg.authorization | length == 0 + fail: + msg: "rslcfg.authorization must not empty" + +- name: "Validate rsl configuration" + when: rslcfg.url | length == 0 + fail: + msg: "rslcfg.url must not empty" + +- name: "Validate rsl configuration" + when: rslcfg.rsl_token | length == 0 + fail: + msg: "rslcfg.rsl_token must not empty" + +- name: "Validate rsl configuration" + when: rslcfg.org_id | length == 0 + fail: + msg: "rslcfg.org_id must not empty" + +- name: Create rsl secret + kubernetes.core.k8s: + state: present + namespace: "{{ aibroker_namespace }}" + template: "templates/rsl/rsl-secret.yml.j2" From 0d1f1222d027cc0e63cb67d831aee9c2a6d42144 Mon Sep 17 00:00:00 2001 From: ravi makwana <62693371+sidd101010@users.noreply.github.com> Date: Mon, 12 May 2025 18:05:44 +0530 Subject: [PATCH 167/169] create the rsl-secret.yml.j2 --- .../templates/rsl/rsl-secret.yml.j2 | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 ibm/mas_devops/roles/aibroker_tenant/templates/rsl/rsl-secret.yml.j2 diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/rsl/rsl-secret.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/rsl/rsl-secret.yml.j2 new file mode 100644 index 0000000000..3d426ba725 --- /dev/null +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/rsl/rsl-secret.yml.j2 @@ -0,0 +1,17 @@ +--- +kind: Secret +apiVersion: v1 +type: Opaque +metadata: + name: {{ mas_aibroker_rsl_token_secret }} + namespace: {{ aibroker_namespace }} + labels: + mas.ibm.com/applicationId: aibroker + mas.ibm.com/instanceId: "{{ mas_instance_id }}" +{% if custom_labels is defined and custom_labels.items() %} +{% for key, value in custom_labels.items() %} + "{{ key }}": "{{ value }}" +{% endfor %} +{% endif %} +data: + RSL_TOKEN: "{{ rslcfg.registration_key | b64encode }}" From e3b359533932a51b8b5fb8c05ea077ffc9ef2cb5 Mon Sep 17 00:00:00 2001 From: ravi makwana <62693371+sidd101010@users.noreply.github.com> Date: Mon, 12 May 2025 18:12:44 +0530 Subject: [PATCH 168/169] Added the few var --- .../templates/aibroker/aibrokerworkspace.yml.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 904125ef3c..2c272d74a7 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -21,6 +21,14 @@ spec: url: "{{ drocfg.url }}" secretName: "{{ mas_aibroker_dro_token_secret }}" ca: "{{ drocfg.ca }}" + + rsl: + url: "{{ rslcfg.url }}" + secretName: "{{ rslcfg.secret_name }}" + authorization : "{{ rslcfg.authorization }}" + token : "{{ rslcfg.rsl_token }}" + orgId: "{{ rslcfg.org_id }}" + sls: url: "{{ slscfg.url }}" secretName: "{{ mas_aibroker_sls_secret }}" From 2102ab97971e52c5eaf40630299a9275cf9c01af Mon Sep 17 00:00:00 2001 From: ravi makwana <62693371+sidd101010@users.noreply.github.com> Date: Mon, 12 May 2025 18:55:20 +0530 Subject: [PATCH 169/169] Remove the unwanted vars --- .../roles/aibroker_tenant/tasks/rsl/main.yml | 13 ------------- .../templates/aibroker/aibrokerworkspace.yml.j2 | 2 -- 2 files changed, 15 deletions(-) diff --git a/ibm/mas_devops/roles/aibroker_tenant/tasks/rsl/main.yml b/ibm/mas_devops/roles/aibroker_tenant/tasks/rsl/main.yml index 9d7759104d..4d51e09bcb 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/tasks/rsl/main.yml +++ b/ibm/mas_devops/roles/aibroker_tenant/tasks/rsl/main.yml @@ -1,31 +1,18 @@ - name: Read rsl config from environment ansible.builtin.set_fact: rslcfg: - authorization : "{{ lookup('env', 'RSL_AUTHORIZATION') }}" url: "{{ lookup('env', 'RSL_URL') }}" rsl_token: "{{ lookup('env', 'RSL_TOKEN') }}" org_id : "{{ lookup('env', 'RSL_ORG_ID') }}" - secret_name: "{{ lookup('env', 'RSL_SECRET_NAME }}" - name: "Debug: rsl information" #TODO: remove before PR debug: msg: - - "rsl secret name ......... {{ rslcfg.secret_name }}" - - "rsl authorization key ............. {{ rslcfg.authorization }}" - "rsl url ................. {{ rslcfg.url }}" - "rsl token ............. {{ rslcfg.rsl_token }}" - "rsl org id .............. {{ rslcfg.org_id }}" -- name: "Validate rsl configuration" - when: rslcfg.secret_name | length == 0 - fail: - msg: "rslcfg.secret_name must not empty" - -- name: "Validate rsl configuration" - when: rslcfg.authorization | length == 0 - fail: - msg: "rslcfg.authorization must not empty" - name: "Validate rsl configuration" when: rslcfg.url | length == 0 diff --git a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 index 2c272d74a7..81b844a089 100644 --- a/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 +++ b/ibm/mas_devops/roles/aibroker_tenant/templates/aibroker/aibrokerworkspace.yml.j2 @@ -24,8 +24,6 @@ spec: rsl: url: "{{ rslcfg.url }}" - secretName: "{{ rslcfg.secret_name }}" - authorization : "{{ rslcfg.authorization }}" token : "{{ rslcfg.rsl_token }}" orgId: "{{ rslcfg.org_id }}"