diff --git a/ibm/mas_devops/plugins/filter/filters.py b/ibm/mas_devops/plugins/filter/filters.py index e2b7769d28..d68e7d9253 100644 --- a/ibm/mas_devops/plugins/filter/filters.py +++ b/ibm/mas_devops/plugins/filter/filters.py @@ -399,6 +399,42 @@ def get_db2_instance_name(binding_scope, mas_instance_id, mas_workspace_id, mas_ } return jdbc_instance_names[binding_scope] +def get_ecr_repositories(image_mirror_output): + """ + filter: get_ecr_repositories + author: Terence Quinn + version_added: 0.1 + short_description: Get the names of ECR repositories relevant to a mirror command + description: + - Parse the output of oc image mirror --dry-run=true command for repository names. The + list of repository names will be used to create all the required repositories in ECR + before actually performing the mirror. This is needed for AWS ECR only. + options: + image_mirror_output: + description: Output of the oc image mirror --dry-run=true command + required: True + notes: + - limited error handling, will not handle unexpected data currently + """ + # marker to indicate summary of images from oc image mirror command + phase_found = False + repositories = [] + for line in image_mirror_output: + line = line.strip() + # once we find a line starting with phase - start parsing repositories names + if line.startswith('phase '): + phase_found = True + continue + if line == "": + phase_found = False + continue + if phase_found: + # parse out the repository name + if len(line.split()) > 1: + repo_to_add = line.split()[1] + repositories.append(repo_to_add) + return repositories + class FilterModule(object): def filters(self): @@ -420,5 +456,6 @@ def filters(self): 'format_pre_version_with_plus': format_pre_version_with_plus, 'format_pre_version_without_buildid': format_pre_version_without_buildid, 'format_pre_version_with_buildid': format_pre_version_with_buildid, - 'get_db2_instance_name': get_db2_instance_name + 'get_db2_instance_name': get_db2_instance_name, + 'get_ecr_repositories': get_ecr_repositories } diff --git a/ibm/mas_devops/roles/mirror_images/defaults/main.yml b/ibm/mas_devops/roles/mirror_images/defaults/main.yml index 7312b6b3ee..d1e68d743f 100644 --- a/ibm/mas_devops/roles/mirror_images/defaults/main.yml +++ b/ibm/mas_devops/roles/mirror_images/defaults/main.yml @@ -6,6 +6,11 @@ registry_public_url: "{{ registry_public_host }}{% if registry_public_port != '' registry_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') | default('', true) }}" registry_public_url_with_path: "{{ registry_public_url }}{% if registry_prefix | length > 0 %}/{{ registry_prefix }}{% endif %}" +# AWS Elastic Container Registry +# ----------------------------------------------------------------------------- +registry_is_ecr: "{{ lookup('env', 'REGISTRY_IS_ECR') | default(false, true) | bool }}" +registry_ecr_aws_region: "{{ lookup('env', 'REGISTRY_ECR_AWS_REGION') | default('', true) }}" + # Development config # ----------------------------------------------------------------------------- artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" @@ -30,3 +35,4 @@ manifest_version: "{{ lookup('env', 'MANIFEST_VERSION') | replace('_', '.') }}" auth_file: "{{ ansible_env.HOME }}/.ibm-mas/auth.json" mirror_single_arch: "{{ lookup('env', 'MIRROR_SINGLE_ARCH') }}" + diff --git a/ibm/mas_devops/roles/mirror_images/tasks/main.yml b/ibm/mas_devops/roles/mirror_images/tasks/main.yml index 1936866a4d..4bf24d4e13 100644 --- a/ibm/mas_devops/roles/mirror_images/tasks/main.yml +++ b/ibm/mas_devops/roles/mirror_images/tasks/main.yml @@ -109,7 +109,65 @@ that: ocimagemirror_result.rc == 0 fail_msg: "Image mirroring failed (see debug information above)." -# 4B. Execute the mirror (from filesystem) +# 4B. Create AWS ECR Repositories if required +# ----------------------------------------------------------------------------- +- name: "Create AWS ECR Repositories" + when: + - mirror_mode == "direct" or mirror_mode == "from-filesystem" + - registry_is_ecr == true + block: + - name: "{{ manifest_name }} : dry mode mirror command - direct" + shell: | + oc image mirror \ + -a {{ auth_file }} \ + -f {{ mirror_working_dir }}/manifests/direct/{{ manifest_name }}_{{ manifest_version }}.txt \ + --filter-by-os '.*' \ + --insecure \ + --skip-multiple-scopes \ + --max-per-registry=1 \ + --dry-run=true + register: ocimagemirror_dryrun_result + ignore_errors: true + when: mirror_mode == "direct" + + - name: "{{ manifest_name }} : dry mode mirror command - from-filesystem" + shell: | + oc image mirror \ + -a {{ auth_file }} \ + -f {{ mirror_working_dir }}/manifests/from-filesystem/{{ manifest_name }}_{{ manifest_version }}.txt \ + --filter-by-os '.*' \ + --insecure \ + --skip-multiple-scopes \ + --max-per-registry=1 \ + --dir={{ mirror_working_dir }} \ + --dry-run=true + register: ocimagemirror_dryrun_result + ignore_errors: true + when: mirror_mode == "from-filesystem" + + - name: "{{ manifest_name }} : Debug dry mode mirror command" + debug: + msg: "{{ (ocimagemirror_dryrun_result.rc == 0) | ternary(ocimagemirror_dryrun_result.stdout_lines, ocimagemirror_dryrun_result.stderr_lines) }}" + + - name: "{{ manifest_name }} : Fail if dry mode mirror is not successful" + assert: + that: ocimagemirror_dryrun_result.rc == 0 + fail_msg: "mirror command in dry mode failed." + + - name: Filter out repository names that must be created in ECR + set_fact: + repos_to_add: "{{ ocimagemirror_dryrun_result.stderr_lines | ibm.mas_devops.get_ecr_repositories }}" + + - name: Repositories that will be created in ECR + debug: + msg: "{{ repos_to_add }}" + + - name: Create ECR Repositories + ansible.builtin.shell: "aws ecr create-repository --repository-name {{ item }} --region {{ registry_ecr_aws_region }}" + ignore_errors: true + loop: "{{ repos_to_add }}" + +# 4C. Execute the mirror (from filesystem) # ----------------------------------------------------------------------------- - name: "{{ manifest_name }} : Mirror Images from local filesystem" when: mirror_mode == "from-filesystem" @@ -139,7 +197,7 @@ that: ocimagemirror_result.rc == 0 fail_msg: "Image mirroring failed (see debug information above)." -# 4C. Execute the mirror (direct) +# 4D. Execute the mirror (direct) # ----------------------------------------------------------------------------- - name: "Mirror Images directly" when: mirror_mode == "direct" diff --git a/ibm/mas_devops/roles/mirror_ocp/README.md b/ibm/mas_devops/roles/mirror_ocp/README.md index 5caf3273f6..c085c2633a 100644 --- a/ibm/mas_devops/roles/mirror_ocp/README.md +++ b/ibm/mas_devops/roles/mirror_ocp/README.md @@ -118,6 +118,20 @@ The public port number for the target registry - Environment Variable: `REGISTRY_PUBLIC_PORT` - Default: None +### registry_is_ecr +Specifies if the target registry is an AWS Elastic Container Registry instance + +- Optional +- Environment Variable: `REGISTRY_IS_ECR` +- Default: false + +### registry_ecr_aws_region +The AWS region of the AWS Elastic Container Registry. Only required if the target registry is an AWS Elastic Container Registry instance. + +- Optional +- Environment Variable: `REGISTRY_ECR_AWS_REGION` +- Default: None + ### registry_prefix_redhat The prefix used for the target registry. The images will not be mirrored to the registry at this time but will define the final destination in the form: `{host}[:{port}]/{prefix}/{reponame}` diff --git a/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml b/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml index f28ef5a862..6154393bbf 100644 --- a/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml +++ b/ibm/mas_devops/roles/mirror_ocp/defaults/main.yml @@ -20,6 +20,11 @@ registry_username: "{{ lookup('env', 'REGISTRY_USERNAME') }}" registry_password: "{{ lookup('env', 'REGISTRY_PASSWORD') }}" registry_auth: "{{ registry_username }}:{{ registry_password }}" +# AWS Elastic Container Registry +# ----------------------------------------------------------------------------- +registry_is_ecr: "{{ lookup('env', 'REGISTRY_IS_ECR') | default(false, true) | bool }}" +registry_ecr_aws_region: "{{ lookup('env', 'REGISTRY_ECR_AWS_REGION') | default('', true) }}" + # Release Config # ----------------------------------------------------------------------------- ocp_release: "{{ lookup('env', 'OCP_RELEASE') }}" diff --git a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml index 4daab84c6e..556be9ecf2 100644 --- a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml +++ b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml @@ -9,7 +9,7 @@ - mirror_working_dir is defined and mirror_working_dir != "" fail_msg: "One or more required properties are missing" -# 2. Perform Mirroring +# 2a. Perform Mirroring # ----------------------------------------------------------------------------- - name: "Generate ImageSetConfiguration" ansible.builtin.template: @@ -17,6 +17,46 @@ dest: "{{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml" mode: "664" + +# 2b. Create AWS ECR Repositories if required +# ----------------------------------------------------------------------------- +- name: "Create AWS ECR Repositories" + when: + - registry_is_ecr == true + block: + - name: Create validation repository required for oc mirror command + ansible.builtin.shell: "aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/oc-mirror --region {{ registry_ecr_aws_region }}" + ignore_errors: true + + - name: "{{ ocp_release }} : dry mode mirror command" + shell: > + DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dry-run --dest-skip-tls --config={{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml docker://{{ registry_public_url_with_path_redhat }} + register: ocimagemirror_dryrun_result + ignore_errors: true + + - name: "{{ ocp_release }} : Debug mirror command dry mode" + debug: + msg: + - "{{ ocimagemirror_dryrun_result.stderr_lines }} " + + - name: "{{ ocp_release }} : Fail if dry mode mirror is not successful" + assert: + that: ocimagemirror_dryrun_result.rc == 0 + fail_msg: "mirror command in dry mode failed." + + - name: Filter out repository names that must be created in ECR + set_fact: + repos_to_add: "{{ ocimagemirror_dryrun_result.stderr_lines | ibm.mas_devops.get_ecr_repositories }}" + + - name: Repositories that will be created in ECR + debug: + msg: "{{ repos_to_add }}" + + - name: Create ECR Repositories + ansible.builtin.shell: "aws ecr create-repository --repository-name {{ item }} --region {{ registry_ecr_aws_region }}" + ignore_errors: true + loop: "{{ repos_to_add }}" + - name: "Debug Information" debug: msg: diff --git a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml index b44b5fa2a4..afe02f2da3 100644 --- a/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml +++ b/ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml @@ -1,5 +1,5 @@ --- -# 1. Check for undefined properties that do not have a default +# 1a. Check for undefined properties that do not have a default # ----------------------------------------------------------------------------- - name: "Fail if required properties are not provided" assert: @@ -8,6 +8,49 @@ - mirror_working_dir is defined and mirror_working_dir != "" fail_msg: "One or more required properties are missing" +# 1b. Create AWS ECR Repositories if required +# ----------------------------------------------------------------------------- +- name: "Create AWS ECR Repositories" + when: + - registry_is_ecr == true + block: + - name: Create validation repository required for oc mirror command + ansible.builtin.shell: | + aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/oc-mirror --region {{ registry_ecr_aws_region }} + aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/redhat/certified-operator-index --region {{ registry_ecr_aws_region }} + aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/redhat/community-operator-index --region {{ registry_ecr_aws_region }} + aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/redhat/redhat-operator-index --region {{ registry_ecr_aws_region }} + ignore_errors: true + + - name: "{{ ocp_release }} : dry mode mirror command" + shell: > + DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dry-run --dest-skip-tls --max-per-registry=1 --from={{ mirror_working_dir }}/mirror_seq1_000000.tar docker://{{ registry_public_url_with_path_redhat }} + register: ocimagemirror_dryrun_result + ignore_errors: true + + - name: "{{ ocp_release }} : Debug mirror command dry mode" + debug: + msg: + - "{{ ocimagemirror_dryrun_result.stderr_lines }} " + + - name: "{{ ocp_release }} : Fail if dry mode mirror is not successful" + assert: + that: ocimagemirror_dryrun_result.rc == 0 + fail_msg: "mirror command in dry mode failed." + + - name: Filter out repository names that must be created in ECR + set_fact: + repos_to_add: "{{ ocimagemirror_dryrun_result.stderr_lines | ibm.mas_devops.get_ecr_repositories }}" + + - name: Repositories that will be created in ECR + debug: + msg: "{{ repos_to_add }}" + + - name: Create ECR Repositories + ansible.builtin.shell: "aws ecr create-repository --repository-name {{ item }} --region {{ registry_ecr_aws_region }}" + ignore_errors: true + loop: "{{ repos_to_add }}" + # 2. Perform Mirroring # ----------------------------------------------------------------------------- - name: "Debug Information"