[minor] Add support for image mirroring to AWS ACR

ibm/mas_devops/plugins/filter/filters.py

@@ -399,6 +399,42 @@ def get_db2_instance_name(binding_scope, mas_instance_id, mas_workspace_id, mas_
   }
   return jdbc_instance_names[binding_scope]
 
+def get_ecr_repositories(image_mirror_output):
+  """
+    filter: get_ecr_repositories
+    author: Terence Quinn <[email protected]>
+    version_added: 0.1
+    short_description: Get the names of ECR repositories relevant to a mirror command
+    description:
+        - Parse the output of oc image mirror --dry-run=true command for repository names. The
+          list of repository names will be used to create all the required repositories in ECR
+          before actually performing the mirror. This is needed for AWS ECR only.
+    options:
+      image_mirror_output:
+        description: Output of the oc image mirror --dry-run=true command
+        required: True
+    notes:
+      - limited error handling, will not handle unexpected data currently
+  """
+  # marker to indicate summary of images from oc image mirror command
+  phase_found = False
+  repositories = []
+  for line in image_mirror_output:
+    line = line.strip()
+    # once we find a line starting with phase - start parsing repositories names
+    if line.startswith('phase '):
+      phase_found = True
+      continue
+    if line == "":
+      phase_found = False
+      continue
+    if phase_found:
+      # parse out the repository name
+      if len(line.split()) > 1:
+        repo_to_add = line.split()[1]
+        repositories.append(repo_to_add)
+  return repositories
+
 
 class FilterModule(object):
   def filters(self):
@@ -420,5 +456,6 @@ def filters(self):
       'format_pre_version_with_plus': format_pre_version_with_plus,
       'format_pre_version_without_buildid': format_pre_version_without_buildid,
       'format_pre_version_with_buildid': format_pre_version_with_buildid,
-      'get_db2_instance_name': get_db2_instance_name
+      'get_db2_instance_name': get_db2_instance_name,
+      'get_ecr_repositories': get_ecr_repositories
     }

ibm/mas_devops/roles/mirror_images/defaults/main.yml

@@ -6,6 +6,11 @@ registry_public_url: "{{ registry_public_host }}{% if registry_public_port != ''
 registry_prefix: "{{ lookup('env', 'REGISTRY_PREFIX') | default('', true) }}"
 registry_public_url_with_path: "{{ registry_public_url }}{% if registry_prefix | length > 0 %}/{{ registry_prefix }}{% endif %}"
 
+# AWS Elastic Container Registry
+# -----------------------------------------------------------------------------
+registry_is_ecr: "{{ lookup('env', 'REGISTRY_IS_ECR') | default(false, true) | bool }}"
+registry_ecr_aws_region: "{{ lookup('env', 'REGISTRY_ECR_AWS_REGION') | default('', true) }}"
+
 # Development config
 # -----------------------------------------------------------------------------
 artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}"
@@ -30,3 +35,4 @@ manifest_version: "{{ lookup('env', 'MANIFEST_VERSION') | replace('_', '.') }}"
 auth_file: "{{ ansible_env.HOME }}/.ibm-mas/auth.json"
 
 mirror_single_arch: "{{ lookup('env', 'MIRROR_SINGLE_ARCH') }}"
+

ibm/mas_devops/roles/mirror_images/tasks/main.yml

@@ -109,7 +109,65 @@
         that: ocimagemirror_result.rc == 0
         fail_msg: "Image mirroring failed (see debug information above)."
 
-# 4B. Execute the mirror (from filesystem)
+# 4B. Create AWS ECR Repositories if required
+# -----------------------------------------------------------------------------
+- name: "Create AWS ECR Repositories"
+  when:
+    - mirror_mode == "direct" or mirror_mode == "from-filesystem"
+    - registry_is_ecr == true
+  block:
+    - name: "{{ manifest_name }} : dry mode mirror command - direct"
+      shell: |
+        oc image mirror \
+          -a {{ auth_file }} \
+          -f {{ mirror_working_dir }}/manifests/direct/{{ manifest_name }}_{{ manifest_version }}.txt \
+          --filter-by-os '.*'  \
+          --insecure  \
+          --skip-multiple-scopes \
+          --max-per-registry=1 \
+          --dry-run=true
+      register: ocimagemirror_dryrun_result
+      ignore_errors: true
+      when: mirror_mode == "direct"
+
+    - name: "{{ manifest_name }} : dry mode mirror command - from-filesystem"
+      shell: |
+        oc image mirror \
+          -a {{ auth_file }} \
+          -f {{ mirror_working_dir }}/manifests/from-filesystem/{{ manifest_name }}_{{ manifest_version }}.txt \
+          --filter-by-os '.*'  \
+          --insecure  \
+          --skip-multiple-scopes \
+          --max-per-registry=1 \
+          --dir={{ mirror_working_dir }} \
+          --dry-run=true
+      register: ocimagemirror_dryrun_result
+      ignore_errors: true
+      when: mirror_mode == "from-filesystem"
+
+    - name: "{{ manifest_name }} : Debug dry mode mirror command"
+      debug:
+        msg: "{{ (ocimagemirror_dryrun_result.rc == 0) | ternary(ocimagemirror_dryrun_result.stdout_lines, ocimagemirror_dryrun_result.stderr_lines) }}"
+
+    - name: "{{ manifest_name }} : Fail if dry mode mirror is not successful"
+      assert:
+        that: ocimagemirror_dryrun_result.rc == 0
+        fail_msg: "mirror command in dry mode failed."
+
+    - name: Filter out repository names that must be created in ECR
+      set_fact:
+        repos_to_add: "{{ ocimagemirror_dryrun_result.stderr_lines | ibm.mas_devops.get_ecr_repositories }}"
+
+    - name: Repositories that will be created in ECR
+      debug:
+        msg: "{{ repos_to_add }}"
+
+    - name: Create ECR Repositories
+      ansible.builtin.shell: "aws ecr create-repository --repository-name {{ item }} --region {{ registry_ecr_aws_region }}"
+      ignore_errors: true
+      loop: "{{ repos_to_add }}"
+
+# 4C. Execute the mirror (from filesystem)
 # -----------------------------------------------------------------------------
 - name: "{{ manifest_name }} : Mirror Images from local filesystem"
   when: mirror_mode == "from-filesystem"
@@ -139,7 +197,7 @@
         that: ocimagemirror_result.rc == 0
         fail_msg: "Image mirroring failed (see debug information above)."
 
-# 4C. Execute the mirror (direct)
+# 4D. Execute the mirror (direct)
 # -----------------------------------------------------------------------------
 - name: "Mirror Images directly"
   when: mirror_mode == "direct"

ibm/mas_devops/roles/mirror_ocp/README.md

@@ -118,6 +118,20 @@ The public port number for the target registry
 - Environment Variable: `REGISTRY_PUBLIC_PORT`
 - Default: None
 
+### registry_is_ecr
+Specifies if the target registry is an AWS Elastic Container Registry instance
+
+- Optional
+- Environment Variable: `REGISTRY_IS_ECR`
+- Default: false
+
+### registry_ecr_aws_region
+The AWS region of the AWS Elastic Container Registry. Only required if the target registry is an AWS Elastic Container Registry instance.
+
+- Optional
+- Environment Variable: `REGISTRY_ECR_AWS_REGION`
+- Default: None
+
 ### registry_prefix_redhat
 The prefix used for the target registry.  The images will not be mirrored to the registry at this time but will define the final destination in the form: `{host}[:{port}]/{prefix}/{reponame}`
 

ibm/mas_devops/roles/mirror_ocp/defaults/main.yml

@@ -20,6 +20,11 @@ registry_username: "{{ lookup('env', 'REGISTRY_USERNAME') }}"
 registry_password: "{{ lookup('env', 'REGISTRY_PASSWORD') }}"
 registry_auth: "{{ registry_username }}:{{ registry_password }}"
 
+# AWS Elastic Container Registry
+# -----------------------------------------------------------------------------
+registry_is_ecr: "{{ lookup('env', 'REGISTRY_IS_ECR') | default(false, true) | bool }}"
+registry_ecr_aws_region: "{{ lookup('env', 'REGISTRY_ECR_AWS_REGION') | default('', true) }}"
+
 # Release Config
 # -----------------------------------------------------------------------------
 ocp_release: "{{ lookup('env', 'OCP_RELEASE') }}"

ibm/mas_devops/roles/mirror_ocp/tasks/actions/direct.yml

@@ -9,14 +9,54 @@
       - mirror_working_dir is defined and mirror_working_dir != ""
     fail_msg: "One or more required properties are missing"
 
-# 2. Perform Mirroring
+# 2a. Perform Mirroring
 # -----------------------------------------------------------------------------
 - name: "Generate ImageSetConfiguration"
   ansible.builtin.template:
     src: imagesetconfiguration.yml.j2
     dest: "{{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml"
     mode: "664"
 
+
+# 2b. Create AWS ECR Repositories if required
+# -----------------------------------------------------------------------------
+- name: "Create AWS ECR Repositories"
+  when:
+    - registry_is_ecr == true
+  block:
+    - name: Create validation repository required for oc mirror command
+      ansible.builtin.shell: "aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/oc-mirror --region {{ registry_ecr_aws_region }}"
+      ignore_errors: true
+
+    - name: "{{ ocp_release }} : dry mode mirror command"
+      shell: >
+        DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dry-run --dest-skip-tls --config={{ mirror_working_dir }}/imageset-ocp{{ ocp_release }}.yml docker://{{ registry_public_url_with_path_redhat }}
+      register: ocimagemirror_dryrun_result
+      ignore_errors: true
+
+    - name: "{{ ocp_release }} : Debug mirror command dry mode"
+      debug:
+        msg:
+          - "{{ ocimagemirror_dryrun_result.stderr_lines }} "
+
+    - name: "{{ ocp_release }} : Fail if dry mode mirror is not successful"
+      assert:
+        that: ocimagemirror_dryrun_result.rc == 0
+        fail_msg: "mirror command in dry mode failed."
+
+    - name: Filter out repository names that must be created in ECR
+      set_fact:
+        repos_to_add: "{{ ocimagemirror_dryrun_result.stderr_lines | ibm.mas_devops.get_ecr_repositories }}"
+
+    - name: Repositories that will be created in ECR
+      debug:
+        msg: "{{ repos_to_add }}"
+
+    - name: Create ECR Repositories
+      ansible.builtin.shell: "aws ecr create-repository --repository-name {{ item }} --region {{ registry_ecr_aws_region }}"
+      ignore_errors: true
+      loop: "{{ repos_to_add }}"
+
 - name: "Debug Information"
   debug:
     msg:

ibm/mas_devops/roles/mirror_ocp/tasks/actions/from-filesystem.yml

@@ -1,5 +1,5 @@
 ---
-# 1. Check for undefined properties that do not have a default
+# 1a. Check for undefined properties that do not have a default
 # -----------------------------------------------------------------------------
 - name: "Fail if required properties are not provided"
   assert:
@@ -8,6 +8,49 @@
       - mirror_working_dir is defined and mirror_working_dir != ""
     fail_msg: "One or more required properties are missing"
 
+# 1b. Create AWS ECR Repositories if required
+# -----------------------------------------------------------------------------
+- name: "Create AWS ECR Repositories"
+  when:
+    - registry_is_ecr == true
+  block:
+    - name: Create validation repository required for oc mirror command
+      ansible.builtin.shell: |
+        aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/oc-mirror --region {{ registry_ecr_aws_region }}
+        aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/redhat/certified-operator-index --region {{ registry_ecr_aws_region }}
+        aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/redhat/community-operator-index --region {{ registry_ecr_aws_region }}
+        aws ecr create-repository --repository-name {{ registry_prefix_redhat }}/redhat/redhat-operator-index --region {{ registry_ecr_aws_region }}
+      ignore_errors: true
+
+    - name: "{{ ocp_release }} : dry mode mirror command"
+      shell: >
+        DOCKER_CONFIG={{ mirror_working_dir }} oc mirror --dry-run --dest-skip-tls --max-per-registry=1 --from={{ mirror_working_dir }}/mirror_seq1_000000.tar docker://{{ registry_public_url_with_path_redhat }}
+      register: ocimagemirror_dryrun_result
+      ignore_errors: true
+
+    - name: "{{ ocp_release }} : Debug mirror command dry mode"
+      debug:
+        msg:
+          - "{{ ocimagemirror_dryrun_result.stderr_lines }} "
+
+    - name: "{{ ocp_release }} : Fail if dry mode mirror is not successful"
+      assert:
+        that: ocimagemirror_dryrun_result.rc == 0
+        fail_msg: "mirror command in dry mode failed."
+
+    - name: Filter out repository names that must be created in ECR
+      set_fact:
+        repos_to_add: "{{ ocimagemirror_dryrun_result.stderr_lines | ibm.mas_devops.get_ecr_repositories }}"
+
+    - name: Repositories that will be created in ECR
+      debug:
+        msg: "{{ repos_to_add }}"
+
+    - name: Create ECR Repositories
+      ansible.builtin.shell: "aws ecr create-repository --repository-name {{ item }} --region {{ registry_ecr_aws_region }}"
+      ignore_errors: true
+      loop: "{{ repos_to_add }}"
+
 # 2. Perform Mirroring
 # -----------------------------------------------------------------------------
 - name: "Debug Information"