Add OTA_IMAGES required for Software Update CI Jobs (project-chip#40939)

jtrejoespinoza-grid · restyled-commits · web-flow · commit 42efed9360a2 · 2025-11-04T19:00:32.000Z
* Fix ruff and isort

* Updated build paths for OTA_IMAGES

* Added missing append into test_env.yaml

* Fix issue

* Fix rm path

* Fix version assignement

* Moved py script to bash

* Restyled by whitespace

* Restyled by shellharden

* Restyled by shfmt

* Fix lint issues

* Restyled by whitespace

* Restyled by shfmt

* Removed python version. Added info to show which files where created during the script.

* Restyled by shellharden

* Added test to verify the version is correct

* Added version 5

* Restyled by autopep8

* Removed long string and replaced with arg list. Updated to check up to 5 ota images

* fix: Added missing comma that was causing two invalid parameters

* Updated range to build one image

* Removed harcoded paths for image build. Added  range in the tests.yaml. Added paths for apps to validate in tests.yaml. Removed OTA env variables that will not be used for the moment

* Ruff check

* isort fix

* Removed code modification to update the version, now using target_cflags. Updated paths for version creation.

* Restyled by whitespace

* Restyled by shellharden

* Env variable name corrected.

* Fix after review

* Restyled by shfmt

---------

Co-authored-by: Restyled.io &lt;commits@restyled.io&gt;
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -795,6 +795,35 @@ jobs:
             - name: ccache stats
               run: ccache -s
 
+            - name: Build linux-x64-requestor-app
+              env:
+                  CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache"
+              run: >-
+                  ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py
+                  --target linux-x64-ota-requestor-${BUILD_VARIANT}-tsan-clang-test
+                  --pw-command-launcher=ccache build --copy-artifacts-to objdir-clone
+                  && rm -rf out/linux-x64-ota-requestor-${BUILD_VARIANT}-tsan-clang-test"
+
+            - name: Build linux-x64-provider-app
+              env:
+                  CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache"
+              run: >-
+                  ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py
+                  --target linux-x64-ota-provider-${BUILD_VARIANT}-tsan-clang-test
+                  --pw-command-launcher=ccache build --copy-artifacts-to objdir-clone
+                  && rm -rf out/linux-x64-ota-provider-${BUILD_VARIANT}-tsan-clang-test"
+
+            - name: Build Software Update ota images with new version.
+              env:
+                  CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache"
+              run: >- 
+                  ./scripts/run_in_build_env.sh "./scripts/build/build_ota_images.sh --out-prefix out/su_ota_images_min --max-range 2
+                  && mv out/su_ota_images_min-v*/*.ota objdir-clone
+                  && rm -rf out/su_ota_images_min-v*"
+
+            - name: ccache stats
+              run: ccache -s
+
             - name: Install push_av_server dependencies
               run: >-
                   ./scripts/run_in_python_env.sh out/venv \
@@ -822,19 +851,25 @@ jobs:
                   echo "FABRIC_SYNC_APP: objdir-clone/linux-x64-fabric-sync-${BUILD_VARIANT}-clang/fabric-sync" >> /tmp/test_env.yaml
                   echo "LIGHTING_APP_NO_UNIQUE_ID: objdir-clone/linux-x64-light-data-model-no-unique-id-${BUILD_VARIANT}-clang/chip-lighting-app" >> /tmp/test_env.yaml
                   echo "TERMS_AND_CONDITIONS_APP: objdir-clone/linux-x64-terms-and-conditions/chip-terms-and-conditions-app" >> /tmp/test_env.yaml
+                  echo "OTA_PROVIDER_APP: objdir-clone/linux-x64-ota-provider-${BUILD_VARIANT}-tsan-clang-test/chip-ota-provider-app" >> /tmp/test_env.yaml
+                  echo "OTA_REQUESTOR_APP: objdir-clone/linux-x64-ota-requestor-${BUILD_VARIANT}-tsan-clang-test/chip-ota-requestor-app" >> /tmp/test_env.yaml
                   echo "TRACE_APP: out/trace_data/app-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml
                   echo "JF_CONTROL_APP: objdir-clone/linux-x64-jf-control-app/jfc-app" >> /tmp/test_env.yaml
                   echo "JF_ADMIN_APP: objdir-clone/linux-x64-jf-admin-app/jfa-app" >> /tmp/test_env.yaml
                   echo "CLOSURE_APP: objdir-clone/linux-x64-closure-${BUILD_VARIANT}-tsan-clang-test-unified/closure-app" >> /tmp/test_env.yaml
                   echo "WATER_LEAK_DETECTOR_APP: objdir-clone/linux-x64-water-leak-detector-${BUILD_VARIANT}-tsan-clang-test-unified/water-leak-detector-app" >> /tmp/test_env.yaml
                   echo "PUSH_AV_SERVER: src/tools/push_av_server/server.py" >> /tmp/test_env.yaml
 
+                  # SU OTA Images
+                  echo "SU_OTA_REQUESTOR_V2: objdir-clone/chip-ota-requestor-app_v2.min.ota" >> /tmp/test_env.yaml
+
                   # Generic trace setups after applications
                   echo "TRACE_TEST_JSON: out/trace_data/test-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml
                   echo "TRACE_TEST_PERFETTO: out/trace_data/test-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml
 
             - name: Verify Testing Support
               run: |
+                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_ota_images_versions.py -i objdir-clone/chip-ota-requestor-app_v2.min.ota -v 2' 
                   scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_IDM_10_4.py'
                   scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_TC_ICDM_2_1_full_pics.py'
                   scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_TC_ICDM_2_1_min_pics.py'
@@ -853,7 +888,6 @@ jobs:
                   scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/TestSpecParsingSelection.py'
                   scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/TestSpecParsingSupport.py'
                   scripts/run_in_python_env.sh out/venv 'PYTHONPATH=src/python_testing:$PYTHONPATH python3 -m unittest discover -s src/python_testing/mdns_discovery/tests -p "test_*.py"'
-
             - name: Run Tests
               run: |
                   mkdir -p out/trace_data
diff --git a/scripts/build/build_ota_images.sh b/scripts/build/build_ota_images.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+#
+# Copyright (c) 2025 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -x
+CHIP_ROOT="$(dirname "$0")/../.."
+
+# Default values for optional arguments
+VENDOR_ID="0xDEAD"
+PRODUCT_ID="0xBEEF"
+MAX_RANGE=6
+OUT_PREFIX="out/su_ota_images_min"
+BUILT_IMAGES_STACK=()
+
+# Parse command line arguments
+while [[ "$#" -gt 0 ]]; do
+    case "$1" in
+        --vendor-id)
+            VENDOR_ID="$2"
+            shift 2
+            ;;
+        --product-id)
+            PRODUCT_ID="$2"
+            shift 2
+            ;;
+        --max-range)
+            MAX_RANGE="$2"
+            shift 2
+            ;;
+        --out-prefix)
+            OUT_PREFIX="$2"
+            shift 2
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
+echo "Values to use for build"
+echo "Vendor ID: $VENDOR_ID"
+echo "Product ID: $PRODUCT_ID"
+echo "Max Range: $MAX_RANGE"
+echo "Output Prefix: $OUT_PREFIX"
+
+BASE_OUT_PREFIX="$CHIP_ROOT/$OUT_PREFIX"
+echo "PREFIX $BASE_OUT_PREFIX"
+
+echo "Starting building the ota images with different version number."
+
+for ((i = 2; i <= "$MAX_RANGE"; i++)); do
+    echo "Building for version $i"
+
+    # Use a per-version GN output dir to avoid clobbering / to enable reuse
+    OUT_DIR="$BASE_OUT_PREFIX-v$i"
+    mkdir -p "$OUT_DIR"
+
+    # Pass version via GN args (target_cflags). Quotes must be escaped so the GN arg
+    # arrives intact. gn_build_example.sh forwards key=value args to gn gen.
+    GN_VERSION_ARGS="target_cflags=[\"-DCHIP_DEVICE_CONFIG_DEVICE_SOFTWARE_VERSION=$i\",\"-DCHIP_DEVICE_CONFIG_DEVICE_SOFTWARE_VERSION_STRING=\\\"$i.0\\\"\"]"
+
+    # Build the image without editing source files
+    echo "Building the requestor app (out: $OUT_DIR)"
+    ./"$CHIP_ROOT"/scripts/examples/gn_build_example.sh \
+        "$CHIP_ROOT"/examples/ota-requestor-app/linux "$OUT_DIR" \
+        chip_config_network_layer_ble=false is_debug=false "$GN_VERSION_ARGS" >/dev/null
+
+    # Strip command (create .min binary in the same OUT_DIR)
+    if [ "$(uname -s)" = "Darwin" ]; then
+        strip "$OUT_DIR"/chip-ota-requestor-app -o "$OUT_DIR"/chip-ota-requestor-app.min >/dev/null 2>&1
+    else
+        strip --strip-all "$OUT_DIR"/chip-ota-requestor-app -o "$OUT_DIR"/chip-ota-requestor-app.min >/dev/null 2>&1
+    fi
+
+    # Create ota image (store images under the shared OUT_PREFIX directory)
+    OTA_IMAGE_PATH="$OUT_DIR/chip-ota-requestor-app_v$i.min.ota"
+    python3 "$CHIP_ROOT"/src/app/ota_image_tool.py create -v "$VENDOR_ID" -p "$PRODUCT_ID" -vn "$i" -vs "$i.0" -da sha256 "$OUT_DIR"/chip-ota-requestor-app.min "$OTA_IMAGE_PATH"
+
+    BUILT_IMAGES_STACK+=("$OTA_IMAGE_PATH")
+done
+
+echo "Generated files"
+for item in "${BUILT_IMAGES_STACK[@]}"; do
+    echo "$item"
+done
diff --git a/src/python_testing/test_testing/test_ota_images_versions.py b/src/python_testing/test_testing/test_ota_images_versions.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env -S python3 -B
+#
+#    Copyright (c) 2025 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+import logging
+import os
+import subprocess
+import sys
+
+import click
+
+CHIP_ROOT = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '../../..'))
+RUNNER_SCRIPT_DIR = os.path.join(CHIP_ROOT, 'scripts/tests')
+OTA_TOOL_DIR = os.path.join(CHIP_ROOT, 'src/app')
+
+
+def run_single_test(otaimage: str, otaimage_version: int) -> int:
+
+    # Find the image
+    ota_image = os.path.join(CHIP_ROOT, otaimage)
+
+    ota_image_bin = os.path.join(CHIP_ROOT, f'{ota_image}.bin')
+
+    # Extract the image into the directory
+    ota_tool = os.path.abspath(os.path.join(
+        OTA_TOOL_DIR, 'ota_image_tool.py'))
+    extract_cmd = str(ota_tool) + ' extract ' + str(ota_image) + ' ' + str(ota_image_bin)
+    status = subprocess.call(extract_cmd, shell=True)
+    logging.info("Extract image : " + str(status))
+    if status != 0:
+        logging.error(f"Failed to extract the image from {ota_image}")
+        exit(1)
+    logging.info(f"Image extracted into {ota_image_bin}")
+
+    # Mod update
+    cmd_mod = "chmod +x " + str(ota_image_bin)
+    status = subprocess.call(cmd_mod, shell=True)
+    if status != 0:
+        logging.info(f"Failed to change +x permission on the image  {ota_image_bin}")
+        exit(1)
+
+    app_args = ' --discriminator 1234 '
+
+    script_args = [
+        "--commissioning-method on-network",
+        "--passcode 20202021",
+        "--discriminator 1234",
+        f"--int-arg SOFTWAREVERSION:{otaimage_version}",
+        "--storage-path admin_storage.json"
+    ]
+
+    script_args = " ".join(script_args)
+
+    script = os.path.abspath(os.path.join(
+        CHIP_ROOT, 'src/python_testing/test_testing/test_ota_version.py'))
+
+    # run_python_test uses click so call as a command
+    run_python_test = os.path.abspath(os.path.join(
+        RUNNER_SCRIPT_DIR, 'run_python_test.py'))
+    test_cmd = str(run_python_test) + ' --factory-reset  --app ' + str(ota_image_bin) + ' --app-args "' + \
+        app_args + '" --script ' + str(script) + ' --script-args "' + script_args + '"'
+
+    process_status = subprocess.call(test_cmd, shell=True)
+
+    cmd_clean = "rm " + str(ota_image_bin)
+    subprocess.call(cmd_clean, shell=True)
+    return process_status
+
+
+@click.command()
+@click.option('--otaimages', '-i', multiple=True, type=click.Path(exists=True))
+@click.option('--otaimagesversions', '-v', multiple=True, type=(int))
+def main(otaimages: str, otaimagesversions: int):
+    if len(otaimages) == 0 or otaimagesversions == 0:
+        logging.error("Must provide at least one image to verify")
+        exit(1)
+    if len(otaimages) != len(otaimagesversions):
+        logging.error("Provided non matching images to provided versions")
+    passes = []
+    main_status = 0
+    for index in range(len(otaimages)):
+        otaimage = otaimages[index]
+        otaimage_version = otaimagesversions[index]
+        logging.info(f"Verifying the image {otaimage} has the reported version {otaimage_version}")
+        status = run_single_test(otaimage=otaimage, otaimage_version=otaimage_version)
+        passes.append((otaimage, otaimage_version, status))
+
+    for iter in passes:
+        if iter[2] != 0:
+            logging.error(f"Image version missmatched for ota image: {iter[0]} expected: {iter[1]}")
+            main_status = 1
+
+    sys.exit(main_status)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/src/python_testing/test_testing/test_ota_version.py b/src/python_testing/test_testing/test_ota_version.py
@@ -0,0 +1,44 @@
+#
+#    Copyright (c) 2025 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+import logging
+
+from mobly import asserts
+
+import matter.clusters as Clusters
+from matter.testing.matter_testing import MatterBaseTest, async_test_body, default_matter_test_main
+
+
+class TestCheckSoftwareVersion(MatterBaseTest):
+
+    @async_test_body
+    async def setup_class(self):
+        super().setup_class()
+        logging.info("This log has started")
+
+    @async_test_body
+    async def test_ota_image_version(self):
+        self.print_step(0, "Commissioning - already done")
+        expected_software_version = self.matter_test_config.global_test_params['SOFTWAREVERSION']
+
+        self.print_step(1, f"Verify cluster version is: {expected_software_version}")
+        current_software_version = await self.read_single_attribute_check_success(cluster=Clusters.BasicInformation, attribute=Clusters.BasicInformation.Attributes.SoftwareVersion)
+        asserts.assert_equal(current_software_version, expected_software_version,
+                             f"Version {current_software_version} is not the expected {expected_software_version}")
+
+
+if __name__ == '__main__':
+    default_matter_test_main()
