From c6908e9fddbe1a5b29c4829f5bf081111e0fca0b Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 17 Sep 2025 03:15:39 -0700 Subject: [PATCH 01/18] Add automatic device attribute dump on composition test failures: - Move log_structured_data to module level in basic_composition support module for reusability - Add on_fail override in BasicCompositionTests to automatically dump device data - Ensures all composition tests automatically log device attributes on failure - Improves debugging by providing IDM-12.1 equivalent data without manual intervention - Add missing logging import to TC_DeviceConformance.py --- .../TC_DeviceBasicComposition.py | 9 +---- src/python_testing/TC_DeviceConformance.py | 3 +- .../matter/testing/basic_composition.py | 36 +++++++++++++++++++ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index 555d3ca3d9c090..7d2280fbb2e98d 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -189,7 +189,7 @@ from matter.clusters.Types import Nullable from matter.exceptions import ChipStackError from matter.interaction_model import InteractionModelError, Status -from matter.testing.basic_composition import BasicCompositionTests +from matter.testing.basic_composition import BasicCompositionTests, log_structured_data from matter.testing.global_attribute_ids import (AttributeIdType, ClusterIdType, CommandIdType, GlobalAttributeIds, attribute_id_type, cluster_id_type, command_id_type) from matter.testing.matter_testing import MatterBaseTest, TestStep, async_test_body, default_matter_test_main @@ -965,13 +965,6 @@ def test_TC_IDM_12_1(self): json_str, txt_str = self.dump_wildcard(dump_device_composition_path) # Structured dump so we can pull these back out of the logs - def log_structured_data(start_tag: str, dump_string): - lines = dump_string.splitlines() - logging.info(f'{start_tag}BEGIN ({len(lines)} lines)====') - for line in lines: - logging.info(f'{start_tag}{line}') - logging.info(f'{start_tag}END ====') - log_structured_data('==== json: ', json_str) log_structured_data('==== txt: ', txt_str) diff --git a/src/python_testing/TC_DeviceConformance.py b/src/python_testing/TC_DeviceConformance.py index 8be2a1f8a389e9..237e6d0b5171e6 100644 --- a/src/python_testing/TC_DeviceConformance.py +++ b/src/python_testing/TC_DeviceConformance.py @@ -36,10 +36,11 @@ # === END CI TEST ARGUMENTS === # TODO: Enable 10.5 in CI once the door lock OTA requestor problem is sorted. +import logging from typing import Callable import matter.clusters as Clusters -from matter.testing.basic_composition import BasicCompositionTests +from matter.testing.basic_composition import BasicCompositionTests, log_structured_data from matter.testing.choice_conformance import (evaluate_attribute_choice_conformance, evaluate_command_choice_conformance, evaluate_feature_choice_conformance) from matter.testing.conformance import conformance_allowed diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index ba21019a30c38b..18f34dbd0eaa0a 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -39,6 +39,23 @@ from matter.testing.spec_parsing import PrebuiltDataModelDirectory, build_xml_clusters, build_xml_device_types, dm_from_spec_version +def log_structured_data(start_tag: str, dump_string: str): + """Log structured data with a clear start and end marker. + + This function is used to output device attribute dumps and other structured + data to logs in a format that can be easily extracted for debugging. + + Args: + start_tag: A prefix tag to identify the type of data being logged + dump_string: The data to be logged + """ + lines = dump_string.splitlines() + logging.info(f'{start_tag}BEGIN ({len(lines)} lines)====') + for line in lines: + logging.info(f'{start_tag}{line}') + logging.info(f'{start_tag}END ====') + + @dataclass class ArlData: have_arl: bool @@ -276,3 +293,22 @@ def build_spec_xmls(self): self.xml_clusters, self.problems = build_xml_clusters(dm) self.xml_device_types, problems = build_xml_device_types(dm) self.problems.extend(problems) + + def on_fail(self, record): + """Override on_fail to automatically dump device data when any composition test fails. + + This ensures that whenever any test inheriting from BasicCompositionTests fails, + we automatically get the device attribute dump for debugging purposes. + """ + # Call the parent on_fail method first (this will be MatterBaseTest.on_fail) + super().on_fail(record) + + # Dump device composition data if available for debugging + try: + if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: + json_str, txt_str = self.dump_wildcard(None) # Don't write to file, just get strings + log_structured_data('==== FAILURE_DUMP_json: ', json_str) + log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) + except Exception as e: + # Don't let logging errors interfere with the original test failure + logging.warning(f"Failed to dump device data on test failure: {e}") From e2154ab385cdee3182aff258f83f1f1f4e711e68 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 17 Sep 2025 10:28:57 +0000 Subject: [PATCH 02/18] Restyled by autopep8 --- .../matter/testing/basic_composition.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index 18f34dbd0eaa0a..0bf332f8fcfc0f 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -41,10 +41,10 @@ def log_structured_data(start_tag: str, dump_string: str): """Log structured data with a clear start and end marker. - + This function is used to output device attribute dumps and other structured data to logs in a format that can be easily extracted for debugging. - + Args: start_tag: A prefix tag to identify the type of data being logged dump_string: The data to be logged @@ -296,13 +296,13 @@ def build_spec_xmls(self): def on_fail(self, record): """Override on_fail to automatically dump device data when any composition test fails. - + This ensures that whenever any test inheriting from BasicCompositionTests fails, we automatically get the device attribute dump for debugging purposes. """ # Call the parent on_fail method first (this will be MatterBaseTest.on_fail) super().on_fail(record) - + # Dump device composition data if available for debugging try: if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: From 9c36c192584b0177382cd517f55c8dce1ba89bc4 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 17 Sep 2025 03:44:25 -0700 Subject: [PATCH 03/18] Remove duplicate on_fail() and unused import from TC_DeviceConformance - Remove on_fail() override as it's now inherited from BasicCompositionTests - Remove unused log_structured_data import --- src/python_testing/TC_DeviceConformance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/TC_DeviceConformance.py b/src/python_testing/TC_DeviceConformance.py index 237e6d0b5171e6..b4c817c401a41b 100644 --- a/src/python_testing/TC_DeviceConformance.py +++ b/src/python_testing/TC_DeviceConformance.py @@ -40,7 +40,7 @@ from typing import Callable import matter.clusters as Clusters -from matter.testing.basic_composition import BasicCompositionTests, log_structured_data +from matter.testing.basic_composition import BasicCompositionTests from matter.testing.choice_conformance import (evaluate_attribute_choice_conformance, evaluate_command_choice_conformance, evaluate_feature_choice_conformance) from matter.testing.conformance import conformance_allowed From 5edf7773f9535b9395acf480fd902992bfbfba94 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 17 Sep 2025 03:52:49 -0700 Subject: [PATCH 04/18] Removing unused logging import --- src/python_testing/TC_DeviceConformance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/TC_DeviceConformance.py b/src/python_testing/TC_DeviceConformance.py index b4c817c401a41b..8be2a1f8a389e9 100644 --- a/src/python_testing/TC_DeviceConformance.py +++ b/src/python_testing/TC_DeviceConformance.py @@ -36,7 +36,6 @@ # === END CI TEST ARGUMENTS === # TODO: Enable 10.5 in CI once the door lock OTA requestor problem is sorted. -import logging from typing import Callable import matter.clusters as Clusters From 143855018511ce9faeaeeda7d0ca688e184bf36c Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 17 Sep 2025 04:42:09 -0700 Subject: [PATCH 05/18] Change inheritance order to enable automatic device attribute dump on composition test failures - Change TC_DeviceConformance from (MatterBaseTest, DeviceConformanceTests) to (DeviceConformanceTests, MatterBaseTest) - Change TC_DeviceBasicComposition from (MatterBaseTest, BasicCompositionTests) to (BasicCompositionTests, MatterBaseTest) - This ensures BasicCompositionTests.teardown_class() takes precedence in Method Resolution Order - Enables automatic device attribute dumps when composition tests detect problems - Improves debugging efficiency by providing IDM-12.1 equivalent data without manual intervention The inheritance order change allows the teardown_class override in BasicCompositionTests to execute before MatterBaseTest.teardown_class, ensuring device data is dumped before the standard problem logging occurs. --- .../TC_DeviceBasicComposition.py | 2 +- src/python_testing/TC_DeviceConformance.py | 2 +- .../matter/testing/basic_composition.py | 45 +++++++++++++++---- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index 7d2280fbb2e98d..548a53acc89db2 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -271,7 +271,7 @@ def check_no_duplicates(obj: Any) -> None: raise ValueError(f"Value {str(obj)} contains duplicate values") -class TC_DeviceBasicComposition(MatterBaseTest, BasicCompositionTests): +class TC_DeviceBasicComposition(BasicCompositionTests, MatterBaseTest): @async_test_body async def setup_class(self): super().setup_class() diff --git a/src/python_testing/TC_DeviceConformance.py b/src/python_testing/TC_DeviceConformance.py index 8be2a1f8a389e9..eabd6ac03cd77c 100644 --- a/src/python_testing/TC_DeviceConformance.py +++ b/src/python_testing/TC_DeviceConformance.py @@ -505,7 +505,7 @@ def check_root_node_restricted_clusters(self) -> list[ProblemNotice]: return problems -class TC_DeviceConformance(MatterBaseTest, DeviceConformanceTests): +class TC_DeviceConformance(DeviceConformanceTests, MatterBaseTest): @async_test_body async def setup_class(self): super().setup_class() diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index 0bf332f8fcfc0f..e2449dbf6f6fff 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -294,21 +294,48 @@ def build_spec_xmls(self): self.xml_device_types, problems = build_xml_device_types(dm) self.problems.extend(problems) - def on_fail(self, record): - """Override on_fail to automatically dump device data when any composition test fails. + def teardown_class(self): + """Override teardown_class to dump device attribute data when problems are found. + + This ensures that whenever any test inheriting from BasicCompositionTests has problems, + we automatically get the device attribute dump for debugging purposes. + """ + # Check if we have problems and device attributes are available + if len(self.problems) > 0: + logging.info("BasicCompositionTests: Problems detected - attempting device attribute dump") + try: + if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: + logging.info("Device attribute data available - generating dump") + _, txt_str = self.dump_wildcard(None) + # Only dump the text format - it's more readable for debugging + log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) + else: + logging.info("No device attribute data available (endpoints_tlv not populated)") + except Exception as e: + # Don't let logging errors interfere with the original test failure + logging.warning(f"Failed to generate device attribute dump: {e}") + + # Call the parent teardown_class method to handle normal teardown and problem logging + super().teardown_class() + def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # type: ignore[misc] + """Override fail_current_test to automatically dump device attribute data when any composition test fails. + This ensures that whenever any test inheriting from BasicCompositionTests fails, we automatically get the device attribute dump for debugging purposes. """ - # Call the parent on_fail method first (this will be MatterBaseTest.on_fail) - super().on_fail(record) - - # Dump device composition data if available for debugging + # Dump device attribute data if available for debugging BEFORE failing the test try: if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: - json_str, txt_str = self.dump_wildcard(None) # Don't write to file, just get strings - log_structured_data('==== FAILURE_DUMP_json: ', json_str) + logging.info("Device attribute dump available - generating dump") + _, txt_str = self.dump_wildcard(None) + # Only dump the text format - it's more readable for debugging log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) + else: + logging.info("No device attribute dump available (endpoints_tlv not populated)") except Exception as e: # Don't let logging errors interfere with the original test failure - logging.warning(f"Failed to dump device data on test failure: {e}") + logging.warning(f"Failed to generate device attribute dump on test failure: {e}") + + # Call the parent fail_current_test method to actually fail the test + super().fail_current_test(msg) From 9abde9b6602c6f4b4a9cfaf63f6c86dbbf55d7dc Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 17 Sep 2025 11:42:52 +0000 Subject: [PATCH 06/18] Restyled by autopep8 --- .../matter/testing/basic_composition.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index e2449dbf6f6fff..45ae8e8b903501 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -296,7 +296,7 @@ def build_spec_xmls(self): def teardown_class(self): """Override teardown_class to dump device attribute data when problems are found. - + This ensures that whenever any test inheriting from BasicCompositionTests has problems, we automatically get the device attribute dump for debugging purposes. """ @@ -306,7 +306,7 @@ def teardown_class(self): try: if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: logging.info("Device attribute data available - generating dump") - _, txt_str = self.dump_wildcard(None) + _, txt_str = self.dump_wildcard(None) # Only dump the text format - it's more readable for debugging log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) else: @@ -314,13 +314,13 @@ def teardown_class(self): except Exception as e: # Don't let logging errors interfere with the original test failure logging.warning(f"Failed to generate device attribute dump: {e}") - + # Call the parent teardown_class method to handle normal teardown and problem logging super().teardown_class() def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # type: ignore[misc] """Override fail_current_test to automatically dump device attribute data when any composition test fails. - + This ensures that whenever any test inheriting from BasicCompositionTests fails, we automatically get the device attribute dump for debugging purposes. """ @@ -328,7 +328,7 @@ def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # ty try: if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: logging.info("Device attribute dump available - generating dump") - _, txt_str = self.dump_wildcard(None) + _, txt_str = self.dump_wildcard(None) # Only dump the text format - it's more readable for debugging log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) else: @@ -336,6 +336,6 @@ def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # ty except Exception as e: # Don't let logging errors interfere with the original test failure logging.warning(f"Failed to generate device attribute dump on test failure: {e}") - + # Call the parent fail_current_test method to actually fail the test super().fail_current_test(msg) From dc5df9e053dc8749ee2dbe61168a2ff264a9704d Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 17 Sep 2025 05:01:56 -0700 Subject: [PATCH 07/18] resolving linting errors and making sure that on test failures we emit the failure msg correctly --- .../matter/testing/basic_composition.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index 45ae8e8b903501..63c0ce91fe6de1 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -261,13 +261,6 @@ def get_test_name(self) -> str: return "" return frame.f_code.co_name - def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # type: ignore[misc] - if not msg: - # Without a message, just log the last problem seen - asserts.fail(msg=self.problems[-1].problem) - else: - asserts.fail(msg) - def _get_dm(self) -> PrebuiltDataModelDirectory: # type: ignore[return] # mypy doesn't understand that asserts.fail always raises a TestFailure try: @@ -337,5 +330,9 @@ def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # ty # Don't let logging errors interfere with the original test failure logging.warning(f"Failed to generate device attribute dump on test failure: {e}") - # Call the parent fail_current_test method to actually fail the test - super().fail_current_test(msg) + if not msg: + # Without a message, just log the last problem seen + asserts.fail(msg=self.problems[-1].problem) + else: + asserts.fail(msg) + From 6756ca75996153fd08173163e161b73489df9e19 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 17 Sep 2025 12:02:36 +0000 Subject: [PATCH 08/18] Restyled by autopep8 --- .../matter/testing/basic_composition.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index 63c0ce91fe6de1..753d7231b284db 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -335,4 +335,3 @@ def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # ty asserts.fail(msg=self.problems[-1].problem) else: asserts.fail(msg) - From 10f1ca6cc3cfa7a178bd1493795b40ea8f5c2858 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 24 Sep 2025 14:00:29 -0700 Subject: [PATCH 09/18] Update basic_composition.py Restoring fail_current_test() as per Cecille's suggestion, thank you! --- .../matter/testing/basic_composition.py | 32 ++++--------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index 753d7231b284db..2b2636e082a958 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -261,6 +261,13 @@ def get_test_name(self) -> str: return "" return frame.f_code.co_name + def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # type: ignore[misc] + if not msg: + # Without a message, just log the last problem seen + asserts.fail(msg=self.problems[-1].problem) + else: + asserts.fail(msg) + def _get_dm(self) -> PrebuiltDataModelDirectory: # type: ignore[return] # mypy doesn't understand that asserts.fail always raises a TestFailure try: @@ -310,28 +317,3 @@ def teardown_class(self): # Call the parent teardown_class method to handle normal teardown and problem logging super().teardown_class() - - def fail_current_test(self, msg: Optional[str] = None) -> typing.NoReturn: # type: ignore[misc] - """Override fail_current_test to automatically dump device attribute data when any composition test fails. - - This ensures that whenever any test inheriting from BasicCompositionTests fails, - we automatically get the device attribute dump for debugging purposes. - """ - # Dump device attribute data if available for debugging BEFORE failing the test - try: - if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: - logging.info("Device attribute dump available - generating dump") - _, txt_str = self.dump_wildcard(None) - # Only dump the text format - it's more readable for debugging - log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) - else: - logging.info("No device attribute dump available (endpoints_tlv not populated)") - except Exception as e: - # Don't let logging errors interfere with the original test failure - logging.warning(f"Failed to generate device attribute dump on test failure: {e}") - - if not msg: - # Without a message, just log the last problem seen - asserts.fail(msg=self.problems[-1].problem) - else: - asserts.fail(msg) From a026a85c6e7c1184a64c45f17b7f182522d43f09 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 24 Sep 2025 14:23:40 -0700 Subject: [PATCH 10/18] Update basic_composition.py Updated to using module-level logger instance instead of using logging directly --- .../matter/testing/basic_composition.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index 2b2636e082a958..937c7a96e5fea3 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -38,6 +38,7 @@ from matter.testing.matter_testing import MatterTestConfig, ProblemNotice from matter.testing.spec_parsing import PrebuiltDataModelDirectory, build_xml_clusters, build_xml_device_types, dm_from_spec_version +logger = logging.getLogger(__name__) def log_structured_data(start_tag: str, dump_string: str): """Log structured data with a clear start and end marker. @@ -50,10 +51,10 @@ def log_structured_data(start_tag: str, dump_string: str): dump_string: The data to be logged """ lines = dump_string.splitlines() - logging.info(f'{start_tag}BEGIN ({len(lines)} lines)====') + logger.info(f'{start_tag}BEGIN ({len(lines)} lines)====') for line in lines: - logging.info(f'{start_tag}{line}') - logging.info(f'{start_tag}END ====') + logger.info(f'{start_tag}{line}') + logger.info(f'{start_tag}END ====') @dataclass @@ -302,18 +303,18 @@ def teardown_class(self): """ # Check if we have problems and device attributes are available if len(self.problems) > 0: - logging.info("BasicCompositionTests: Problems detected - attempting device attribute dump") + logger.info("BasicCompositionTests: Problems detected - attempting device attribute dump") try: if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: - logging.info("Device attribute data available - generating dump") + logger.info("Device attribute data available - generating dump") _, txt_str = self.dump_wildcard(None) # Only dump the text format - it's more readable for debugging log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) else: - logging.info("No device attribute data available (endpoints_tlv not populated)") + logger.info("No device attribute data available (endpoints_tlv not populated)") except Exception as e: # Don't let logging errors interfere with the original test failure - logging.warning(f"Failed to generate device attribute dump: {e}") + logger.warning(f"Failed to generate device attribute dump: {e}") # Call the parent teardown_class method to handle normal teardown and problem logging super().teardown_class() From b149ff69477a137c156974bf29597ae4b0f6ede5 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 24 Sep 2025 21:28:42 +0000 Subject: [PATCH 11/18] Restyled by autopep8 --- .../matter/testing/basic_composition.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index ed39aa443af2fe..5820759999c740 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -40,6 +40,7 @@ LOGGER = logging.getLogger(__name__) + def log_structured_data(start_tag: str, dump_string: str): """Log structured data with a clear start and end marker. @@ -56,6 +57,7 @@ def log_structured_data(start_tag: str, dump_string: str): LOGGER.info(f'{start_tag}{line}') LOGGER.info(f'{start_tag}END ====') + @dataclass class ArlData: have_arl: bool From 69924046ee682682c78fe8db955ee074e800c72c Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Tue, 30 Sep 2025 17:38:56 -0700 Subject: [PATCH 12/18] Move device attribute dumping from BasicCompositionTests to MatterBaseTest for universal test debugging --- .../matter/testing/basic_composition.py | 23 --------- .../matter/testing/matter_testing.py | 48 ++++++++++++++++++- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index 5820759999c740..db4d0422820420 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -296,26 +296,3 @@ def build_spec_xmls(self): self.xml_device_types, problems = build_xml_device_types(dm) self.problems.extend(problems) - def teardown_class(self): - """Override teardown_class to dump device attribute data when problems are found. - - This ensures that whenever any test inheriting from BasicCompositionTests has problems, - we automatically get the device attribute dump for debugging purposes. - """ - # Check if we have problems and device attributes are available - if len(self.problems) > 0: - LOGGER.info("BasicCompositionTests: Problems detected - attempting device attribute dump") - try: - if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: - LOGGER.info("Device attribute data available - generating dump") - _, txt_str = self.dump_wildcard(None) - # Only dump the text format - it's more readable for debugging - log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) - else: - LOGGER.info("No device attribute data available (endpoints_tlv not populated)") - except Exception as e: - # Don't let logging errors interfere with the original test failure - LOGGER.warning(f"Failed to generate device attribute dump: {e}") - - # Call the parent teardown_class method to handle normal teardown and problem logging - super().teardown_class() diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py index a9bed0941cabe9..3e97af6d38c2f5 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py @@ -206,7 +206,7 @@ def setup_class(self): self.stored_global_wildcard = None def teardown_class(self): - """Final teardown after all tests: log all problems. + """Final teardown after all tests: log all problems and dump device attributes if available. Test authors may overwrite this method in the derived class to perform teardown that is common for all tests This function is called only once per class. To perform teardown after each test, use teardown_test. Test authors that implement steps in this function need to be careful of step handling if there is @@ -222,8 +222,54 @@ def teardown_class(self): for problem in self.problems: LOGGER.info(str(problem)) LOGGER.info("###########################################################") + + # Attempt to dump device attribute data for debugging when problems are found + self._dump_device_attributes_on_failure() super().teardown_class() + def _dump_device_attributes_on_failure(self): + """ + Dump device attribute data when problems are found for debugging purposes. + + This method attempts to generate a device attribute dump if the test has + collected endpoint data. It's designed to be safe and not interfere with + the original test failure reporting. + """ + try: + # Check if we have endpoints_tlv data (from BasicCompositionTests or similar) + if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: + LOGGER.info("MatterBaseTest: Problems detected - generating device attribute dump") + + # Check if we have the dump_wildcard method (from BasicCompositionTests) + if hasattr(self, 'dump_wildcard'): + LOGGER.info("Device attribute data available - generating dump") + _, txt_str = self.dump_wildcard(None) + # Only dump the text format - it's more readable for debugging + self._log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) + else: + LOGGER.info("dump_wildcard method not available - skipping device attribute dump") + else: + LOGGER.debug("No device attribute data available (endpoints_tlv not populated)") + except Exception as e: + # Don't let logging errors interfere with the original test failure + LOGGER.warning(f"Failed to generate device attribute dump: {e}") + + def _log_structured_data(self, start_tag: str, dump_string: str): + """Log structured data with a clear start and end marker. + + This function is used to output device attribute dumps and other structured + data to logs in a format that can be easily extracted for debugging. + + Args: + start_tag: A prefix tag to identify the type of data being logged + dump_string: The data to be logged + """ + lines = dump_string.splitlines() + LOGGER.info(f'{start_tag}BEGIN ({len(lines)} lines)====') + for line in lines: + LOGGER.info(f'{start_tag}{line}') + LOGGER.info(f'{start_tag}END ====') + def setup_test(self): """Set up for each individual test execution. From fee26490455e30f3baf994424e0e7effe78f7a22 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 1 Oct 2025 00:39:40 +0000 Subject: [PATCH 13/18] Restyled by autopep8 --- src/python_testing/TC_DeviceBasicComposition.py | 12 ++++++++---- .../matter/testing/basic_composition.py | 1 - .../matter/testing/matter_testing.py | 6 +++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index 548a53acc89db2..95b5c4b0460a00 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -603,12 +603,14 @@ class RequiredMandatoryAttribute: attribute_id=manufacturer_value) if suffix > attribute_standard_range_max and suffix < global_range_min: self.record_error(self.get_test_name(), location=location, - problem=f"Manufacturer attribute in undefined range {manufacturer_value} in cluster {cluster_id}", + problem=f"Manufacturer attribute in undefined range { + manufacturer_value} in cluster {cluster_id}", spec_location=f"Cluster {cluster_id}") success = False elif suffix >= global_range_min: self.record_error(self.get_test_name(), location=location, - problem=f"Manufacturer attribute in global range {manufacturer_value} in cluster {cluster_id}", + problem=f"Manufacturer attribute in global range { + manufacturer_value} in cluster {cluster_id}", spec_location=f"Cluster {cluster_id}") success = False @@ -929,7 +931,8 @@ def record_problems(problems): for ep, problem in problems.items(): location = AttributePathLocation(endpoint_id=ep, cluster_id=Clusters.Descriptor.id, attribute_id=Clusters.Descriptor.Attributes.TagList.attribute_id) - msg = f'problem on ep {ep}: missing feature = {problem.missing_feature}, missing attribute = {problem.missing_attribute}, duplicates = {problem.duplicates}, same_tags = {problem.same_tag}' + msg = f'problem on ep {ep}: missing feature = {problem.missing_feature}, missing attribute = { + problem.missing_attribute}, duplicates = {problem.duplicates}, same_tags = {problem.same_tag}' self.record_error(self.get_test_name(), location=location, problem=msg, spec_location="Descriptor TagList") record_problems(problems) @@ -1208,7 +1211,8 @@ async def test_TC_DESC_2_1(self): self.record_error( self.get_test_name(), location=location, - problem=f"EndpointUniqueId attribute length is {len(value)} bytes which exceeds the maximum allowed 32 bytes", + problem=f"EndpointUniqueId attribute length is { + len(value)} bytes which exceeds the maximum allowed 32 bytes", spec_location="EndpointUniqueId attribute" ) self.fail_current_test( diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index db4d0422820420..d0fde4b900338a 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -295,4 +295,3 @@ def build_spec_xmls(self): self.xml_clusters, self.problems = build_xml_clusters(dm) self.xml_device_types, problems = build_xml_device_types(dm) self.problems.extend(problems) - diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py index 3e97af6d38c2f5..75e4628172e3f0 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py @@ -222,7 +222,7 @@ def teardown_class(self): for problem in self.problems: LOGGER.info(str(problem)) LOGGER.info("###########################################################") - + # Attempt to dump device attribute data for debugging when problems are found self._dump_device_attributes_on_failure() super().teardown_class() @@ -230,7 +230,7 @@ def teardown_class(self): def _dump_device_attributes_on_failure(self): """ Dump device attribute data when problems are found for debugging purposes. - + This method attempts to generate a device attribute dump if the test has collected endpoint data. It's designed to be safe and not interfere with the original test failure reporting. @@ -239,7 +239,7 @@ def _dump_device_attributes_on_failure(self): # Check if we have endpoints_tlv data (from BasicCompositionTests or similar) if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: LOGGER.info("MatterBaseTest: Problems detected - generating device attribute dump") - + # Check if we have the dump_wildcard method (from BasicCompositionTests) if hasattr(self, 'dump_wildcard'): LOGGER.info("Device attribute data available - generating dump") From 530ac3a1d0d4828aa0e5091cb3f09762f2e53a62 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Thu, 16 Oct 2025 15:42:33 -0700 Subject: [PATCH 14/18] Fix Python 3.11 compatibility and improve test failure User Experience: - Fix f-strings in TC_DeviceBasicComposition.py for Python < 3.12 - Move device dump before problem logging for better failure visibility - Remove noisy log messages from device dump on failure - Use specific exceptions instead of catching all exceptions so that it isnt blanketed. --- .../TC_DeviceBasicComposition.py | 12 ++++-------- .../matter/testing/matter_testing.py | 19 ++++++------------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index e4c54a6b7d606a..d412d1c8dd8510 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -604,14 +604,12 @@ class RequiredMandatoryAttribute: attribute_id=manufacturer_value) if suffix > attribute_standard_range_max and suffix < global_range_min: self.record_error(self.get_test_name(), location=location, - problem=f"Manufacturer attribute in undefined range { - manufacturer_value} in cluster {cluster_id}", + problem=f"Manufacturer attribute in undefined range {manufacturer_value} in cluster {cluster_id}", spec_location=f"Cluster {cluster_id}") success = False elif suffix >= global_range_min: self.record_error(self.get_test_name(), location=location, - problem=f"Manufacturer attribute in global range { - manufacturer_value} in cluster {cluster_id}", + problem=f"Manufacturer attribute in global range {manufacturer_value} in cluster {cluster_id}", spec_location=f"Cluster {cluster_id}") success = False @@ -932,8 +930,7 @@ def record_problems(problems): for ep, problem in problems.items(): location = AttributePathLocation(endpoint_id=ep, cluster_id=Clusters.Descriptor.id, attribute_id=Clusters.Descriptor.Attributes.TagList.attribute_id) - msg = f'problem on ep {ep}: missing feature = {problem.missing_feature}, missing attribute = { - problem.missing_attribute}, duplicates = {problem.duplicates}, same_tags = {problem.same_tag}' + msg = f'problem on ep {ep}: missing feature = {problem.missing_feature}, missing attribute = {problem.missing_attribute}, duplicates = {problem.duplicates}, same_tags = {problem.same_tag}' self.record_error(self.get_test_name(), location=location, problem=msg, spec_location="Descriptor TagList") record_problems(problems) @@ -1212,8 +1209,7 @@ async def test_TC_DESC_2_1(self): self.record_error( self.get_test_name(), location=location, - problem=f"EndpointUniqueId attribute length is { - len(value)} bytes which exceeds the maximum allowed 32 bytes", + problem=f"EndpointUniqueId attribute length is {len(value)} bytes which exceeds the maximum allowed 32 bytes", spec_location="EndpointUniqueId attribute" ) self.fail_current_test( diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py index 75e4628172e3f0..e9e954ada27da6 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py @@ -216,15 +216,15 @@ def teardown_class(self): """ if len(self.problems) > 0: + # Attempt to dump device attribute data for debugging when problems are found + self._dump_device_attributes_on_failure() + LOGGER.info("###########################################################") LOGGER.info("Problems found:") LOGGER.info("===============") for problem in self.problems: LOGGER.info(str(problem)) LOGGER.info("###########################################################") - - # Attempt to dump device attribute data for debugging when problems are found - self._dump_device_attributes_on_failure() super().teardown_class() def _dump_device_attributes_on_failure(self): @@ -238,21 +238,14 @@ def _dump_device_attributes_on_failure(self): try: # Check if we have endpoints_tlv data (from BasicCompositionTests or similar) if hasattr(self, 'endpoints_tlv') and self.endpoints_tlv: - LOGGER.info("MatterBaseTest: Problems detected - generating device attribute dump") - # Check if we have the dump_wildcard method (from BasicCompositionTests) if hasattr(self, 'dump_wildcard'): - LOGGER.info("Device attribute data available - generating dump") _, txt_str = self.dump_wildcard(None) # Only dump the text format - it's more readable for debugging self._log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) - else: - LOGGER.info("dump_wildcard method not available - skipping device attribute dump") - else: - LOGGER.debug("No device attribute data available (endpoints_tlv not populated)") - except Exception as e: - # Don't let logging errors interfere with the original test failure - LOGGER.warning(f"Failed to generate device attribute dump: {e}") + except (AttributeError, KeyError, ValueError, TypeError): + # Don't let data access or serialization errors interfere with the original test failure + pass def _log_structured_data(self, start_tag: str, dump_string: str): """Log structured data with a clear start and end marker. From f2fc773695406b5373da80e00eca936b80b93048 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Thu, 6 Nov 2025 10:11:10 -0800 Subject: [PATCH 15/18] removed duplicate log_structured_data function from basic_composition module, now inheriting it directly from matter_testing in test_TC_IDM_12_1 test --- src/python_testing/TC_DeviceBasicComposition.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index d412d1c8dd8510..a3f8db05765753 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -189,7 +189,7 @@ from matter.clusters.Types import Nullable from matter.exceptions import ChipStackError from matter.interaction_model import InteractionModelError, Status -from matter.testing.basic_composition import BasicCompositionTests, log_structured_data +from matter.testing.basic_composition import BasicCompositionTests from matter.testing.global_attribute_ids import (AttributeIdType, ClusterIdType, CommandIdType, GlobalAttributeIds, attribute_id_type, cluster_id_type, command_id_type) from matter.testing.matter_testing import MatterBaseTest, TestStep, async_test_body, default_matter_test_main @@ -966,8 +966,8 @@ def test_TC_IDM_12_1(self): json_str, txt_str = self.dump_wildcard(dump_device_composition_path) # Structured dump so we can pull these back out of the logs - log_structured_data('==== json: ', json_str) - log_structured_data('==== txt: ', txt_str) + self.log_structured_data('==== json: ', json_str) + self.log_structured_data('==== txt: ', txt_str) @async_test_body async def test_TC_DESC_2_1(self): From 7c93c8ce8fd5ca801c5016e0bdfc623270678e0e Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Thu, 6 Nov 2025 10:13:20 -0800 Subject: [PATCH 16/18] removed duplicate log_structured_data function from basic_composition module, now inheriting it directly from matter_testing in test_TC_IDM_12_1 test in TC_DeviceBasicCompositon test module --- .../matter/testing/basic_composition.py | 18 ------------------ .../matter/testing/matter_testing.py | 4 ++-- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index 57200a80d9f11b..c34cf73cd9ddff 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -40,24 +40,6 @@ LOGGER = logging.getLogger(__name__) - -def log_structured_data(start_tag: str, dump_string: str): - """Log structured data with a clear start and end marker. - - This function is used to output device attribute dumps and other structured - data to logs in a format that can be easily extracted for debugging. - - Args: - start_tag: A prefix tag to identify the type of data being logged - dump_string: The data to be logged - """ - lines = dump_string.splitlines() - LOGGER.info(f'{start_tag}BEGIN ({len(lines)} lines)====') - for line in lines: - LOGGER.info(f'{start_tag}{line}') - LOGGER.info(f'{start_tag}END ====') - - @dataclass class ArlData: have_arl: bool diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py index e9e954ada27da6..ca1cccd5919ff6 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py @@ -242,12 +242,12 @@ def _dump_device_attributes_on_failure(self): if hasattr(self, 'dump_wildcard'): _, txt_str = self.dump_wildcard(None) # Only dump the text format - it's more readable for debugging - self._log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) + self.log_structured_data('==== FAILURE_DUMP_txt: ', txt_str) except (AttributeError, KeyError, ValueError, TypeError): # Don't let data access or serialization errors interfere with the original test failure pass - def _log_structured_data(self, start_tag: str, dump_string: str): + def log_structured_data(self, start_tag: str, dump_string: str): """Log structured data with a clear start and end marker. This function is used to output device attribute dumps and other structured From 914426fd2880d623bc2992bf446e2a0aca1560f1 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 6 Nov 2025 18:14:10 +0000 Subject: [PATCH 17/18] Restyled by autopep8 --- .../matter/testing/basic_composition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py index c34cf73cd9ddff..9479bca3b3934f 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/basic_composition.py @@ -40,6 +40,7 @@ LOGGER = logging.getLogger(__name__) + @dataclass class ArlData: have_arl: bool From 127cb15ba210e26c6e4a0a0415ae93741bf276a9 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Thu, 6 Nov 2025 10:29:39 -0800 Subject: [PATCH 18/18] Resolving linting error --- .../matter/testing/matter_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py index ca1cccd5919ff6..2a0b8100c94b89 100644 --- a/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/matter/testing/matter_testing.py @@ -250,7 +250,7 @@ def _dump_device_attributes_on_failure(self): def log_structured_data(self, start_tag: str, dump_string: str): """Log structured data with a clear start and end marker. - This function is used to output device attribute dumps and other structured + This function is used to output device attribute dumps and other structured data to logs in a format that can be easily extracted for debugging. Args: