draft

.github/workflows/run-end-to-end.yml

@@ -415,3 +415,14 @@ jobs:
       env:
         FP_API_KEY: ${{ secrets.FP_API_KEY }}
         FP_IMPORT_URL: ${{ secrets.FP_IMPORT_URL }}
+
+    - name: Print Compliance Reports
+      if: always()
+      run: |
+        for folder in logs*/ ; do
+          if [ -f "${folder}compliance.json" ]; then
+            echo "=== Compliance Report for ${folder} ==="
+            cat "${folder}compliance.json"
+            echo -e "\n"
+          fi
+        done

conftest.py

@@ -500,6 +500,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
             )
 
             export_feature_parity_dashboard(session, data)
+            merge_compliance_reports(session, data)
         except Exception:
             logger.exception("Fail to export export reports", exc_info=True)
 
@@ -540,6 +541,41 @@ def convert_test_to_feature_parity_model(test: dict) -> dict | None:
     return result if -1 not in result["features"] else None
 
 
+def merge_compliance_reports(session: pytest.Session, data: dict) -> None:
+    """Merge all individual compliance reports into a single aggregated report."""
+    compliance_dir = Path(context.scenario.host_log_folder) / "compliance_reports"
+    if not compliance_dir.exists():
+        return
+
+    merged_report = {
+        "runUrl": session.config.option.report_run_url or "https://github.com/DataDog/system-tests",
+        "runDate": data["created"],
+        "environment": session.config.option.report_environment or "local",
+        "testSource": "systemtests",
+        "language": context.library.name,
+        "variant": context.weblog_variant,
+        "reports": [],
+    }
+
+    # Collect all individual reports
+    for report_file in compliance_dir.glob("*.json"):
+        try:
+            with open(report_file, "r") as f:
+                report = json.load(f)
+                report.pop("language", None)
+                merged_report["reports"].append(report)
+        except Exception as e:
+            logger.error(f"Failed to read compliance report {report_file}: {e}")
+
+    # Sort reports by framework name for consistency
+    merged_report["reports"].sort(key=lambda x: x.get("integration", ""))
+
+    # Write the merged report
+    output_path = Path(context.scenario.host_log_folder) / "compliance.json"
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(merged_report, f, indent=2)
+
+
 ## Fixtures corners
 @pytest.fixture(scope="session", name="session")
 def fixture_session(request: pytest.FixtureRequest) -> pytest.Session:

manifests/cpp_httpd.yml

@@ -6,6 +6,9 @@ tests/:
   appsec/: irrelevant (ASM is not implemented in C++)
   debugger/: irrelevant
   integrations/:
+    compliance/:
+      test_databases.py: missing_feature
+      test_web_frameworks.py: missing_feature
     crossed_integrations/: missing_feature (Endpoint not implemented)
     test_db_integrations_sql.py: missing_feature
     test_dbm.py: missing_feature

manifests/cpp_nginx.yml

@@ -68,6 +68,8 @@ tests/:
     test_debugger_telemetry.py:
       Test_Debugger_Telemetry: irrelevant
   integrations/:
+    compliance/:
+      test_databases.py: missing_feature
     crossed_integrations/:
       test_kafka.py:
         Test_Kafka: missing_feature

manifests/dotnet.yml

@@ -445,6 +445,8 @@ tests/:
     test_debugger_telemetry.py:
       Test_Debugger_Telemetry: v2.53.0
   integrations/:
+    compliance/:
+      test_databases.py: missing_feature
     crossed_integrations/:
       test_kafka.py:
         Test_Kafka: v2.0.0

manifests/golang.yml

@@ -529,6 +529,8 @@ tests/:
     test_debugger_telemetry.py:
       Test_Debugger_Telemetry: missing_feature
   integrations/:
+    compliance/:
+      test_databases.py: missing_feature
     crossed_integrations/:
       test_kafka.py:
         Test_Kafka:

manifests/java.yml

@@ -1825,6 +1825,17 @@ tests/:
         spring-boot-undertow: v1.38.0
         uds-spring-boot: v1.38.0
   integrations/:
+    compliance/:
+      test_databases.py:
+        Test_MsSql:
+          '*': missing_feature
+          spring-boot: v1.12.0
+        Test_MySql:
+          '*': missing_feature
+          spring-boot: v1.12.0
+        Test_Postgres:
+          '*': missing_feature
+          spring-boot: v1.12.0
     crossed_integrations/:
       test_kafka.py:
         Test_Kafka:

manifests/nodejs.yml

@@ -861,6 +861,20 @@ tests/:
         express4-typescript: v5.39.0
         uds-express4: v5.39.0
   integrations/:
+    compliance/:
+      test_databases.py:
+        Test_MsSql:
+          '*': missing_feature
+          express4: v1.0.0
+          express5: v1.0.0
+        Test_MySql:
+          '*': missing_feature
+          express4: v1.0.0
+          express5: v1.0.0
+        Test_Postgres:
+          '*': missing_feature
+          express4: v1.0.0
+          express5: v1.0.0
     crossed_integrations/:
       test_kafka.py:
         Test_Kafka:

manifests/php.yml

@@ -437,6 +437,9 @@ tests/:
     test_debugger_telemetry.py:
       Test_Debugger_Telemetry: missing_feature
   integrations/:
+    compliance/:
+      test_databases.py: missing_feature
+      test_web_frameworks.py: missing_feature
     crossed_integrations/:
       test_kafka.py:
         Test_Kafka: missing_feature

manifests/python.yml

@@ -805,6 +805,17 @@ tests/:
     test_debugger_telemetry.py:
       Test_Debugger_Telemetry: v2.11.0
   integrations/:
+    compliance/:
+      test_databases.py:
+        Test_MsSql:
+          '*': missing_feature
+          flask-poc: v1.18.3
+        Test_MySql:
+          '*': missing_feature
+          flask-poc: v1.18.3
+        Test_Postgres:
+          '*': missing_feature
+          flask-poc: v1.18.3
     crossed_integrations/:
       test_kafka.py:
         Test_Kafka:

manifests/ruby.yml

@@ -627,6 +627,8 @@ tests/:
         "*": irrelevant
         rails70: missing_feature (feature not implemented)
   integrations/:
+    compliance/:
+      test_databases.py: missing_feature
     crossed_integrations/:
       test_kafka.py:
         Test_Kafka:

tests/integrations/compliance/README.md

@@ -0,0 +1,45 @@
+# Integration Compliance Tests
+
+This directory contains compliance tests that verify if integration spans (e.g., for web frameworks, databases) include a required set of attributes. These tests are meant to ensure consistent span formatting across libraries.
+
+## Structure
+
+- `test_<category>.py`: The actual compliance test for an integration category (e.g., `test_web_frameworks.py`, `test_databases.py`).
+- `utils.py`: Contains shared utilities for schema loading, validation, and report generation.
+- `schemas/`: YAML schema files specifying required attributes for spans. See [schemas/README.md](schemas/README.md) for details on schema format and best practices.
+
+## How It Works
+
+1. Each test loads a schema that defines required attributes for a specific integration type
+2. The schema is merged with a generic schema that contains common requirements
+3. Tests validate spans against the merged schema
+4. Results are written to individual JSON reports in the `compliance_reports` directory
+5. At the end of the test run, all reports are merged into a single `compliance.json` file
+
+## Adding a New Category
+
+To test a new integration category:
+
+1. Add a `schemas/<category>.yaml` file with category-specific keys. See [schemas/README.md](schemas/README.md) for detailed schema format and best practices.
+
+2. Create a `test_<category>.py` using the shared utilities:
+```python
+from utils import interfaces, weblog, context
+from .utils import assert_required_keys, generate_compliance_report, load_schema
+
+class Test_NewCategory:
+    def setup_attributes(self):
+        # Ensure the weblog simulates an appropriate request to trigger a representative span
+        pass
+
+    def test_attributes(self):
+        schema = load_schema("<category>")
+        span = interfaces.library.get_root_span(self.r)
+        missing, deprecated = assert_required_keys(span, schema)
+        generate_compliance_report(
+            category="<category>",
+            name="<integration_name>",
+            missing=missing,
+            deprecated=deprecated
+        )
+```

tests/integrations/compliance/schemas/README.md

@@ -0,0 +1,45 @@
+# Compliance Schemas
+
+Each YAML file in this directory defines the required attributes for spans in a particular integration category.
+
+## Files
+
+- `generic.yaml`: Contains attributes required for **all** spans, regardless of integration type.
+- `<category>.yaml`: Contains attributes specific to that integration category (e.g., `web_frameworks.yaml`, `databases.yaml`).
+
+## Schema Structure
+
+Each schema can define the following sections:
+
+```yaml
+span_attributes:
+  mandatory:
+    - <required_attribute1>
+    - <required_attribute2>
+  best_effort:
+    - <optional_attribute1>
+    - <optional_attribute2>
+
+# Deprecated attribute aliases
+deprecated_aliases:
+  <current_attribute>:
+    - <deprecated_name1>
+    - <deprecated_name2>
+```
+
+## Attribute Types
+
+- `mandatory`: Attributes that must be present for the span to be considered compliant
+- `best_effort`: Optional attributes that are recommended but not required. These are currently no-op in the test.
+- `deprecated_aliases`: Alternative attribute names that are considered deprecated but still accepted
+
+## Best Practices
+
+1. Keep schemas focused and minimal:
+   - Put common attributes in `generic.yaml`
+   - Put integration-specific attributes in category files
+   - Avoid duplicating keys between files
+
+2. Document deprecated attributes:
+   - Use `deprecated_aliases` to maintain backward compatibility
+   - List all known alternative names for an attribute

tests/integrations/compliance/schemas/databases.yaml

@@ -0,0 +1,13 @@
+span_attributes:
+  mandatory:
+    - meta.db.system
+  best_effort:
+    - meta.db.name
+    - meta.db.user
+    - meta.out.host
+    - meta.out.port
+    - metrics.db.row_count
+
+deprecated_aliases:
+  meta.db.system:
+    - meta.db.type

tests/integrations/compliance/schemas/generic.yaml

@@ -0,0 +1,15 @@
+root_span_attributes:
+  mandatory:
+    - meta.language
+    - meta.runtime-id
+    - metrics.process_id
+
+span_attributes:
+  mandatory:
+    - duration
+    - meta.span.kind
+    - name
+    - resource
+    - service
+    - start
+    - type

tests/integrations/compliance/schemas/web_frameworks.yaml

@@ -0,0 +1,5 @@
+span_attributes:
+  mandatory:
+    - meta.http.method
+    - meta.http.status_code
+    - meta.http.url

tests/integrations/compliance/test_databases.py

@@ -0,0 +1,59 @@
+from utils import logger, scenarios, features
+from .utils import assert_required_keys, generate_compliance_report, load_schema
+
+from tests.integrations.utils import BaseDbIntegrationsTestClass
+
+
+class _BaseDatabaseComplianceTest(BaseDbIntegrationsTestClass):
+    def get_spans(self, excluded_operations=(), operations=None):
+        for db_operation, request in self.get_requests(excluded_operations, operations=operations):
+            logger.debug(f"Validating {self.db_service}/{db_operation}")
+            yield db_operation, self.get_span_from_tracer(request)
+            yield db_operation, self.get_span_from_agent(request)
+
+    def test_attributes(self):
+        schema = load_schema("databases")
+
+        all_missing = []
+        all_deprecated = []
+
+        for _db_operation, span in self.get_spans(excluded_operations=["procedure", "select_error"]):
+            # print(f"Span found for {_db_operation}: {json.dumps(span, indent=2)}")
+
+            missing, deprecated = assert_required_keys(span, schema)
+            all_missing.extend(missing)
+            all_deprecated.extend(deprecated)
+
+        all_missing = sorted(set(all_missing))
+        all_deprecated = sorted(set(all_deprecated))
+
+        generate_compliance_report(
+            category="database", name=self.db_service, missing=all_missing, deprecated=all_deprecated
+        )
+
+        if all_missing:
+            raise AssertionError(f"Missing required attributes: {all_missing}")
+
+
+@features.not_reported
+@scenarios.integrations
+class Test_MsSql(_BaseDatabaseComplianceTest):
+    """MSSQL compliance tests."""
+
+    db_service = "mssql"
+
+
+@features.not_reported
+@scenarios.integrations
+class Test_MySql(_BaseDatabaseComplianceTest):
+    """MySQL compliance tests."""
+
+    db_service = "mysql"
+
+
+@features.not_reported
+@scenarios.integrations
+class Test_Postgres(_BaseDatabaseComplianceTest):
+    """Postgres compliance tests."""
+
+    db_service = "postgresql"

tests/integrations/compliance/test_web_frameworks.py

@@ -0,0 +1,21 @@
+from utils import interfaces, weblog, context, features
+from .utils import assert_required_keys, generate_compliance_report, load_schema
+
+
+@features.not_reported
+class Test_WebFrameworks:
+    def setup_simple(self):
+        self.r = weblog.get("/")
+
+    def test_simple(self):
+        schema = load_schema("web_frameworks")
+        root_span = interfaces.library.get_root_span(self.r)
+
+        missing, deprecated = assert_required_keys(root_span, schema)
+
+        generate_compliance_report(
+            category="web_framework", name=context.weblog_variant, missing=missing, deprecated=deprecated
+        )
+
+        if missing:
+            raise AssertionError(f"Missing required attributes: {missing}")