Adding upgrade rule mechanism in ZAP for SDKs (#1589)

- Users can now include an upgrade rules json file in the zcl.json file which has a list of all the upgrade rules that may be needed when a user updates an existing app from one version of the GSDK to another.
- The upgrade rules json file lists a set of upgrade rules scripts with their priority as can be seen in upgrade-rules.json file. The scripts are run in order of priority with lower number signifying higher priority
- The upgrade rules return an object including a message and status which helps determine if an upgrade rules was actually executed or not. These results can be output into a yaml file if required for post analysis on which upgrade rules ran on a certain .zap file once the GSDK was updated.
- Adding unit tests to update cluster revision attribute of all clusters
- Handling the multi protocol use case and adding unit tests that make sure that only the appropriate endpoints are updated when upgrade rules json files from both zigbee and matter exist
- Adding unit tests for matter only .zap file
- Adding documentation on how to add upgrade rules to your SDK
- Adding the update keys available to the context in postLoad
- Add unit tests to check the order in which the upgrade rules were run
- Github: ZAP#1020

Author: brdandu

README.md

@@ -40,6 +40,7 @@ This software is licensed under [Apache 2.0 license](LICENSE.txt).
 - [ZAP Template Helpers](docs/helpers.md)
 - [ZAP External Template Helpers](docs/external-helpers.md)
 - [ZAP file Extensions](docs/zap-file-extensions.md)
+- [Upgrading ZAP files with GSDK upgrades](docs/zap-file-upgrade.md)
 - [FAQ/Developer dependencies](docs/faq.md)
 - [Release instructions](docs/release.md)
 - [Development Instructions](docs/development-instructions.md)

docs/zap-file-upgrade.md

@@ -0,0 +1,91 @@
+# Upgrading ZAP Files with SDK upgrades
+
+## Overview
+
+Upgrading ZAP files is a critical step when transitioning to newer versions of the SDK. This process ensures compatibility with the latest features, bug fixes, and improvements provided by the SDK. By following the upgrade guidelines, you can maintain the integrity of your ZAP configuration files and avoid potential issues during development or deployment. This document outlines the steps and best practices for performing ZAP file upgrades effectively.
+
+## How to upgrade the ZAP File?
+
+Run the following command to update your .zap file
+
+```bash
+"${Path to ZAP executable} upgrade  --results ${path to .yaml file to see results of upgrade} -d ${directory containing ZAP files} --zcl ${path to zcl json file} --generationTemplate ${path to templates json file} --noLoadingFailure"
+```
+
+## How to add upgrade rules for .zap files through your SDK
+
+### - Create an `upgrade-rules.json` file with the following information if one already does not exist
+
+```json
+{
+  "version": 1,
+  "description": "Upgrade Rules for Zigbee .zap files",
+  "category": "matter",
+  "upgradeRuleScripts": [
+    {
+      "path": "../../test/resource/test-matter-attribute-default-value-update.js",
+      "priority": 101
+    },
+    {
+      "path": "../../test/resource/test-attribute-default-value-update.js",
+      "priority": 100
+    }
+  ]
+}
+```
+
+**category**: Determines that these upgrade rules need to run for matter.
+
+**upgradeRuleScripts**: List of upgrade rules to run on .zap files. Includes the relative path from upgrade-rules.json to the upgrade scripts written in Javascript. Priority determines the order of execution for the upgrade rules. The scripts are run in order of priority with lower number signifying higher priority.
+
+### - Add relative path to the upgrade-rules.json from your zcl.json file
+
+```json
+"upgradeRules": "./upgrade-rules-matter.json"
+```
+
+### - Creating your own javascript upgrade rule
+
+Add a postLoad function as below with api and context as parameters. Api argument gives access to all APIs that can be used within the `postLoad` function. [Refer to the post-import API documentation](../src-electron/util/post-import-api.js) . Context gives the state of the Data-Model/ZCL with respect to the .zap file that will be passied along to the API calls.
+
+```javascript
+// Example upgrade rule to update default value of Level Control cluster attribute in Matter.
+async function postLoad(api, context) {
+  let resMsg = ''
+  let epts = await api.endpoints(context)
+  for (let i = 0; i < epts.length; i++) {
+    let clusters = await api.clusters(context, epts[i])
+    for (let j = 0; j < clusters.length; j++) {
+      if (clusters[j].code == '0x0008') {
+        let attributes = await api.attributes(context, epts[i], clusters[j])
+        for (let k = 0; k < attributes.length; k++) {
+          let attributeCode = parseInt(attributes[k].code)
+          let attributeValue = parseInt(attributes[k].defaultValue)
+          if (
+            attributeCode == 0 &&
+            (attributeValue == 0 || !attributeValue || attributeValue == 'null')
+          ) {
+            let params = [
+              {
+                key: context.updateKey.attributeDefault,
+                value: 10
+              }
+            ]
+            await api.updateAttribute(
+              context,
+              epts[i],
+              clusters[j],
+              attributes[k],
+              params
+            )
+            resMsg += `Current Value attribute's default value updated to 10 for Level Control cluster on endpoint ${epts[i].endpointIdentifier} ${epts[i].category}\n`
+          }
+        }
+      }
+    }
+  }
+  return { message: resMsg, status: 'automatic' } // Status can be 'nothing', 'automatic', 'user_verification', 'impossible'.
+}
+
+exports.postLoad = postLoad
+```

src-electron/db/query-endpoint.js

@@ -31,9 +31,39 @@ const dbEnum = require('../../src-shared/db-enum.js')
  *
  * @param {*} db
  * @param {*} sessionId
+ * @param {*} category
  * @returns Promise resolving into all endpoints.
  */
-async function selectAllEndpoints(db, sessionId) {
+async function selectAllEndpoints(db, sessionId, category = null) {
+  let categorySqlJoinString = category
+    ? `
+LEFT JOIN
+  ENDPOINT_TYPE
+ON
+  E1.ENDPOINT_TYPE_REF = ENDPOINT_TYPE.ENDPOINT_TYPE_ID
+LEFT JOIN
+  ENDPOINT_TYPE_DEVICE
+ON
+  ENDPOINT_TYPE_DEVICE.ENDPOINT_TYPE_REF = ENDPOINT_TYPE.ENDPOINT_TYPE_ID
+LEFT JOIN
+  DEVICE_TYPE
+ON
+  ENDPOINT_TYPE_DEVICE.DEVICE_TYPE_REF = DEVICE_TYPE.DEVICE_TYPE_ID
+LEFT JOIN
+  PACKAGE
+ON
+  PACKAGE.PACKAGE_ID = DEVICE_TYPE.PACKAGE_REF`
+    : ``
+
+  let categorySqlSelectString = category
+    ? `,
+  PACKAGE.CATEGORY`
+    : ``
+
+  let categorySqlWhereString = category
+    ? `AND
+  PACKAGE.CATEGORY = '${category}'`
+    : ``
   let rows = await dbApi.dbAll(
     db,
     `
@@ -46,18 +76,23 @@ SELECT
   E1.NETWORK_IDENTIFIER,
   E2.ENDPOINT_ID AS PARENT_ENDPOINT_REF,
   E2.ENDPOINT_IDENTIFIER AS PARENT_ENDPOINT_IDENTIFIER
+  ${categorySqlSelectString}
 FROM
   ENDPOINT AS E1
 LEFT JOIN
   ENDPOINT AS E2
 ON
   E2.ENDPOINT_ID = E1.PARENT_ENDPOINT_REF
-WHERE E1.SESSION_REF = ?
-ORDER BY E1.ENDPOINT_IDENTIFIER
+${categorySqlJoinString}
+WHERE
+  E1.SESSION_REF = ?
+${categorySqlWhereString}
+ORDER BY
+  E1.ENDPOINT_IDENTIFIER
     `,
     [sessionId]
   )
-  return rows.map(dbMapping.map.endpoint)
+  return rows.map(dbMapping.map.endpointExtended)
 }
 
 /**
@@ -197,6 +232,7 @@ ORDER BY C.CODE
   return rows.map((row) => {
     return {
       clusterId: row['CLUSTER_ID'],
+      id: row['CLUSTER_ID'],
       endpointTypeId: row['ENDPOINT_TYPE_REF'],
       endpointTypeClusterId: row['ENDPOINT_TYPE_CLUSTER_ID'],
       hexCode: '0x' + bin.int16ToHex(row['CODE']),

src-electron/importexport/import.js

@@ -23,6 +23,8 @@
  */
 
 const fsp = require('fs').promises
+const fs = require('fs')
+const path = require('path')
 const importIsc = require('./import-isc.js')
 const importJson = require('./import-json.js')
 const dbApi = require('../db/db-api.js')
@@ -66,19 +68,16 @@ async function readDataFromFile(filePath, defaultZclMetafile) {
  *
  * @param {*} db
  * @param {*} sessionId
- * @param {*} scriptFile
+ * @param {*} scriptInfo
  * @returns Promise of function execution.
  */
-async function executePostImportScript(db, sessionId, scriptFile) {
+async function executePostImportScript(db, sessionId, scriptInfo) {
   let context = {
     db: db,
-    sessionId: sessionId
+    sessionId: sessionId,
+    script: scriptInfo
   }
-  return script.executeScriptFunction(
-    script.functions.postLoad,
-    context,
-    scriptFile
-  )
+  return script.executeScriptFunction(script.functions.postLoad, context)
 }
 
 /**
@@ -232,11 +231,32 @@ async function importDataFromFile(
       options.upgradeTemplatePackages
     )
     if (options.postImportScript != null) {
-      await executePostImportScript(
-        db,
-        loaderResult.sessionId,
-        options.postImportScript
-      )
+      await executePostImportScript(db, loaderResult.sessionId, {
+        path: path.resolve(options.postImportScript),
+        category: null // When running individual scripts no need for category
+      })
+    }
+    if (options.upgradeRuleScripts != null) {
+      const upgradeScripts = options.upgradeRuleScripts
+      let upgradeMessages = []
+      for (let i = 0; i < upgradeScripts.length; i++) {
+        let upgradeScript = upgradeScripts[i]
+        if (fs.existsSync(upgradeScript.path)) {
+          let upgradeMessage = await executePostImportScript(
+            db,
+            loaderResult.sessionId,
+            upgradeScript
+          )
+          if (upgradeMessage) {
+            upgradeMessages.push(upgradeMessage)
+          }
+        } else {
+          throw new Error(
+            `Post import script path does not exist: ${upgradeScript.path}`
+          )
+        }
+      }
+      loaderResult.upgradeMessages = upgradeMessages
     }
     return loaderResult
   } finally {

src-electron/main-process/startup.js

@@ -312,13 +312,57 @@ async function upgradeZapFile(argv, options) {
       dbEnum.packageType.genTemplatesJson
     )
 
+    let upgradeRules = []
+    // If more than one upgrade package is present then it is a multiprotocol
+    // application so upgrade rules should be added to the corresponding endpoints.
+    let isMultiProtocol = upgradeZclPackages.length > 1
+    for (const pkg of upgradeZclPackages) {
+      if (pkg.path) {
+        try {
+          const jsonData = JSON.parse(fs.readFileSync(pkg.path, 'utf-8'))
+          if (jsonData.upgradeRules !== undefined) {
+            const upgradeRulesJsonPath = path.resolve(
+              path.dirname(pkg.path),
+              jsonData.upgradeRules
+            )
+            try {
+              const upgradeRulesData = JSON.parse(
+                fs.readFileSync(upgradeRulesJsonPath, 'utf-8')
+              )
+              // Sorting upgrade rules by priority and then run them
+              upgradeRulesData.upgradeRuleScripts
+                .sort((a, b) => a.priority - b.priority)
+                .forEach((ur) => {
+                  upgradeRules.push({
+                    path: path.resolve(path.dirname(pkg.path), ur.path),
+                    category: isMultiProtocol ? upgradeRulesData.category : null
+                  })
+                })
+            } catch (error) {
+              console.error(
+                `Error reading or parsing upgrade rules from path ${upgradeRulesJsonPath}:`,
+                error
+              )
+            }
+          }
+        } catch (error) {
+          console.error(
+            `Error reading or parsing JSON from path ${pkg.path}:`,
+            error
+          )
+        }
+      }
+    }
+
     let importResult = await importJs.importDataFromFile(db, zapFile, {
       defaultZclMetafile: argv.zclProperties,
       postImportScript: argv.postImportScript,
       packageMatch: argv.packageMatch,
       upgradeZclPackages: upgradeZclPackages,
-      upgradeTemplatePackages: upgradeTemplatePackages
+      upgradeTemplatePackages: upgradeTemplatePackages,
+      upgradeRuleScripts: upgradeRules // Used to apply all the upgrade rules to the .zap file
     })
+
     let sessionId = importResult.sessionId
     await util.ensurePackagesAndPopulateSessionOptions(db, sessionId, {
       zcl: argv.zclProperties,
@@ -345,15 +389,23 @@ async function upgradeZapFile(argv, options) {
       fileFormat: argv.saveFileFormat
     })
     options.logger(`    👉 write out: ${outputPath}`)
+    try {
+      if (upgrade_results != null) {
+        if (!fs.existsSync(path.dirname(upgrade_results))) {
+          fs.mkdirSync(path.dirname(upgrade_results), { recursive: true })
+        }
+        await writeConversionResultsFile(
+          upgrade_results,
+          importResult.upgradeMessages
+        )
+      }
+      options.logger(`    👉 write out: ${upgrade_results}`)
+    } catch (error) {
+      options.logger(`    ⚠️  failed to write out: ${upgrade_results}`)
+    }
+    options.logger('😎 Upgrade done!')
   }
-  try {
-    if (upgrade_results != null)
-      await writeConversionResultsFile(upgrade_results)
-    options.logger(`    👉 write out: ${upgrade_results}`)
-  } catch (error) {
-    options.logger(`    ⚠️  failed to write out: ${upgrade_results}`)
-  }
-  options.logger('😎 Upgrade done!')
+
   if (options.quitFunction != null) {
     options.quitFunction()
   }
@@ -468,20 +520,25 @@ async function startConvert(argv, options) {
  * Write conversion results into file given.
  *
  * @param {*} file
+ * @param {*} messages
  * @returns promise of a file write operation.
  */
-async function writeConversionResultsFile(file) {
+async function writeConversionResultsFile(file, messages = null) {
   return fsp.writeFile(
     file,
-    YAML.stringify({
-      upgrade_results: [
-        {
-          message:
-            'ZCL Advanced Platform (ZAP) configuration has been successfully upgraded.',
-          status: 'automatic'
-        }
-      ]
-    })
+    messages
+      ? YAML.stringify({
+          upgrade_results: messages
+        })
+      : YAML.stringify({
+          upgrade_results: [
+            {
+              message:
+                'ZCL Advanced Platform (ZAP) configuration has been successfully upgraded.',
+              status: 'automatic'
+            }
+          ]
+        })
   )
 }