Add upgrade rules to be implemented to zap file open and implement user-selected package persistence for .zap file imports when the paths in it a wrong relatively (#1643)

* Add getUpgradePackageMatch function to util.js for package matching logic
* Modify httpPostFileOpen to extract and use selectedZclPackages/selectedTemplatePackages from request
* Update ZapConfig.vue submitForm to include selected packages in file open requests
* Enhance importDataFromFile to respect user-selected packages over embedded packages
* Add upgrade package matching and rule execution during import process

This ensures that user package selections from the UI are respected when
opening .zap files, rather than defaulting to packages embedded in the
.zap file itself. The changes maintain backward compatibility while
adding support for package upgrade rules and multi-protocol configurations.

- Resolves issue where package selections were not persisted during file operations.

- Adding unit tests for the upgrade rule on file open/import

- eliminate redundant upgrade rules code in startup.js and fix unit test failures
- Github: ZAP #1638

Author: brdandu

docs/api.md

@@ -19161,6 +19161,7 @@ back into a JS integer.
     * [~collectJsonData(jsonFile)](#module_JS API_ random utilities..collectJsonData)
     * [~patternFormat(pattern, data)](#module_JS API_ random utilities..patternFormat)
     * [~toErrorObject(err, message)](#module_JS API_ random utilities..toErrorObject) ⇒ <code>Error</code>
+    * [~getUpgradePackageMatch(db, upgradePackages, zapFilePackages, packageType)](#module_JS API_ random utilities..getUpgradePackageMatch) ⇒
 
 <a name="module_JS API_ random utilities..checksum"></a>
 
@@ -19459,6 +19460,22 @@ If message is provided, it will be used as the error message.
 | err | <code>\*</code> |  | 
 | message | <code>\*</code> | <code></code> | 
 
+<a name="module_JS API_ random utilities..getUpgradePackageMatch"></a>
+
+### JS API: random utilities~getUpgradePackageMatch(db, upgradePackages, zapFilePackages, packageType) ⇒
+Go over the zap file's top level packages and see if they can be upgraded
+based on the upgrade packages given.
+
+**Kind**: inner method of [<code>JS API: random utilities</code>](#module_JS API_ random utilities)  
+**Returns**: list of packages  
+
+| Param | Type |
+| --- | --- |
+| db | <code>\*</code> | 
+| upgradePackages | <code>\*</code> | 
+| zapFilePackages | <code>\*</code> | 
+| packageType | <code>\*</code> | 
+
 <a name="module_REST API_ various zcl utilities"></a>
 
 ## REST API: various zcl utilities

src-electron/importexport/import.js

@@ -143,6 +143,58 @@ function mergeZapExtension(baseState, extensionState) {
   }
 }
 
+/**
+ * Extract upgrade rules from ZCL packages
+ * @param {Array} upgradeZclPackages - Array of ZCL packages
+ * @returns {Array} Array of upgrade rule objects
+ */
+function extractUpgradeRules(upgradeZclPackages) {
+  let upgradeRules = []
+  // If more than one upgrade package is present then it is a multiprotocol
+  // application so upgrade rules should be added to the corresponding endpoints.
+  let isMultiProtocol = upgradeZclPackages.length > 1
+
+  for (const pkg of upgradeZclPackages) {
+    if (pkg.path) {
+      try {
+        const jsonData = JSON.parse(fs.readFileSync(pkg.path, 'utf-8'))
+        if (jsonData.upgradeRules !== undefined) {
+          const upgradeRulesJsonPath = path.resolve(
+            path.dirname(pkg.path),
+            jsonData.upgradeRules
+          )
+          try {
+            const upgradeRulesData = JSON.parse(
+              fs.readFileSync(upgradeRulesJsonPath, 'utf-8')
+            )
+            // Sorting upgrade rules by priority and then run them
+            upgradeRulesData.upgradeRuleScripts
+              .sort((a, b) => a.priority - b.priority)
+              .forEach((ur) => {
+                upgradeRules.push({
+                  path: path.resolve(path.dirname(pkg.path), ur.path),
+                  category: isMultiProtocol ? upgradeRulesData.category : null
+                })
+              })
+          } catch (error) {
+            env.logError(
+              `Error reading or parsing upgrade rules from path ${upgradeRulesJsonPath}:`,
+              error
+            )
+          }
+        }
+      } catch (error) {
+        env.logError(
+          `Error reading or parsing JSON from path ${pkg.path}:`,
+          error
+        )
+      }
+    }
+  }
+
+  return upgradeRules
+}
+
 /**
  * Writes the data from the file into a new session.
  * NOTE: This function does NOT initialize session packages.
@@ -179,6 +231,92 @@ async function importDataFromFile(
   }
 
   state = ff.convertFromFile(state)
+
+  // If upgrade rules are not known then figure them out.
+  if (!options.upgradeRuleScripts) {
+    // If defaultZclMetafile doesn't exist, figure it out based on filePath
+    if (
+      !options.defaultZclMetafile &&
+      state.package &&
+      Array.isArray(state.package)
+    ) {
+      // Find all ZCL properties packages from the state
+      const zclPackages = state.package.filter(
+        (pkg) => pkg.type === dbEnum.packageType.zclProperties
+      )
+      if (zclPackages.length > 0) {
+        options.defaultZclMetafile = zclPackages
+          .map((pkg) => {
+            if (pkg.path) {
+              // If the path is relative, resolve it relative to the filePath directory
+              if (!path.isAbsolute(pkg.path)) {
+                return path.resolve(path.dirname(filePath), pkg.path)
+              } else {
+                return pkg.path
+              }
+            }
+            return null
+          })
+          .filter((path) => path !== null)
+      } else {
+        // Fallback to builtin if no ZCL package found in state
+        options.defaultZclMetafile = [env.builtinSilabsZclMetafile()]
+      }
+    }
+
+    // If defaultTemplateFile doesn't exist, figure it out based on filePath
+    if (
+      !options.defaultTemplateFile &&
+      state.package &&
+      Array.isArray(state.package)
+    ) {
+      // Find all template packages from the state
+      const templatePackages = state.package.filter(
+        (pkg) => pkg.type === dbEnum.packageType.genTemplatesJson
+      )
+      if (templatePackages.length > 0) {
+        options.defaultTemplateFile = templatePackages
+          .map((pkg) => {
+            if (pkg.path) {
+              // If the path is relative, resolve it relative to the filePath directory
+              if (!path.isAbsolute(pkg.path)) {
+                return path.resolve(path.dirname(filePath), pkg.path)
+              } else {
+                return pkg.path
+              }
+            }
+            return null
+          })
+          .filter((path) => path !== null)
+      } else {
+        // Fallback to builtin if no template package found in state
+        options.defaultTemplateFile = [env.builtinTemplateMetafile()]
+      }
+    }
+
+    // Add upgrade package matching logic
+    let upgradeZclPackages = await util.getUpgradePackageMatch(
+      db,
+      options.zclProperties || options.defaultZclMetafile,
+      state.package,
+      dbEnum.packageType.zclProperties
+    )
+    let upgradeTemplatePackages = await util.getUpgradePackageMatch(
+      db,
+      options.generationTemplate || options.defaultTemplateFile,
+      state.package,
+      dbEnum.packageType.genTemplatesJson
+    )
+
+    let upgradeRules = extractUpgradeRules(upgradeZclPackages)
+
+    // Set upgrade rules in options if they exist
+    if (upgradeRules.length > 0) {
+      options.upgradeRuleScripts = upgradeRules
+    }
+    options.upgradeZclPackages = upgradeZclPackages
+    options.upgradeTemplatePackages = upgradeTemplatePackages
+  }
   try {
     await dbApi.dbBeginTransaction(db)
     let sid
@@ -269,3 +407,4 @@ async function importDataFromFile(
 exports.readDataFromFile = readDataFromFile
 exports.importDataFromFile = importDataFromFile
 exports.executePostImportScript = executePostImportScript
+exports.extractUpgradeRules = extractUpgradeRules

src-electron/main-process/startup.js

@@ -222,45 +222,6 @@ function findZapFiles(dir) {
   return zapFiles
 }
 
-/**
- * Go over the zap file's top level packages and see if they can be upgraded
- * based on the upgrade packages given.
- *
- * @param {*} db
- * @param {*} upgradePackages
- * @param {*} zapFilePackages
- * @param {*} packageType
- * @returns list of packages
- */
-async function getUpgradePackageMatch(
-  db,
-  upgradePackages,
-  zapFilePackages,
-  packageType
-) {
-  let matchedUpgradePackages = []
-  if (Array.isArray(upgradePackages) && Array.isArray(zapFilePackages)) {
-    for (let i = 0; i < upgradePackages.length; i++) {
-      let upgradePackage = await queryPackage.getPackageByPathAndType(
-        db,
-        upgradePackages[i],
-        packageType
-      )
-      if (upgradePackage) {
-        for (let j = 0; j < zapFilePackages.length; j++) {
-          if (
-            zapFilePackages[j].category == upgradePackage.category &&
-            zapFilePackages[j].type == upgradePackage.type
-          ) {
-            matchedUpgradePackages.push(upgradePackage)
-          }
-        }
-      }
-    }
-  }
-  return matchedUpgradePackages
-}
-
 /**
  * Upgrade the top level packages(.json files) of a .zap file when a gsdk is
  * updated.
@@ -299,60 +260,20 @@ async function upgradeZapFile(argv, options) {
       options.logger(`    🐝 New templates loaded: ${argv.generationTemplate}`)
     }
     let state = await importJs.readDataFromFile(zapFile)
-    let upgradeZclPackages = await getUpgradePackageMatch(
+    let upgradeZclPackages = await util.getUpgradePackageMatch(
       db,
       argv.zclProperties,
       state.package,
       dbEnum.packageType.zclProperties
     )
-    let upgradeTemplatePackages = await getUpgradePackageMatch(
+    let upgradeTemplatePackages = await util.getUpgradePackageMatch(
       db,
       argv.generationTemplate,
       state.package,
       dbEnum.packageType.genTemplatesJson
     )
 
-    let upgradeRules = []
-    // If more than one upgrade package is present then it is a multiprotocol
-    // application so upgrade rules should be added to the corresponding endpoints.
-    let isMultiProtocol = upgradeZclPackages.length > 1
-    for (const pkg of upgradeZclPackages) {
-      if (pkg.path) {
-        try {
-          const jsonData = JSON.parse(fs.readFileSync(pkg.path, 'utf-8'))
-          if (jsonData.upgradeRules !== undefined) {
-            const upgradeRulesJsonPath = path.resolve(
-              path.dirname(pkg.path),
-              jsonData.upgradeRules
-            )
-            try {
-              const upgradeRulesData = JSON.parse(
-                fs.readFileSync(upgradeRulesJsonPath, 'utf-8')
-              )
-              // Sorting upgrade rules by priority and then run them
-              upgradeRulesData.upgradeRuleScripts
-                .sort((a, b) => a.priority - b.priority)
-                .forEach((ur) => {
-                  upgradeRules.push({
-                    path: path.resolve(path.dirname(pkg.path), ur.path),
-                    category: isMultiProtocol ? upgradeRulesData.category : null
-                  })
-                })
-            } catch (error) {
-              console.error(
-                `Error reading or parsing upgrade rules from path ${upgradeRulesJsonPath}:`,
-                error
-              )
-            }
-          }
-        } catch (error) {
-          console.error(
-            `Error reading or parsing JSON from path ${pkg.path}:`,
-            error
-          )
-        }
-      }
-    }
+    let upgradeRules = importJs.extractUpgradeRules(upgradeZclPackages)
 
     let importResult = await importJs.importDataFromFile(db, zapFile, {
       defaultZclMetafile: argv.zclProperties,
@@ -837,13 +758,13 @@ async function generateSingleFile(
     }
 
     if (!isZapPackagePathPresent) {
-      upgradeZclPackages = await getUpgradePackageMatch(
+      upgradeZclPackages = await util.getUpgradePackageMatch(
         db,
         options.zcl,
         state.package,
         dbEnum.packageType.zclProperties
       )
-      upgradeTemplatePackages = await getUpgradePackageMatch(
+      upgradeTemplatePackages = await util.getUpgradePackageMatch(
         db,
         options.template,
         state.package,

src-electron/rest/file-ops.js

@@ -45,6 +45,11 @@ function httpPostFileOpen(db) {
     let search = req.body.search
     const query = new URLSearchParams(search)
     let file = query.get('filePath')
+
+    // Extract selected packages from request body or query
+    let selectedZclPackages = req.body.selectedZclPackages
+    let selectedTemplatePackages = req.body.selectedTemplatePackages
+
     // Gather .zapExtension files
     let zapFileExtensions = query.get('zapFileExtensions')
     if (file) {
@@ -82,6 +87,31 @@ function httpPostFileOpen(db) {
         if (zapFileExtensions) {
           options.extensionFiles = [zapFileExtensions]
         }
+        // Update options if selectedZclPackages exists and has size > 0
+        if (
+          selectedZclPackages &&
+          Array.isArray(selectedZclPackages) &&
+          selectedZclPackages.length > 0
+        ) {
+          options.zclProperties = selectedZclPackages.map((pkg) => pkg.path)
+          env.logInfo(
+            `Using ${selectedZclPackages.length} selected ZCL packages`
+          )
+        }
+
+        // Update options if selectedTemplatePackages exists and has size > 0
+        if (
+          selectedTemplatePackages &&
+          Array.isArray(selectedTemplatePackages) &&
+          selectedTemplatePackages.length > 0
+        ) {
+          options.generationTemplate = selectedTemplatePackages.map(
+            (pkg) => pkg.path
+          )
+          env.logInfo(
+            `Using ${selectedTemplatePackages.length} selected template packages`
+          )
+        }
         let importResult = await importJs.importDataFromFile(
           db,
           zapFilePath,