Work for multi template and multi zcl SDK. (#785)

* Clean up gen template loading.
* Minor cleanup with the context for template loading.
* Fix a brokenness with incorrect packageId.
* Deal with separate array case.
* Load multiple generation template packages.
* Add the ability to bypass template loading failure.
* Add secondary mechanism for loading templates to be compatible with primary.
* Update apack.json to support multiple templates.
* Lower the test threshold temporarily.
* Remove xvfb from some places.

.github/workflows/release.yml

@@ -38,7 +38,6 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         run: |
           sudo ./src-script/install-packages-ubuntu
-          sudo apt-get install --fix-missing xvfb
 
       - name: Prepare macOS build environment
         if: startsWith(matrix.os, 'macos')

.github/workflows/zap.yml

@@ -27,7 +27,7 @@ jobs:
           node-version: ${{ matrix.node-version }}
           cache: 'npm'
       - run: sudo apt-get update
-      - run: sudo apt-get install --fix-missing libpixman-1-dev libcairo-dev libsdl-pango-dev libjpeg-dev libgif-dev xvfb libxml2-utils
+      - run: sudo apt-get install --fix-missing libpixman-1-dev libcairo-dev libsdl-pango-dev libjpeg-dev libgif-dev libxml2-utils
       - run: node --version
       - run: npm --version
       - run: npm ci
@@ -40,7 +40,7 @@ jobs:
       - run: npm run lint
       - run: npm run xml-validate
       - run: npm run test:unit
-      - run: xvfb-run -a npm run self-check
+      - run: npm run self-check
       - run: npm run gen
       - run: npm run genmatter
       - run: npm run gendotdot

apack.json

@@ -4,7 +4,7 @@
   "description": "Graphical configuration tool for application and libraries based on Zigbee Cluster Library.",
   "path": [".", "node_modules/.bin/", "ZAP.app/Contents/MacOS"],
   "requiredFeatureLevel": "apack.core:9",
-  "featureLevel": 84,
+  "featureLevel": 85,
   "uc.triggerExtension": "zap",
   "executable": {
     "zap:win32.x86_64": {
@@ -61,10 +61,10 @@
       "cmd": "$(zap-cli) --version"
     },
     "uc_generate": {
-      "cmd": "$(zap-cli) generate --noUi --noServer -o ${generationOutput} --zcl ${sdkRoot}/app/zcl/zcl-zap.json --generationTemplate ${sdkRoot}/protocol/zigbee/app/framework/gen-template/gen-templates.json --in ${contentFolder}"
+      "cmd": "$(zap-cli) generate --noUi --noServer -o ${generationOutput} --zcl ${sdkRoot}/app/zcl/zcl-zap.json --zcl ${sdkRoot}/extension/matter/src/app/zap-templates/zcl/zcl.json --generationTemplate ${sdkRoot}/protocol/zigbee/app/framework/gen-template/gen-templates.json --generationTemplate ${sdkRoot}/extension/matter/src/app/zap-templates/app-templates.json --in ${contentFolder} --noLoadingFailure"
     },
     "uc_upgrade": {
-      "cmd": "$(zap-cli) convert --noUi --noServer -o ${tempContentFolder}/zcl_config.zap --zcl ${sdkRoot}/app/zcl/zcl-zap.json --generationTemplate ${sdkRoot}/protocol/zigbee/app/framework/gen-template/gen-templates.json ${tempContentFolder}/zcl_config.zap --results ${results}"
+      "cmd": "$(zap-cli) convert --noUi --noServer -o ${tempContentFolder}/zcl_config.zap --zcl ${sdkRoot}/app/zcl/zcl-zap.json --zcl ${sdkRoot}/extension/matter/src/app/zap-templates/zcl/zcl.json --generationTemplate ${sdkRoot}/protocol/zigbee/app/framework/gen-template/gen-templates.json --generationTemplate ${sdkRoot}/extension/matter/src/app/zap-templates/app-templates.json ${tempContentFolder}/zcl_config.zap --results ${results} --noLoadingFailure"
     },
     "zapHelp": {
       "cmd": "$(zap) --help"

jest.config.js

@@ -29,7 +29,7 @@ module.exports = {
       statements: 64,
       branches: 51,
       functions: 60,
-      lines: 65,
+      lines: 64,
     },
   },
   testMatch: ['<rootDir>/test/*.test.js', '<rootDir>/test/*.test.ts'],

src-electron/generator/generation-engine.js

@@ -28,6 +28,7 @@ const dbEnum = require('../../src-shared/db-enum.js')
 const env = require('../util/env')
 const templateEngine = require('./template-engine.js')
 const dbApi = require('../db/db-api.js')
+const e = require('express')
 
 /**
  * Given a path, it will read generation template object into memory.
@@ -462,14 +463,54 @@ async function loadZclExtensions(db, packageId, zclExt, defaultsPath) {
   return Promise.all(promises)
 }
 
+/**
+ * Api that loads an array of template JSON files or a single file if
+ * you just pass in one String.
+ *
+ * @param {*} db
+ * @param {*} genTemplatesJsonArray
+ */
+async function loadTemplates(
+  db,
+  genTemplatesJsonArray,
+  options = {
+    failOnLoadingError: true,
+  }
+) {
+  if (Array.isArray(genTemplatesJsonArray)) {
+    let globalCtx = {
+      packageIds: [],
+      packageId: null,
+    }
+    if (genTemplatesJsonArray != null && genTemplatesJsonArray.length > 0) {
+      for (let jsonFile of genTemplatesJsonArray) {
+        let ctx = await loadSingleTemplate(db, jsonFile)
+        if (ctx.error) {
+          if (options.failOnLoadingError) globalCtx.error = ctx.error
+        } else {
+          if (globalCtx.packageId == null) {
+            globalCtx.packageId = ctx.packageId
+          }
+          globalCtx.packageIds.push(ctx.packageId)
+        }
+      }
+    }
+    return globalCtx
+  } else {
+    let ctx = await loadSingleTemplate(db, genTemplatesJsonArray)
+    ctx.packageIds = [ctx.packageId]
+    return ctx
+  }
+}
+
 /**
  * Main API async function to load templates from a gen-template.json file.
  *
  * @param {*} db Database
- * @param {*} genTemplatesJson Path to the JSON file
+ * @param {*} genTemplatesJson Path to the JSON file or an array of paths to JSON file
  * @returns the loading context, contains: db, path, crc, packageId and templateData, or error
  */
-async function loadTemplates(db, genTemplatesJson) {
+async function loadSingleTemplate(db, genTemplatesJson) {
   let context = {
     db: db,
   }
@@ -485,23 +526,19 @@ async function loadTemplates(db, genTemplatesJson) {
     env.logWarning(context.error)
     return Promise.resolve(context)
   }
-
   context.path = file
-  return dbApi
-    .dbBeginTransaction(db)
-    .then(() => fsPromise.access(context.path, fs.constants.R_OK))
-    .then(() => {
-      env.logDebug(`Loading generation templates from: ${context.path}`)
-      return loadGenTemplate(context)
-    })
-    .then((ctx) => recordTemplatesPackage(ctx))
-    .catch((err) => {
-      env.logInfo(`Can not read templates from: ${context.path}`)
-      throw err
-    })
-    .finally(() => {
-      dbApi.dbCommit(db)
-    })
+  try {
+    await dbApi.dbBeginTransaction(db)
+    await fsPromise.access(file, fs.constants.R_OK)
+    context = await loadGenTemplate(context)
+    context = await recordTemplatesPackage(context)
+    return context
+  } catch (err) {
+    env.logInfo(`Can not read templates from: ${context.file}`)
+    throw err
+  } finally {
+    dbApi.dbCommit(db)
+  }
 }
 
 async function retrievePackageMetaInfo(db, genTemplatesPkgId) {

src-electron/main-process/startup.js

@@ -67,24 +67,21 @@ async function startNormal(quitFunction, argv) {
 
   try {
     await zclLoader.loadZclMetafiles(db, argv.zclProperties)
-    let ctx = await generatorEngine.loadTemplates(db, argv.generationTemplate)
+    let ctx = await generatorEngine.loadTemplates(db, argv.generationTemplate, {
+      failOnLoadingError: !argv.noLoadingFailure,
+    })
 
     if (ctx.error) {
       env.logWarning(ctx.error)
     }
 
     if (!argv.noServer) {
-      await httpServer.initHttpServer(
-        ctx.db,
-        argv.httpPort,
-        argv.studioHttpPort,
-        {
-          zcl: argv.zclProperties,
-          template: argv.generationTemplate,
-          allowCors: argv.allowCors,
-        }
-      )
-      await ipcServer.initServer(ctx.db, argv.httpPort)
+      await httpServer.initHttpServer(db, argv.httpPort, argv.studioHttpPort, {
+        zcl: argv.zclProperties,
+        template: argv.generationTemplate,
+        allowCors: argv.allowCors,
+      })
+      await ipcServer.initServer(db, argv.httpPort)
     }
     let port = httpServer.httpServerPort()
 
@@ -193,7 +190,9 @@ async function startConvert(argv, options) {
   await zclLoader.loadZclMetafiles(db, argv.zclProperties)
   options.logger(`    🐝 zcl package loaded: ${argv.zclProperties}`)
   if (argv.generationTemplate != null) {
-    await generatorEngine.loadTemplates(db, argv.generationTemplate)
+    await generatorEngine.loadTemplates(db, argv.generationTemplate, {
+      failOnLoadingError: !argv.noLoadingFailure,
+    })
     options.logger(`    🐝 templates loaded: ${argv.generationTemplate}`)
   }
 
@@ -298,7 +297,9 @@ async function startRegenerateSdk(argv, options) {
     for (let key of Object.keys(sdk.rt.genTemplates)) {
       let p = sdk.rt.genTemplates[key]
       options.logger(`    👈 ${p}`)
-      let loadData = await generatorEngine.loadTemplates(db, p)
+      let loadData = await generatorEngine.loadTemplates(db, p, {
+        failOnLoadingError: !argv.noLoadingFailure,
+      })
       sdk.templatePackageId[key] = loadData.packageId
     }
     options.logger('🐝 Performing generation')
@@ -399,21 +400,18 @@ async function startServer(argv, quitFunction) {
   mainDatabase = db
   try {
     await zclLoader.loadZclMetafiles(db, argv.zclProperties)
-    let ctx = await generatorEngine.loadTemplates(db, argv.generationTemplate)
+    let ctx = await generatorEngine.loadTemplates(db, argv.generationTemplate, {
+      failOnLoadingError: !argv.noLoadingFailure,
+    })
     if (ctx.error) {
       env.logWarning(ctx.error)
     }
-    await httpServer.initHttpServer(
-      ctx.db,
-      argv.httpPort,
-      argv.studioHttpPort,
-      {
-        zcl: argv.zclProperties,
-        template: argv.generationTemplate,
-        allowCors: argv.allowCors,
-      }
-    )
-    await ipcServer.initServer(ctx.db, argv.httpPort)
+    await httpServer.initHttpServer(db, argv.httpPort, argv.studioHttpPort, {
+      zcl: argv.zclProperties,
+      template: argv.generationTemplate,
+      allowCors: argv.allowCors,
+    })
+    await ipcServer.initServer(db, argv.httpPort)
     logRemoteData(httpServer.httpServerStartupMessage())
   } catch (err) {
     env.logError(err)
@@ -445,13 +443,26 @@ async function startSelfCheck(
     env.zapVersion()
   )
   options.logger('    👉 database and schema initialized')
-  await zclLoader.loadZclMetafiles(mainDb, argv.zclProperties)
-  options.logger('    👉 zcl data loaded')
-  let ctx = await generatorEngine.loadTemplates(mainDb, argv.generationTemplate)
+  let zclPackageIds = await zclLoader.loadZclMetafiles(
+    mainDb,
+    argv.zclProperties
+  )
+  options.logger(
+    `    👉 zcl metadata packlages loaded: ${zclPackageIds.length}`
+  )
+  let ctx = await generatorEngine.loadTemplates(
+    mainDb,
+    argv.generationTemplate,
+    {
+      failOnLoadingError: !argv.noLoadingFailure,
+    }
+  )
   if (ctx.error) {
     options.logger(`    ⚠️  ${ctx.error}`)
   } else {
-    options.logger('    👉 generation templates loaded')
+    options.logger(
+      `    👉 generation template packages loaded: ${ctx.packageIds.length}`
+    )
   }
 
   // This is a hack to prevent too quick shutdown that causes core dumps.
@@ -549,11 +560,15 @@ async function startGeneration(argv, options) {
   )
 
   await zclLoader.loadZclMetafiles(mainDb, zclProperties)
-  let ctx = await generatorEngine.loadTemplates(mainDb, templateMetafile)
+  let ctx = await generatorEngine.loadTemplates(mainDb, templateMetafile, {
+    failOnLoadingError: !argv.noLoadingFailure,
+  })
   if (ctx.error) {
     throw ctx.error
   }
 
+  let packageId = ctx.packageId
+
   let files = gatherFiles(zapFiles, { suffix: '.zap', doBlank: true })
   if (files.length == 0) {
     options.logger(`    👎 no zap files found in: ${zapFiles}`)
@@ -571,7 +586,7 @@ async function startGeneration(argv, options) {
   options.logger(`🕐 Setup time: ${util.duration(nsDuration)} `)
 
   await util.executePromisesSequentially(files, (f, index) =>
-    generateSingleFile(mainDb, f, ctx.packageId, output, index, options)
+    generateSingleFile(mainDb, f, packageId, output, index, options)
   )
 
   if (options.quitFunction != null) options.quitFunction()

src-electron/util/args.ts

@@ -118,7 +118,7 @@ export function processCommandLineArguments(argv: string[]) {
     .option('generationTemplate', {
       desc: 'generation template metafile (gen-template.json) to read in.',
       alias: ['gen', 'g'],
-      type: 'string',
+      type: 'array',
       default: env.builtinTemplateMetafile(),
     })
     .option('uiMode', {
@@ -208,6 +208,11 @@ export function processCommandLineArguments(argv: string[]) {
       type: 'string',
       default: null,
     })
+    .option('noLoadingFailure', {
+      desc: 'If you specify an invalid file for templates or zcl metafiles, zap will not fail, but will ignore it.',
+      type: 'boolean',
+      default: false,
+    })
     .option('results', {
       desc: 'Specifying the output YAML file to capture convert results.',
       type: 'string',