Fix the long standing problem of multiple transactions not being allowed simultaneously.

src-electron/db/db-api.js

@@ -29,6 +29,33 @@ const dbEnum = require('../../src-shared/db-enum.js')
 
 var inTransaction = false
 
+function executeBeginTransaction(db, resolve, reject) {
+  db.run('BEGIN TRANSACTION', [], function (err) {
+    if (err) {
+      env.logError('Failed to BEGIN TRANSACTION')
+      reject(err)
+    } else {
+      env.logSql('Executed BEGIN TRANSACTION')
+      resolve()
+    }
+  })
+}
+
+function delayBeginTransaction(db, resolve, reject) {
+  var cnt = 0
+  var interval = setInterval(() => {
+    if (inTransaction) {
+      cnt++
+      if (cnt > 50) {
+        reject('Waited for 5s for transaction to relinquish, but it did not.')
+      }
+    } else {
+      clearInterval(interval)
+      executeBeginTransaction(db, resolve, reject)
+    }
+  }, 100)
+}
+
 /**
  * Returns a promise to begin a transaction
  *
@@ -38,16 +65,12 @@ var inTransaction = false
  */
 function dbBeginTransaction(db) {
   return new Promise((resolve, reject) => {
-    db.run('BEGIN TRANSACTION', [], function (err) {
-      if (err) {
-        env.logError('Failed to BEGIN TRANSACTION')
-        reject(err)
-      } else {
-        env.logSql('Executed BEGIN TRANSACTION')
-        inTransaction = true
-        resolve()
-      }
-    })
+    if (inTransaction) {
+      delayBeginTransaction(db, resolve, reject)
+    } else {
+      inTransaction = true
+      executeBeginTransaction(db, resolve, reject)
+    }
   })
 }
 

src-electron/zcl/zcl-loader-dotdot.js

@@ -15,6 +15,8 @@
  *    limitations under the License.
  */
 
+const fs = require('fs')
+const fsp = fs.promises
 const env = require('../util/env.js')
 const path = require('path')
 const zclLoader = require('./zcl-loader')
@@ -32,8 +34,8 @@ const util = require('../util/util.js')
  */
 function collectDataFromLibraryXml(ctx) {
   env.logInfo(`Collecting ZCL files from: ${ctx.metadataFile}`)
-  return zclLoader
-    .readZclFile(ctx.metadataFile)
+  return fsp
+    .readFile(ctx.metadataFile)
     .then((data) =>
       util.calculateCrc({ filePath: ctx.metadataFile, data: data })
     )
@@ -91,8 +93,8 @@ function parseZclFiles(db, ctx) {
 
   ctx.zclFiles.forEach((file) => {
     env.logInfo(`Starting to parse Dotdot ZCL file: ${file}`)
-    var p = zclLoader
-      .readZclFile(file)
+    var p = fsp
+      .readFile(file)
       .then((data) => util.calculateCrc({ filePath: file, data: data }))
       .then((data) =>
         zclLoader.qualifyZclFile(
@@ -618,13 +620,14 @@ function loadZclData(db, ctx) {
  * @return {*} object w/ following: { packageId: pkgId } or { err: err }
  */
 function loadIndividualDotDotFile(db, filePath) {
-  return zclLoader.readZclFile(filePath).then((data) => {
+  return fsp.readFile(filePath).then((data) => {
     console.log(data)
   })
 }
 
 /**
- * Toplevel function that loads the xml library file and orchestrates the promise chain.
+ * Toplevel function that loads the xml library file
+ * and orchestrates the promise chain.
  *
  * @export
  * @param {*} db

src-electron/zcl/zcl-loader-silabs.js

@@ -16,6 +16,7 @@
  */
 
 const fs = require('fs')
+const fsp = fs.promises
 const path = require('path')
 const properties = require('properties')
 const dbApi = require('../db/db-api.js')
@@ -712,8 +713,8 @@ function parseZclFiles(db, ctx) {
   env.logInfo(`Starting to parse ZCL files: ${ctx.zclFiles}`)
   return Promise.all(
     ctx.zclFiles.map((file) =>
-      zclLoader
-        .readZclFile(file)
+      fsp
+        .readFile(file)
         .then((data) => util.calculateCrc({ filePath: file, data: data }))
         .then((data) =>
           zclLoader.qualifyZclFile(
@@ -738,50 +739,6 @@ function parseZclFiles(db, ctx) {
     .then(() => ctx)
 }
 
-/**
- * Parses a single file.
- *
- * @param {*} db
- * @param {*} filePath
- * @returns Promise of a loaded file.
- */
-function loadIndividualSilabsFile(db, filePath, boundValidator) {
-  var pkgId
-  return zclLoader
-    .readZclFile(filePath)
-    .then((data) => util.calculateCrc({ filePath: filePath, data: data }))
-    .then((data) =>
-      zclLoader.qualifyZclFile(
-        db,
-        data,
-        null,
-        dbEnum.packageType.zclXmlStandalone
-      )
-    )
-    .then((result) => {
-      pkgId = result.packageId
-      return result
-    })
-    .then((result) => zclLoader.parseZclFile(result, boundValidator))
-    .then((result) => {
-      if (result.validation && result.validator.isValid == false) {
-        throw new Error('Validation Failed')
-      }
-      return result
-    })
-    .then((result) => processParsedZclData(db, result))
-    .then((laterPromises) =>
-      Promise.all(laterPromises.flat(1).map((promise) => promise()))
-    )
-    .then(() => zclLoader.processZclPostLoading(db))
-    .then(() => {
-      return { packageId: pkgId }
-    })
-    .catch((err) => {
-      return { err: err }
-    })
-}
-
 /**
  * Parses the manufacturers xml.
  *
@@ -791,8 +748,8 @@ function loadIndividualSilabsFile(db, filePath, boundValidator) {
  */
 function parseManufacturerData(db, ctx) {
   if (!ctx.manufacturersXml) return Promise.resolve(ctx)
-  return zclLoader
-    .readZclFile(ctx.manufacturersXml)
+  return fsp
+    .readFile(ctx.manufacturersXml)
     .then((data) =>
       zclLoader.parseZclFile({ data: data }).then((manufacturerMap) =>
         queryPackage.insertOptionsKeyValues(
@@ -816,8 +773,8 @@ function parseManufacturerData(db, ctx) {
  */
 function parseZclSchema(db, ctx) {
   if (!ctx.zclSchema || !ctx.zclValidation) return Promise.resolve(ctx)
-  return zclLoader
-    .readZclFile(ctx.zclSchema)
+  return fsp
+    .readFile(ctx.zclSchema)
     .then((data) => util.calculateCrc({ filePath: ctx.zclSchema, data: data }))
     .then((data) =>
       zclLoader.qualifyZclFile(
@@ -832,8 +789,8 @@ function parseZclSchema(db, ctx) {
       return result
     })
     .then(() =>
-      zclLoader
-        .readZclFile(ctx.zclValidation)
+      fsp
+        .readFile(ctx.zclValidation)
         .then((data) =>
           util.calculateCrc({ filePath: ctx.zclValidation, data: data })
         )
@@ -894,6 +851,14 @@ function parseTextOptions(db, pkgRef, textOptions) {
   return Promise.all(promises)
 }
 
+/**
+ * Parses the boolean options.
+ *
+ * @param {*} db
+ * @param {*} pkgRef
+ * @param {*} booleanCategories
+ * @returns Promise of a parsed boolean options.
+ */
 function parseBoolOptions(db, pkgRef, booleanCategories) {
   if (!booleanCategories) return Promise.resolve()
   let options
@@ -999,7 +964,52 @@ function parseBoolDefaults(db, pkgRef, booleanCategories) {
 }
 
 /**
- * Toplevel function that loads the properties file and orchestrates the promise chain.
+ * Parses a single file.
+ *
+ * @param {*} db
+ * @param {*} filePath
+ * @returns Promise of a loaded file.
+ */
+function loadIndividualSilabsFile(db, filePath, boundValidator) {
+  var pkgId
+  return fsp
+    .readFile(filePath)
+    .then((data) => util.calculateCrc({ filePath: filePath, data: data }))
+    .then((data) =>
+      zclLoader.qualifyZclFile(
+        db,
+        data,
+        null,
+        dbEnum.packageType.zclXmlStandalone
+      )
+    )
+    .then((result) => {
+      pkgId = result.packageId
+      return result
+    })
+    .then((result) => zclLoader.parseZclFile(result, boundValidator))
+    .then((result) => {
+      if (result.validation && result.validator.isValid == false) {
+        throw new Error('Validation Failed')
+      }
+      return result
+    })
+    .then((result) => processParsedZclData(db, result))
+    .then((laterPromises) =>
+      Promise.all(laterPromises.flat(1).map((promise) => promise()))
+    )
+    .then(() => zclLoader.processZclPostLoading(db))
+    .then(() => {
+      return { packageId: pkgId }
+    })
+    .catch((err) => {
+      return { err: err }
+    })
+}
+
+/**
+ * Toplevel function that loads the toplevel metafile
+ * and orchestrates the promise chain.
  *
  * @export
  * @param {*} db

src-electron/zcl/zcl-loader.js

@@ -75,8 +75,9 @@ function recordToplevelPackage(db, ctx) {
  * @param {*} ctx
  */
 function recordVersion(ctx) {
-  if (ctx.version == null) return Promise.resolve(ctx)
-  else {
+  if (ctx.version == null) {
+    return Promise.resolve(ctx)
+  } else {
     return queryPackage
       .updateVersion(ctx.db, ctx.packageId, ctx.version)
       .then(() => ctx)
@@ -265,17 +266,6 @@ function processZclPostLoading(db) {
   return queryZcl.updateDeviceTypeEntityReferences(db)
 }
 
-/**
- * Promises to read a file and resolve with the content
- *
- * @param {*} file
- * @returns promise that resolves as readFile
- */
-function readZclFile(file) {
-  env.logInfo(`Reading individual file: ${file}`)
-  return fsp.readFile(file)
-}
-
 /**
  * Promises to parse the ZCL file, expecting object of { filePath, data, packageId, msg }
  *
@@ -305,6 +295,5 @@ exports.recordToplevelPackage = recordToplevelPackage
 exports.recordVersion = recordVersion
 exports.processZclPostLoading = processZclPostLoading
 exports.loadIndividualFile = loadIndividualFile
-exports.readZclFile = readZclFile
 exports.qualifyZclFile = qualifyZclFile
 exports.parseZclFile = parseZclFile

test/zcl-loader-consecutive.test.js

@@ -27,6 +27,22 @@ const zclLoader = require('../src-electron/zcl/zcl-loader.js')
 const args = require('../src-electron/util/args.js')
 const env = require('../src-electron/util/env.js')
 
+test('that that parallel loading of zcl and dotdot is possible', async () => {
+  var dotDotZclPropertiesFile = './zcl-builtin/dotdot/library.xml'
+  var zclPropertiesFile = args.zclPropertiesFile
+
+  var db = await dbApi.initRamDatabase()
+  await dbApi.loadSchema(db, env.schemaFile(), env.zapVersion())
+
+  var promises = []
+  promises.push(zclLoader.loadZcl(db, zclPropertiesFile))
+  promises.push(zclLoader.loadZcl(db, dotDotZclPropertiesFile))
+
+  await Promise.all(promises)
+
+  await dbApi.closeDatabase(db)
+}, 10000)
+
 test('test that consecutive loading of metafiles properly avoids duplication', () => {
   var dotDotZclPropertiesFile = './zcl-builtin/dotdot/library.xml'
   var db