fix Nexus asset query issue. (#724)

* fix Nexus asset query issue.

was not constructing continuationToken URL correctly.

* ignore temp artifact files.

* unit test for download-artifact

* update cached download-artifact.js

* ignore download-artifact test from zap unit tests.

* download-artifact: add unit test / delegate to Github if specified branch is not cached on Nexus

intended for internal jenkins PRs that specifies a non-cached ZAP branch
on Nexus

.gitignore

@@ -83,3 +83,6 @@ reports
 
 # Downloaded artifacts from Nexus / Github / etc via download-artifact script
 artifacts
+**/zap-*.zip
+**/zap-*zip.json
+**/zap-*.zip.download

jest.config.js

@@ -58,4 +58,8 @@ module.exports = {
   transformIgnorePatterns: ['<rootDir>/node_modules/(?!quasar/lang)'],
   snapshotSerializers: ['<rootDir>/node_modules/jest-serializer-vue'],
   testResultsProcessor: 'jest-sonar-reporter',
+  testPathIgnorePatterns: [
+    '/node_modules/',
+    '<rootDir>/test/download-artifact.test.js',
+  ],
 }

package.json

@@ -19,6 +19,7 @@
     "download-artifact": "ts-node src-script/download-artifact.ts",
     "pkg:download-artifact": "npx pkg src-script/download-artifact.js  -t node16-linux-x64,node16-linux-arm64,node16-macos-x64,node16-win-x64 --output dist/download-artifact --compress GZip",
     "compile:download-artifact": "tsc src-script/download-artifact.ts --esModuleInterop",
+    "test:download-artifact": "npx jest --collectCoverage=false ./test/download-artifact.test.js --testPathIgnorePatterns=[]",
     "lic": "node src-script/license-check.js --production",
     "lint": "eslint --ext .js,.vue src src-electron src-shared src-script test",
     "lintfix": "eslint --fix --ext .js,.vue src src-electron src-shared src-script test",

src-script/download-artifact.ts

@@ -39,7 +39,7 @@ const DEFAULT_OWNER = 'SiliconLabs'
 const DEFAULT_REPO = 'zap'
 const NEXUS_SERVER = 'https://nexus.silabs.net'
 const NEXUS_REPO_NAME = 'zap-release-package'
-const cachedBranches = ['master', 'rel']
+const nexusCachedBranches = ['master', 'rel']
 
 // cheap and secure
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
@@ -242,19 +242,35 @@ function platforms(argv: any) {
   return list
 }
 
-async function nexusGetArtifacts(url: string, options: DlOptions) {
-  let accumulatedItems: any[] = []
-  let continuationToken = ''
+async function getExistingGithubBranches(
+  options: DlOptions
+): Promise<string[]> {
+  const url = `https://api.github.com/repos/${options.owner}/${options.repo}/branches`
+  let branches = []
 
   try {
     if (DEBUG) console.log(`GET: ${url}`)
     let resp = await axios.get(url)
+    branches = resp?.data?.map((x: any) => x.name)
+  } catch (error) {}
+
+  return branches
+}
+
+async function nexusGetArtifacts(baseUrl: string, options: DlOptions) {
+  let accumulatedItems: any[] = []
+  let continuationToken = ''
+
+  try {
+    if (DEBUG) console.log(`GET: ${baseUrl}`)
+    let resp = await axios.get(baseUrl)
     accumulatedItems = accumulatedItems.concat(resp?.data?.items)
     continuationToken = resp?.data?.continuationToken
 
+    let url = ''
     do {
       if (continuationToken) {
-        url = url + `&continuationToken=${continuationToken}`
+        url = baseUrl + `&continuationToken=${continuationToken}`
       } else {
         break
       }
@@ -414,7 +430,7 @@ async function githubGetArtifacts(options: DlOptions) {
       return [...artifacts]
     }
   } else {
-    console.error('Unable to retrieve any artifacts for download.')
+    console.error('Unable to find any artifacts for download.')
     return []
   }
 }
@@ -521,15 +537,28 @@ async function main() {
     nameOnly: y.argv.nameOnly,
   }
 
-  // Download site sources: Nexus, Github
+  let githubBranches = await getExistingGithubBranches(dlOptions)
+
+  // evaluate artifact source
   if (dlOptions.src === 'nexus' && (await isReachable(NEXUS_SERVER))) {
-    if (!cachedBranches.includes(dlOptions.branch)) {
+    if (
+      githubBranches.includes(dlOptions.branch) &&
+      !nexusCachedBranches.includes(dlOptions.branch)
+    ) {
+      console.log(
+        `Branch ${dlOptions.branch} is not cached on Nexus. Defaulting to master branch on Github instead.`
+      )
+      dlOptions.src = 'github'
+    } else if (!nexusCachedBranches.includes(dlOptions.branch)) {
       console.log(
         `Branch ${dlOptions.branch} is not cached on Nexus. Defaulting to master branch instead.`
       )
       dlOptions.branch = 'master'
     }
+  }
 
+  // Download site sources: Nexus, Github
+  if (dlOptions.src === 'nexus') {
     const nexusUrl = nexusRestApiUrl(
       NEXUS_REPO_NAME,
       `${dlOptions.owner}/${dlOptions.repo}/${dlOptions.branch}/*`

test/download-artifact.test.js

+/**
+ *
+ *    Copyright (c) 2021 Silicon Labs
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ *
+ *
+ * @jest-environment node
+ */
+
+const { exec, execSync, execFile } = require('child_process')
+const path = require('node:path')
+const download_artifact_script = path.join(
+  __dirname,
+  '../src-script/download-artifact.js'
+)
+
+beforeAll(() => {
+  execSync(`npm run compile:download-artifact`)
+  execSync(`chmod a+x ${download_artifact_script}`)
+})
+
+test('Downloading from Nexus (default)', async () => {
+  let output = execSync(download_artifact_script)
+  console.log(output.toString())
+  expect(output.toString()).toMatch(/.*nexus.silabs.*/)
+  expect(output.toString()).toMatch(/.*zap-...-zip.zip[.]*done.*/)
+})
+
+test('Downloading from Github', async () => {
+  let output = execSync(`${download_artifact_script} --src github`)
+  console.log(output.toString())
+  expect(output.toString()).toMatch(/.*github.com.*/)
+  expect(output.toString()).toMatch(/.*zap-...-zip.zip[.]*done.*/)
+})
+
+test('Default back to master branch if unknown branch is specified', async () => {
+  let output = execSync(
+    `${download_artifact_script} -b random_unknown_branch_name`
+  )
+  console.log(output.toString())
+  expect(output.toString()).toMatch(/.Defaulting to master branch instead.*/)
+})
+
+test('Download from Github if specified branch is available on Github but not Nexus', async () => {
+  let output = execSync(`${download_artifact_script} -b unit_test_branch`)
+  console.log(output.toString())
+  expect(output.toString()).toMatch(
+    /.*Defaulting to master branch on Github instead.*/
+  )
+})