Add Jenkins test for chip

Jenkinsfile

 pipeline
 {
     agent { label 'Zap-Build' }
-    
+
     options { buildDiscarder(logRotator(artifactNumToKeepStr: '10')) }
-  
+
     stages
     {
         stage('Git setup')
         {
             steps
             {
-                script 
+                script
                 {
                     checkout scm
                 }
@@ -20,7 +20,7 @@ pipeline
         {
             steps
             {
-                script 
+                script
                 {
                     sh 'uname -a'
                     sh 'npm --version'
@@ -35,7 +35,7 @@ pipeline
         {
             steps
             {
-                script 
+                script
                 {
                     sh 'npm run lic'
                 }
@@ -45,7 +45,7 @@ pipeline
         {
             steps
             {
-                script 
+                script
                 {
                     sh 'npm outdated || true'
                 }
@@ -55,7 +55,7 @@ pipeline
         {
             steps
             {
-                script 
+                script
                 {
                     sh 'npm run build-spa'
                 }
@@ -65,7 +65,7 @@ pipeline
         {
             steps
             {
-                script 
+                script
                 {
                     // Temporarily comment this out: sh 'rm -rf ~/.zap'
                     sh 'npm run test'
@@ -73,7 +73,7 @@ pipeline
             }
         }
 	    stage('Self check')
-	    {   
+	    {
             steps
             {
                 script
@@ -93,11 +93,22 @@ pipeline
                 }
             }
         }
+        stage('Test CHIP generation')
+        {
+            steps
+            {
+                script
+                {
+                    sh 'npm run genchip'
+                    sh 'python ./test/resource/chip/compare.py'
+                }
+            }
+        }
         stage('Linux distribution artifacts')
         {
             steps
             {
-                script 
+                script
                 {
                     sh 'npm run dist-linux'
                     sh 'npm run apack:linux'
@@ -108,7 +119,7 @@ pipeline
         {
             steps
             {
-                script 
+                script
                 {
                     sh 'npm run dist-mac'
                     sh 'npm run apack:mac'
@@ -119,7 +130,7 @@ pipeline
         {
             steps
             {
-                script 
+                script
                 {
                     sh 'npm run dist-win'
                     sh 'npm run apack:win'
@@ -140,7 +151,7 @@ pipeline
         {
             steps
             {
-                script 
+                script
                 {
                     archiveArtifacts artifacts:'dist/zap*', fingerprint: true
                 }
@@ -157,8 +168,8 @@ pipeline
             }
         }
     }
-    post { 
-        always { 
+    /*post {
+        always {
             script
             {
                 def committers = emailextrecipients([[$class: 'CulpritsRecipientProvider'],
@@ -180,5 +191,5 @@ pipeline
             }
             cleanWs()
         }
-    }
+    }*/
 }

package.json

@@ -32,7 +32,7 @@
     "self-check": "electron src-electron/main-process/electron-main.js selfCheck",
     "gen": "node src-script/zap-generate.js -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/zigbee/gen-templates.json -i ./test/resource/save-file-test-regen.zap -o ./tmp",
     "gen2": "node src-script/zap-generate.js -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/zigbee/gen-templates.json -i ./test/resource/generation-test-file-1.zap -o ./tmp",
-    "genchip": "node src-script/zap-generate.js -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/chip/gen-templates.json -i ./test/resource/generation-test-file-1.zap -o ./tmp",
+    "genchip": "node src-script/zap-generate.js -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/chip/gen-test.json -i ./test/resource/chip/chip_test.zap -o ./tmp",
     "pack": "node src-script/build-spa.js && electron-builder --dir",
     "dist": "electron-builder",
     "dist-all": "electron-builder -mwl",

test/gen-template/chip/chip_test.zapt

+{{#all_user_clusters}}
+{{#all_user_cluster_commands}}
+{{#if (isStrEqual clusterSide parent.side)}}
+{{#if (isStrEqual clusterName parent.name)}}
+{{#if (zcl_command_arguments_count this.id)}}
+Cluster Name : {{asCamelCased parent.name false}}+Command Name : {{asCamelCased name false}}+{{#zcl_command_arguments}}Type Name : {{asUnderlyingType type}}+Argument Name : {{asSymbol label}}{{#unless (isLastElement index count)}},{{/unless}}{{/zcl_command_arguments}}
+{{else}}
+Cluster Name : {{asCamelCased parent.name false}}+Command Name : {{asCamelCased name false}}
+{{/if}}
+{{/if}}
+{{/if}}
+{{/all_user_cluster_commands}}
+{{/all_user_clusters}}
+

test/gen-template/chip/gen-test.json

+{
+  "name": "Test templates",
+  "version": "test-v1",
+  "options": {
+    "testOption": {
+      "code1": "label1",
+      "code2": "label2",
+      "code3": "label3",
+      "code4": "label4",
+      "code5": "label5"
+    },
+    "externalOption": "externalOptions.json",
+    "cli": "cli.json"
+  },
+  "helpers": ["helper-chip.js"],
+  "templates": [
+    {
+      "path": "chip_test.zapt",
+      "name": "CHIP testing",
+      "output": "chip_test.h"
+    }
+  ]
+}

test/resource/chip/chip_test.h

+Cluster Name : Groups+Command Name : AddGroup+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t *+Argument Name : groupName
+Cluster Name : Groups+Command Name : AddGroupIfIdentifying+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t *+Argument Name : groupName
+Cluster Name : Groups+Command Name : GetGroupMembership+Type Name : uint8_t+Argument Name : groupCount,Type Name : uint16_t+Argument Name : groupList
+Cluster Name : Groups+Command Name : RemoveAllGroups
+Cluster Name : Groups+Command Name : RemoveGroup+Type Name : uint16_t+Argument Name : groupId
+Cluster Name : Groups+Command Name : ViewGroup+Type Name : uint16_t+Argument Name : groupId
+Cluster Name : Groups+Command Name : AddGroupResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : groupId
+Cluster Name : Groups+Command Name : GetGroupMembershipResponse+Type Name : uint8_t+Argument Name : capacity,Type Name : uint8_t+Argument Name : groupCount,Type Name : uint16_t+Argument Name : groupList
+Cluster Name : Groups+Command Name : RemoveGroupResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : groupId
+Cluster Name : Groups+Command Name : ViewGroupResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t *+Argument Name : groupName
+Cluster Name : Identify+Command Name : Identify+Type Name : uint16_t+Argument Name : identifyTime
+Cluster Name : Identify+Command Name : IdentifyQuery
+Cluster Name : Identify+Command Name : TriggerEffect+Type Name : EmberAfIdentifyEffectIdentifier+Argument Name : effectId,Type Name : EmberAfIdentifyEffectVariant+Argument Name : effectVariant
+Cluster Name : Identify+Command Name : IdentifyQueryResponse+Type Name : uint16_t+Argument Name : timeout
+Cluster Name : LevelControl+Command Name : Move+Type Name : EmberAfMoveMode+Argument Name : moveMode,Type Name : uint8_t+Argument Name : rate,Type Name : uint8_t+Argument Name : optionMask,Type Name : uint8_t+Argument Name : optionOverride
+Cluster Name : LevelControl+Command Name : MoveToLevel+Type Name : uint8_t+Argument Name : level,Type Name : uint16_t+Argument Name : transitionTime,Type Name : uint8_t+Argument Name : optionMask,Type Name : uint8_t+Argument Name : optionOverride
+Cluster Name : LevelControl+Command Name : MoveToLevelWithOnOff+Type Name : uint8_t+Argument Name : level,Type Name : uint16_t+Argument Name : transitionTime
+Cluster Name : LevelControl+Command Name : MoveWithOnOff+Type Name : EmberAfMoveMode+Argument Name : moveMode,Type Name : uint8_t+Argument Name : rate
+Cluster Name : LevelControl+Command Name : Step+Type Name : EmberAfStepMode+Argument Name : stepMode,Type Name : uint8_t+Argument Name : stepSize,Type Name : uint16_t+Argument Name : transitionTime,Type Name : uint8_t+Argument Name : optionMask,Type Name : uint8_t+Argument Name : optionOverride
+Cluster Name : LevelControl+Command Name : StepWithOnOff+Type Name : EmberAfStepMode+Argument Name : stepMode,Type Name : uint8_t+Argument Name : stepSize,Type Name : uint16_t+Argument Name : transitionTime
+Cluster Name : LevelControl+Command Name : Stop+Type Name : uint8_t+Argument Name : optionMask,Type Name : uint8_t+Argument Name : optionOverride
+Cluster Name : LevelControl+Command Name : StopWithOnOff
+Cluster Name : OnOff+Command Name : Off
+Cluster Name : OnOff+Command Name : OffWithEffect+Type Name : EmberAfOnOffEffectIdentifier+Argument Name : effectId,Type Name : uint8_t+Argument Name : effectVariant
+Cluster Name : OnOff+Command Name : On
+Cluster Name : OnOff+Command Name : OnWithRecallGlobalScene
+Cluster Name : OnOff+Command Name : OnWithTimedOff+Type Name : uint8_t+Argument Name : onOffControl,Type Name : uint16_t+Argument Name : onTime,Type Name : uint16_t+Argument Name : offWaitTime
+Cluster Name : OnOff+Command Name : Toggle
+Cluster Name : OverTheAirBootloading+Command Name : ImageBlockRequest+Type Name : uint8_t+Argument Name : fieldControl,Type Name : uint16_t+Argument Name : manufacturerId,Type Name : uint16_t+Argument Name : imageType,Type Name : uint32_t+Argument Name : fileVersion,Type Name : uint32_t+Argument Name : fileOffset,Type Name : uint8_t+Argument Name : maxDataSize,Type Name : uint8_t *+Argument Name : requestNodeAddress
+Cluster Name : OverTheAirBootloading+Command Name : QueryNextImageRequest+Type Name : uint8_t+Argument Name : fieldControl,Type Name : uint16_t+Argument Name : manufacturerId,Type Name : uint16_t+Argument Name : imageType,Type Name : uint32_t+Argument Name : currentFileVersion,Type Name : uint16_t+Argument Name : hardwareVersion
+Cluster Name : OverTheAirBootloading+Command Name : UpgradeEndRequest+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : manufacturerId,Type Name : uint16_t+Argument Name : imageType,Type Name : uint32_t+Argument Name : fileVersion
+Cluster Name : OverTheAirBootloading+Command Name : ImageBlockResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : manufacturerId,Type Name : uint16_t+Argument Name : imageType,Type Name : uint32_t+Argument Name : fileVersion,Type Name : uint32_t+Argument Name : fileOffset,Type Name : uint8_t+Argument Name : dataSize,Type Name : uint8_t+Argument Name : imageData
+Cluster Name : OverTheAirBootloading+Command Name : ImageNotify+Type Name : uint8_t+Argument Name : payloadType,Type Name : uint8_t+Argument Name : queryJitter,Type Name : uint16_t+Argument Name : manufacturerId,Type Name : uint16_t+Argument Name : imageType,Type Name : uint32_t+Argument Name : newFileVersion
+Cluster Name : OverTheAirBootloading+Command Name : QueryNextImageResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : manufacturerId,Type Name : uint16_t+Argument Name : imageType,Type Name : uint32_t+Argument Name : fileVersion,Type Name : uint32_t+Argument Name : imageSize
+Cluster Name : OverTheAirBootloading+Command Name : UpgradeEndResponse+Type Name : uint16_t+Argument Name : manufacturerId,Type Name : uint16_t+Argument Name : imageType,Type Name : uint32_t+Argument Name : fileVersion,Type Name : uint32_t+Argument Name : currentTime,Type Name : uint32_t+Argument Name : upgradeTime
+Cluster Name : Scenes+Command Name : AddScene+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId,Type Name : uint16_t+Argument Name : transitionTime,Type Name : uint8_t *+Argument Name : sceneName,Type Name : EmberAfSceneExtensionFieldSet+Argument Name : extensionFieldSets
+Cluster Name : Scenes+Command Name : CopyScene+Type Name : uint8_t+Argument Name : mode,Type Name : uint16_t+Argument Name : groupIdFrom,Type Name : uint8_t+Argument Name : sceneIdFrom,Type Name : uint16_t+Argument Name : groupIdTo,Type Name : uint8_t+Argument Name : sceneIdTo
+Cluster Name : Scenes+Command Name : EnhancedAddScene+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId,Type Name : uint16_t+Argument Name : transitionTime,Type Name : uint8_t *+Argument Name : sceneName,Type Name : EmberAfSceneExtensionFieldSet+Argument Name : extensionFieldSets
+Cluster Name : Scenes+Command Name : EnhancedViewScene+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId
+Cluster Name : Scenes+Command Name : GetSceneMembership+Type Name : uint16_t+Argument Name : groupId
+Cluster Name : Scenes+Command Name : RecallScene+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId,Type Name : uint16_t+Argument Name : transitionTime
+Cluster Name : Scenes+Command Name : RemoveAllScenes+Type Name : uint16_t+Argument Name : groupId
+Cluster Name : Scenes+Command Name : RemoveScene+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId
+Cluster Name : Scenes+Command Name : StoreScene+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId
+Cluster Name : Scenes+Command Name : ViewScene+Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId
+Cluster Name : Scenes+Command Name : AddSceneResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId
+Cluster Name : Scenes+Command Name : GetSceneMembershipResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint8_t+Argument Name : capacity,Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneCount,Type Name : uint8_t+Argument Name : sceneList
+Cluster Name : Scenes+Command Name : RemoveAllScenesResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : groupId
+Cluster Name : Scenes+Command Name : RemoveSceneResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId
+Cluster Name : Scenes+Command Name : StoreSceneResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId
+Cluster Name : Scenes+Command Name : ViewSceneResponse+Type Name : EmberAfStatus+Argument Name : status,Type Name : uint16_t+Argument Name : groupId,Type Name : uint8_t+Argument Name : sceneId,Type Name : uint16_t+Argument Name : transitionTime,Type Name : uint8_t *+Argument Name : sceneName,Type Name : EmberAfSceneExtensionFieldSet+Argument Name : extensionFieldSets
+Cluster Name : ZLLCommissioning+Command Name : DeviceInformationRequest+Type Name : uint32_t+Argument Name : transaction,Type Name : uint8_t+Argument Name : startIndex
+Cluster Name : ZLLCommissioning+Command Name : GetEndpointListRequest+Type Name : uint8_t+Argument Name : startIndex
+Cluster Name : ZLLCommissioning+Command Name : GetGroupIdentifiersRequest+Type Name : uint8_t+Argument Name : startIndex
+Cluster Name : ZLLCommissioning+Command Name : IdentifyRequest+Type Name : uint32_t+Argument Name : transaction,Type Name : uint16_t+Argument Name : identifyDuration
+Cluster Name : ZLLCommissioning+Command Name : NetworkJoinEndDeviceRequest+Type Name : uint32_t+Argument Name : transaction,Type Name : uint8_t *+Argument Name : extendedPanId,Type Name : EmberAfKeyIndex+Argument Name : keyIndex,Type Name : /* TYPE WARNING: security_key defaults to */ uint8_t * +Argument Name : encryptedNetworkKey,Type Name : uint8_t+Argument Name : networkUpdateId,Type Name : uint8_t+Argument Name : logicalChannel,Type Name : uint16_t+Argument Name : panId,Type Name : uint16_t+Argument Name : networkAddress,Type Name : uint16_t+Argument Name : groupIdentifiersBegin,Type Name : uint16_t+Argument Name : groupIdentifiersEnd,Type Name : uint16_t+Argument Name : freeNetworkAddressRangeBegin,Type Name : uint16_t+Argument Name : freeNetworkAddressRangeEnd,Type Name : uint16_t+Argument Name : freeGroupIdentifierRangeBegin,Type Name : uint16_t+Argument Name : freeGroupIdentifierRangeEnd
+Cluster Name : ZLLCommissioning+Command Name : NetworkJoinRouterRequest+Type Name : uint32_t+Argument Name : transaction,Type Name : uint8_t *+Argument Name : extendedPanId,Type Name : EmberAfKeyIndex+Argument Name : keyIndex,Type Name : /* TYPE WARNING: security_key defaults to */ uint8_t * +Argument Name : encryptedNetworkKey,Type Name : uint8_t+Argument Name : networkUpdateId,Type Name : uint8_t+Argument Name : logicalChannel,Type Name : uint16_t+Argument Name : panId,Type Name : uint16_t+Argument Name : networkAddress,Type Name : uint16_t+Argument Name : groupIdentifiersBegin,Type Name : uint16_t+Argument Name : groupIdentifiersEnd,Type Name : uint16_t+Argument Name : freeNetworkAddressRangeBegin,Type Name : uint16_t+Argument Name : freeNetworkAddressRangeEnd,Type Name : uint16_t+Argument Name : freeGroupIdentifierRangeBegin,Type Name : uint16_t+Argument Name : freeGroupIdentifierRangeEnd
+Cluster Name : ZLLCommissioning+Command Name : NetworkStartRequest+Type Name : uint32_t+Argument Name : transaction,Type Name : uint8_t *+Argument Name : extendedPanId,Type Name : EmberAfKeyIndex+Argument Name : keyIndex,Type Name : /* TYPE WARNING: security_key defaults to */ uint8_t * +Argument Name : encryptedNetworkKey,Type Name : uint8_t+Argument Name : logicalChannel,Type Name : uint16_t+Argument Name : panId,Type Name : uint16_t+Argument Name : networkAddress,Type Name : uint16_t+Argument Name : groupIdentifiersBegin,Type Name : uint16_t+Argument Name : groupIdentifiersEnd,Type Name : uint16_t+Argument Name : freeNetworkAddressRangeBegin,Type Name : uint16_t+Argument Name : freeNetworkAddressRangeEnd,Type Name : uint16_t+Argument Name : freeGroupIdentifierRangeBegin,Type Name : uint16_t+Argument Name : freeGroupIdentifierRangeEnd,Type Name : uint8_t *+Argument Name : initiatorIeeeAddress,Type Name : uint16_t+Argument Name : initiatorNetworkAddress
+Cluster Name : ZLLCommissioning+Command Name : NetworkUpdateRequest+Type Name : uint32_t+Argument Name : transaction,Type Name : uint8_t *+Argument Name : extendedPanId,Type Name : uint8_t+Argument Name : networkUpdateId,Type Name : uint8_t+Argument Name : logicalChannel,Type Name : uint16_t+Argument Name : panId,Type Name : uint16_t+Argument Name : networkAddress
+Cluster Name : ZLLCommissioning+Command Name : ResetToFactoryNewRequest+Type Name : uint32_t+Argument Name : transaction
+Cluster Name : ZLLCommissioning+Command Name : ScanRequest+Type Name : uint32_t+Argument Name : transaction,Type Name : uint8_t+Argument Name : zigbeeInformation,Type Name : uint8_t+Argument Name : zllInformation
+Cluster Name : ZLLCommissioning+Command Name : DeviceInformationResponse+Type Name : uint32_t+Argument Name : transaction,Type Name : uint8_t+Argument Name : numberOfSubDevices,Type Name : uint8_t+Argument Name : startIndex,Type Name : uint8_t+Argument Name : deviceInformationRecordCount,Type Name : EmberAfDeviceInformationRecord+Argument Name : deviceInformationRecordList
+Cluster Name : ZLLCommissioning+Command Name : EndpointInformation+Type Name : uint8_t *+Argument Name : ieeeAddress,Type Name : uint16_t+Argument Name : networkAddress,Type Name : uint8_t+Argument Name : endpointId,Type Name : uint16_t+Argument Name : profileId,Type Name : uint16_t+Argument Name : deviceId,Type Name : uint8_t+Argument Name : version
+Cluster Name : ZLLCommissioning+Command Name : GetEndpointListResponse+Type Name : uint8_t+Argument Name : total,Type Name : uint8_t+Argument Name : startIndex,Type Name : uint8_t+Argument Name : count,Type Name : EmberAfEndpointInformationRecord+Argument Name : endpointInformationRecordList
+Cluster Name : ZLLCommissioning+Command Name : GetGroupIdentifiersResponse+Type Name : uint8_t+Argument Name : total,Type Name : uint8_t+Argument Name : startIndex,Type Name : uint8_t+Argument Name : count,Type Name : EmberAfGroupInformationRecord+Argument Name : groupInformationRecordList
+Cluster Name : ZLLCommissioning+Command Name : NetworkJoinEndDeviceResponse+Type Name : uint32_t+Argument Name : transaction,Type Name : EmberAfZllStatus+Argument Name : status
+Cluster Name : ZLLCommissioning+Command Name : NetworkJoinRouterResponse+Type Name : uint32_t+Argument Name : transaction,Type Name : EmberAfZllStatus+Argument Name : status
+Cluster Name : ZLLCommissioning+Command Name : NetworkStartResponse+Type Name : uint32_t+Argument Name : transaction,Type Name : EmberAfZllStatus+Argument Name : status,Type Name : uint8_t *+Argument Name : extendedPanId,Type Name : uint8_t+Argument Name : networkUpdateId,Type Name : uint8_t+Argument Name : logicalChannel,Type Name : uint16_t+Argument Name : panId
+Cluster Name : ZLLCommissioning+Command Name : ScanResponse+Type Name : uint32_t+Argument Name : transaction,Type Name : uint8_t+Argument Name : rssiCorrection,Type Name : uint8_t+Argument Name : zigbeeInformation,Type Name : uint8_t+Argument Name : zllInformation,Type Name : uint16_t+Argument Name : keyBitmask,Type Name : uint32_t+Argument Name : responseId,Type Name : uint8_t *+Argument Name : extendedPanId,Type Name : uint8_t+Argument Name : networkUpdateId,Type Name : uint8_t+Argument Name : logicalChannel,Type Name : uint16_t+Argument Name : panId,Type Name : uint16_t+Argument Name : networkAddress,Type Name : uint8_t+Argument Name : numberOfSubDevices,Type Name : uint8_t+Argument Name : totalGroupIds,Type Name : uint8_t+Argument Name : endpointId,Type Name : uint16_t+Argument Name : profileId,Type Name : uint16_t+Argument Name : deviceId,Type Name : uint8_t+Argument Name : version,Type Name : uint8_t+Argument Name : groupIdCount
+

test/resource/chip/compare.py

+import os
+from sys import exit
+# Constant
+solution_path = './test/resource/chip/chip_test.h'
+test_result_path = './tmp/chip_test.h'
+word_separator = "+"
+
+solution_list = []
+test_list = []
+try:
+    with open(solution_path) as f:
+        solution_list = f.readlines()
+except IOError:
+    print("Failed to open solution file")
+    exit(1)
+
+try:
+    with open(test_result_path) as f:
+        test_list = f.readlines()
+except IOError:
+    print("Failed to open test file")
+    exit(1)
+
+
+if (len(test_list) != len(solution_list)) :
+    print("Wrong number of lines in test file. Expected {0}, got {1}".format(len(solution_list), len(test_list)))
+    exit(1)
+
+for index in range(0, len(solution_list)):
+    expected_words = solution_list[index].split(word_separator)
+    test_words = test_list[index].split(word_separator)
+    for word_index in range(0, len(expected_words)):
+        if(expected_words[word_index] != test_words[word_index]):
+            print("!!!!!!!Mismatch found!!!!!!\n Expected \"{0}\" got \"{1}\"\n at line : {2}".format(expected_words[word_index], test_words[word_index], index))
+            exit(1)
+
+
+print("All good!")
+exit()
+