CI Adjustment

builder.json

@@ -8,11 +8,6 @@
     "crt"
   ],
   "env": {
-    "DA_TOPIC": "test/da",
-    "DA_SHADOW_PROPERTY": "datest",
-    "DA_SHADOW_VALUE_SET": "ON",
-    "DA_SHADOW_VALUE_DEFAULT": "OFF",
-    "DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs"
   },
   "hosts": {
     "manylinux": {
@@ -28,13 +23,8 @@
     "build",
     "build-samples"
   ],
-  "test_steps": [
-    "python3 -m pip install boto3",
-    "python3 deviceadvisor/script/DATestRun.py",
-    "python3 devicedefender/script/DDTestRun.py"],
   "variants" : {
     "skip_sample": {
-        "!test_steps": [],
         "!build_steps": [
           "build"
         ]

codebuild/samples/linux-smoke-tests.yml

@@ -16,10 +16,6 @@ phases:
       - echo Build started on `date`
       - $CODEBUILD_SRC_DIR/codebuild/samples/setup-linux.sh
       - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh
-      - $CODEBUILD_SRC_DIR/codebuild/samples/connect-websocket-linux.sh
-      - $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-connect-linux.sh
-      - $CODEBUILD_SRC_DIR/codebuild/samples/securetunnel-linux.sh
-      - $CODEBUILD_SRC_DIR/codebuild/samples/connect-custom-authorizer-linux.sh
   post_build:
     commands:
       - echo Build completed on `date`

deviceadvisor/script/DATestConfig.json

@@ -2,11 +2,11 @@
     "tests" :[  "MQTT Connect", "MQTT Publish", "MQTT Subscribe", "Shadow Publish", "Shadow Update"],
     "test_suite_ids" :
     {
-        "MQTT Connect" : "ejbdzmo3hf3v",
-        "MQTT Publish" : "euw7favf6an4",
-        "MQTT Subscribe" : "01o8vo6no7sd",
-        "Shadow Publish" : "elztm2jebc1q",
-        "Shadow Update" : "vuydgrbbbfce"
+        "MQTT Connect" : "mxn32qkm8npn",
+        "MQTT Publish" : "gcjhujhhz50p",
+        "MQTT Subscribe" : "nyiuiwx5yxtj",
+        "Shadow Publish" : "fttdr8ufljnf",
+        "Shadow Update" : "ng9t8am2jnry"
     },
     "test_exe_path" :
     {

deviceadvisor/script/DATestRun.py

@@ -28,30 +28,50 @@ def process_logs(log_group, log_stream, thing_name):
         logGroupName=log_group,
         logStreamName=log_stream
     )
-    log_file = thing_name + ".log"
+    log_file = "DA_Log_CPP_" + thing_name + ".log"
     f = open(log_file, 'w')
     for event in response["events"]:
         f.write(event['message'])
     f.close()
-    s3.Bucket(os.environ['DA_S3_NAME']).upload_file(log_file, log_file)
+
+    try:
+        secrets_client = boto3.client(
+            "secretsmanager", region_name=os.environ["AWS_DEFAULT_REGION"])
+        s3_bucket_name = secrets_client.get_secret_value(SecretId="ci/DeviceAdvisor/s3bucket")["SecretString"]
+        s3.Bucket(s3_bucket_name).upload_file(log_file, log_file)
+        print("[Device Advisor] Device Advisor Log file uploaded to "+ log_file)
+
+    except Exception:
+        print ("[Device Advisor] Error: could not store log in S3 bucket!")
+
     os.remove(log_file)
     print("[Device Advisor] Issues on test " + test_name + ". Please check out the logs at "+thing_name+".log on S3.")
 
+# Sleep for a random time
 def sleep_with_backoff(base, max):
     sleep(random.randint(base,max))
 
 
 ##############################################
 # Initialize variables
 # create aws clients
-client = boto3.client('iot')
-dataClient = boto3.client('iot-data')
-deviceAdvisor = boto3.client('iotdeviceadvisor')
-s3 = boto3.resource('s3')
+try:
+    client = boto3.client('iot')
+    dataClient = boto3.client('iot-data')
+    deviceAdvisor = boto3.client('iotdeviceadvisor')
+    s3 = boto3.resource('s3')
+except Exception:
+    print ("[Device Advisor] Error: could not create boto3 clients")
+    exit(-1)
 
 # const
 BACKOFF_BASE = 5
 BACKOFF_MAX = 10
+# 60 minutes divided by the maximum back-off = longest time a DA run can last with this script.
+MAXIMUM_CYCLE_COUNT = (3600 / BACKOFF_MAX)
+
+# Did Device Advisor fail a test? If so, this should be true
+did_at_least_one_test_fail = False
 
 # load test config
 f = open('deviceadvisor/script/DATestConfig.json')
@@ -80,21 +100,21 @@ def sleep_with_backoff(base, max):
         # 'thingArn': 'string',
         # 'thingId': 'string'
         # }
-        print("[Device Advisor]Info: Started to create thing...")
+        print("[Device Advisor] Info: Started to create thing...")
         create_thing_response = client.create_thing(
             thingName=thing_name
         )
         os.environ["DA_THING_NAME"] = thing_name
 
-    except Exception as e:
-        print("[Device Advisor]Error: Failed to create thing: " + thing_name)
+    except Exception:
+        print("[Device Advisor] Error: Failed to create thing: " + thing_name)
         exit(-1)
 
 
     ##############################################
     # create certificate and keys used for testing
     try:
-        print("[Device Advisor]Info: Started to create certificate...")
+        print("[Device Advisor] Info: Started to create certificate...")
         # create_cert_response:
         # {
         # 'certificateArn': 'string',
@@ -123,25 +143,44 @@ def sleep_with_backoff(base, max):
         os.environ["DA_CERTI"] = certificate_path
         os.environ["DA_KEY"] = key_path
 
-    except:
-        client.delete_thing(thingName = thing_name)
-        print("[Device Advisor]Error: Failed to create certificate.")
+    except Exception:
+        try:
+            client.delete_thing(thingName = thing_name)
+        except Exception:
+            print ("[Device Advisor] Error: Could not delete thing.")
+        print("[Device Advisor] Error: Failed to create certificate.")
+        exit(-1)
+
+    certificate_arn = create_cert_response['certificateArn']
+    certificate_id = create_cert_response['certificateId']
+
+    ##############################################
+    # attach policy to certificate
+    try:
+        secrets_client = boto3.client(
+            "secretsmanager", region_name=os.environ["AWS_DEFAULT_REGION"])
+        policy_name = secrets_client.get_secret_value(SecretId="ci/DeviceAdvisor/policy_name")["SecretString"]
+        client.attach_policy (
+            policyName= policy_name,
+            target = certificate_arn
+        )
+    except Exception as ex:
+        print (ex)
+        delete_thing_with_certi(thing_name, certificate_id, certificate_arn )
+        print("[Device Advisor] Error: Failed to attach policy.")
         exit(-1)
 
     ##############################################
     # attach certification to thing
     try:
-        print("[Device Advisor]Info: Attach certificate to test thing...")
+        print("[Device Advisor] Info: Attach certificate to test thing...")
         # attache the certificate to thing
         client.attach_thing_principal(
             thingName = thing_name,
             principal = create_cert_response['certificateArn']
         )
 
-        certificate_arn = create_cert_response['certificateArn']
-        certificate_id = create_cert_response['certificateId']
-
-    except:
+    except Exception:
         delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
         print("[Device Advisor]Error: Failed to attach certificate.")
         exit(-1)
@@ -153,6 +192,7 @@ def sleep_with_backoff(base, max):
         ######################################
         # set default shadow, for shadow update, if the
         # shadow does not exists, update will fail
+        print("[Device Advisor] Info: About to update shadow.")
         payload_shadow = json.dumps(
         {
         "state": {
@@ -169,6 +209,7 @@ def sleep_with_backoff(base, max):
             payload = payload_shadow)
         get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name)
         # make sure shadow is created before we go to next step
+        print("[Device Advisor] Info: About to wait for shadow update.")
         while(get_shadow_response is None):
             get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name)
 
@@ -179,24 +220,31 @@ def sleep_with_backoff(base, max):
         # 'suiteRunArn': 'string',
         # 'createdAt': datetime(2015, 1, 1)
         # }
-        print("[Device Advisor]Info: Start device advisor test: " + test_name)
+        print("[Device Advisor] Info: Start device advisor test: " + test_name)
         sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX)
         test_start_response = deviceAdvisor.start_suite_run(
-        suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
-        suiteRunConfiguration={
-            'primaryDevice': {
-                'thingArn': create_thing_response['thingArn'],
-            },
-            'parallelRun': True
+            suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
+            suiteRunConfiguration={
+                'primaryDevice': {
+                    'thingArn': create_thing_response['thingArn'],
+                },
+                'parallelRun': True
         })
 
         # get DA endpoint
+        print("[Device Advisor] Info: Getting Device Advisor endpoint.")
         endpoint_response = deviceAdvisor.get_endpoint(
             thingArn = create_thing_response['thingArn']
         )
         os.environ['DA_ENDPOINT'] = endpoint_response['endpoint']
 
+        cycle_number = 0
         while True:
+            cycle_number += 1
+            if (cycle_number >= MAXIMUM_CYCLE_COUNT):
+                print(f"[Device Advisor] Error: {cycle_number} of cycles lasting {BACKOFF_BASE} to {BACKOFF_MAX} seconds have passed.")
+                raise Exception(f"ERROR - {cycle_number} of cycles lasting {BACKOFF_BASE} to {BACKOFF_MAX} seconds have passed.")
+
             # sleep for 1s every loop to avoid TooManyRequestsException
             sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX)
             test_result_responds = deviceAdvisor.get_suite_run(
@@ -214,6 +262,7 @@ def sleep_with_backoff(base, max):
             elif (test_result_responds['status'] == 'RUNNING' and
             test_result_responds['testResult']['groups'][0]['tests'][0]['status'] == 'RUNNING'):
                 try:
+                    print ("[Device Advisor] Info: About to get start Device Advisor companion test application.")
                     exe_path = os.path.join("build/deviceadvisor/tests/",DATestConfig['test_exe_path'][test_name])
                     # Windows and MAC/LINUX has a different build folder structure
                     if platform.system() == 'Windows':
@@ -230,6 +279,7 @@ def sleep_with_backoff(base, max):
                 test_result[test_name] = test_result_responds['status']
                 # If the test failed, upload the logs to S3 before clean up
                 if(test_result[test_name] != "PASS"):
+                    print ("[Device Advisor] Info: About to upload log to S3.")
                     log_url = test_result_responds['testResult']['groups'][0]['tests'][0]['logUrl']
                     group_string = re.search('group=(.*);', log_url)
                     log_group = group_string.group(1)
@@ -238,9 +288,11 @@ def sleep_with_backoff(base, max):
                     process_logs(log_group, log_stream, thing_name)
                 delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
                 break
-    except Exception as e:
+    except Exception:
         delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
         print("[Device Advisor]Error: Failed to test: "+ test_name)
+        did_at_least_one_test_fail = True
+        sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX)
 
 ##############################################
 # print result and cleanup things
@@ -255,4 +307,8 @@ def sleep_with_backoff(base, max):
     # if the test failed, we dont clean the Thing so that we can track the error
     exit(-1)
 
+if (did_at_least_one_test_fail == True):
+    print("[Device Advisor] At least one test failed!")
+    exit(-1)
+
 exit(0)