Shapeup-pure-v5-tasks v3

.circleci/config.yml

@@ -1,25 +1,18 @@
 version: 2
 defaults: &defaults
     docker:
-      - image: docker:18.06.0-ce-git
+      - image: circleci/python:stretch-browsers
 install_dependency: &install_dependency
-          name: Installation of build and deployment dependencies.
-          command: |
-            apk update
-            apk add --no-cache bash
-            apk add --no-cache jq py-pip sudo curl
-            apk upgrade
-            apk add py-pip python-dev libffi-dev openssl-dev gcc libc-dev make 
-            pip install --upgrade pip
-            sudo pip install awscli --upgrade
-            sudo curl -o /usr/local/bin/ecs-cli https://s3.amazonaws.com/amazon-ecs-cli/ecs-cli-linux-amd64-latest
-            sudo pip install docker-compose
-            sudo chmod +x /usr/local/bin/ecs-cli
+    name: Installation of build and deployment dependencies.
+    command: |
+      sudo apt install jq
+      sudo pip3 install awscli --upgrade
+      sudo pip3 install docker-compose      
 
 install_deploysuite: &install_deploysuite
           name: Installation of install_deploysuite.
           command: |
-            git clone --branch v1.3 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+            git clone --branch v1.4.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
             cp ./../buildscript/master_deploy.sh .
             cp ./../buildscript/buildenv.sh .
             cp ./../buildscript/awsconfiguration.sh .
@@ -79,4 +72,4 @@ workflows:
           context : org-global
           filters:
             branches:
-              only: master
+              only: master

README.md

@@ -135,7 +135,6 @@ Please ensure to create the index `submission-test` or the index specified in th
 export ES_INDEX=submission-test
 ```
 
-
 #### Running unit tests and coverage
 
 To run unit tests alone
@@ -172,6 +171,26 @@ To migrate the existing data from DynamoDB to ES, run the following script
 npm run db-to-es
 ```
 
+#### Store v5 challenge id for current records
+
+Submission API started off using the legacy challenge ids. With the v5 upgrade to the challenge api, we now need to make use of the v5 challenge ids. We have thus created a script to update existing `challengeId` attribute on submissions to v5 and store the older challenge ids in the `legacyChallengeId` attribute.
+
+To update the existing challengeId data on submissions in DynamoDB to v5 challengeId, set the following env variables:
+
+```bash
+SUBMISSION_TABLE_NAME // Table name of the submission records. Defaults to 'Submission'
+UPDATE_V5_CHALLENGE_BATCH_SIZE // Number of records that are updated simultaneously. Defaults to 250
+FETCH_CREATED_DATE_START // The start day of fetch latest challenges. Defaults to '2021-01-01'
+FETCH_PAGE_SIZE // The page size of each api request. Defaults to 500
+```
+
+
+and then run the following script
+
+```
+npm run update-to-v5-challengeId
+```
+
 #### Swagger UI
 
 Swagger UI will be served at `http://localhost:3000/docs`

config/default.js

@@ -21,8 +21,8 @@ module.exports = {
   BUSAPI_URL: process.env.BUSAPI_URL || 'https://api.topcoder-dev.com/v5',
   KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC || 'error.notification',
   KAFKA_AGGREGATE_TOPIC: process.env.KAFKA_AGGREGATE_TOPIC || 'submission.notification.aggregate',
-  CHALLENGEAPI_URL: process.env.CHALLENGEAPI_URL || 'https://api.topcoder-dev.com/v4/challenges',
   CHALLENGEAPI_V5_URL: process.env.CHALLENGEAPI_V5_URL || 'https://api.topcoder-dev.com/v5/challenges',
+  RESOURCEAPI_V5_BASE_URL: process.env.RESOURCEAPI_V5_BASE_URL || 'https://api.topcoder-dev.com/v5',
   AUTH0_URL: process.env.AUTH0_URL, // Auth0 credentials for Submission Service
   AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE || 'https://www.topcoder.com',
   TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
@@ -37,5 +37,9 @@ module.exports = {
   PAGE_SIZE: process.env.PAGE_SIZE || 20,
   MAX_PAGE_SIZE: parseInt(process.env.MAX_PAGE_SIZE) || 100,
   ES_BATCH_SIZE: process.env.ES_BATCH_SIZE || 250,
-  AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
+  UPDATE_V5_CHALLENGE_BATCH_SIZE: process.env.UPDATE_V5_CHALLENGE_BATCH_SIZE || 100,
+  SUBMISSION_TABLE_NAME: process.env.SUBMISSION_TABLE_NAME || 'Submission',
+  AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL,
+  FETCH_CREATED_DATE_START: process.env.FETCH_CREATED_DATE_START || '2021-01-01',
+  FETCH_PAGE_SIZE: process.env.FETCH_PAGE_SIZE || 500
 }

config/test.js

@@ -7,7 +7,7 @@ module.exports = {
   LOG_LEVEL: 'info',
   WEB_SERVER_PORT: 3010,
   AUTH_SECRET: 'mysecret',
-  VALID_ISSUERS: '["https://api.topcoder.com"]',
+  VALID_ISSUERS: process.env.VALID_ISSUERS ? process.env.VALID_ISSUERS.replace(/\\"/g, '') : '["https://api.topcoder.com","https://topcoder-dev.auth0.com/"]',
   API_VERSION: process.env.API_VERSION || '/api/v5',
   aws: {
     AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used by the application
@@ -16,14 +16,14 @@ module.exports = {
     S3_BUCKET: process.env.S3_BUCKET_TEST || 'tc-testing-submissions' // S3 Bucket to which submissions need to be uploaded
   },
   BUSAPI_EVENTS_URL: 'https://api.topcoder-dev.com/v5/bus/events',
-  CHALLENGEAPI_URL: 'https://api.topcoder-dev.com/v4/challenges',
+  BUSAPI_URL: 'https://api.topcoder-dev.com/v5',
+  CHALLENGEAPI_V5_URL: 'https://api.topcoder-dev.com/v5/challenges',
   esConfig: {
     ES_INDEX: process.env.ES_INDEX_TEST || 'submission-test',
     ES_TYPE: process.env.ES_TYPE_TEST || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
   },
   AUTH0_URL: process.env.AUTH0_URL, // Auth0 credentials for Submission Service
-  AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE || 'https://www.topcoder.com',
-  TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
+  AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
   AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
   USER_TOKEN: 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIl0sImlzcyI6Imh0dHBzOi8vYXBpLnRvcGNvZGVyLmNvbSIsImhhbmRsZSI6IlNoYXJhdGhrdW1hcjkyIiwiZXhwIjo1NTUzMDE5OTI1OSwidXNlcklkIjoiNDA0OTMwNTAiLCJpYXQiOjE1MzAxOTg2NTksImVtYWlsIjoiU2hhcmF0aGt1bWFyOTJAdG9wY29kZXIuY29tIiwianRpIjoiYzNhYzYwOGEtNTZiZS00NWQwLThmNmEtMzFmZTk0Yjk1NjFjIn0.2gtNJwhcv7MYc-muX3Nv-B0RdWbhMRl7-xrwFUsLazM',

docs/swagger.yaml

@@ -95,6 +95,7 @@ paths:
                 legacySubmissionId: 'a12a4180-65aa-42ec-a945-5fd21dec0502'
                 legacyUploadId: 'a12a4180-65aa-42ec-a945-5fd21dec0502'
                 submissionPhaseId: 764567
+                submittedDate: '2018-05-20T07:00:30.123Z'
                 created: '2018-05-20T07:00:30.123Z'
                 updated: '2018-06-01T07:36:28.178Z'
                 createdBy: 'topcoder user'
@@ -107,6 +108,7 @@ paths:
                 legacySubmissionId: 'a12a4180-65aa-42ec-a945-5fd21dec0502'
                 legacyUploadId: 'a12a4180-65aa-42ec-a945-5fd21dec0502'
                 submissionPhaseId: 764567
+                submittedDate: '2018-05-20T08:00:30.000Z'
                 created: '2018-05-20T08:00:30.000Z'
                 updated: '2018-06-01T09:23:00.178Z'
                 createdBy: 'topcoder user'
@@ -205,6 +207,7 @@ paths:
         Create a new submission.
 
         **Authorization:** Submission creation is accessible by roles `topcoder user`, `admin` and `copilot`.
+        **Note** Value for `submittedDate` attribute can only be provided by users with `admin` role
       tags:
         - Submissions
       operationId: createSubmission
@@ -262,6 +265,11 @@ paths:
           name: submissionPhaseId
           type: integer
           description: Submission Phase Id
+        - in: formData
+          name: submittedDate
+          type: string
+          format: date-time
+          description: Date of submission (defaults to submission creation date if none passed)
       responses:
         201:
           description: Created - The request was successful and the resource is returned.
@@ -612,6 +620,7 @@ paths:
                 scoreCardId: 123456789
                 isPassing: false
                 isFinal: false
+                reviewedDate: '2018-05-20T07:00:30.123Z'
                 created: '2018-05-20T07:00:30.123Z'
                 updated: '2018-06-01T07:36:28.178Z'
                 createdBy: copilot
@@ -622,6 +631,7 @@ paths:
                 scoreCardId: 123456789
                 isPassing: true
                 isFinal: true
+                reviewedDate: '2018-05-20T07:00:30.123Z'
                 created: '2018-05-20T07:00:30.123Z'
                 updated: '2018-06-01T07:36:28.178Z'
                 createdBy: copilot
@@ -718,6 +728,8 @@ paths:
         Create a new review summation.
 
         **Authorization:** Review summation creation is accessible by roles `admin` and `copilot`.
+
+        **Note** Value for `reviewedDate` attribute can only be provided by users with `admin` role
       tags:
         - 'Review summations'
       operationId: createReviewSummation
@@ -931,6 +943,7 @@ paths:
                 scoreCardId: 123456789
                 submissionId: 'd67a4180-65aa-42ec-a945-5fd21dec0503'
                 status: 'queued'
+                reviewedDate: '2018-05-20T07:00:30.123Z'
                 created: '2018-05-20T07:00:30.123Z'
                 updated: '2018-06-01T07:36:28.178Z'
                 createdBy: 'admin'
@@ -942,6 +955,7 @@ paths:
                 scoreCardId: 123456789
                 submissionId: 'd23a4180-65aa-42ec-a945-5fd21dec0503'
                 status: 'completed'
+                reviewedDate: '2018-05-20T07:00:30.123Z'
                 created: '2018-05-20T07:00:30.123Z'
                 updated: '2018-06-01T07:36:28.178Z'
                 createdBy: 'admin'
@@ -1038,6 +1052,8 @@ paths:
         Create a new review.
 
         **Authorization:** Review creation is accessible by roles `admin` and `copilot`.
+
+        **Note** Value for `reviewedDate` attribute can only be provided by users with `admin` role
       tags:
         - 'Reviews'
       operationId: createReview
@@ -1880,6 +1896,11 @@ definitions:
         type: integer
         description: The submission phase id.
         example: '5dea6d9e-161a-4c7a-b316-597c73a7b8f4'
+      submittedDate:
+        type: string
+        format: date-time
+        description: Date of submission (defaults to submission creation date if none passed)
+        example: '2018-05-20T07:00:30.123Z'
 
   UpdatableSubmission:
     description: The submission entity fields that updates whole entity.
@@ -1939,6 +1960,11 @@ definitions:
       metadata:
         type: object
         description: Review summation metadata in JSON format
+      reviewedDate:
+        type: string
+        format: date-time
+        description: Date of review summation (defaults to review summation creation date if none passed)
+        example: '2018-05-20T07:00:30.123Z'
 
   UpdatableReviewSummation:
     description: The review summation entity fields that updates whole entity.
@@ -2006,6 +2032,11 @@ definitions:
       metadata:
         type: object
         description: Review Metadata in JSON format
+      reviewedDate:
+        type: string
+        format: date-time
+        description: Date of review (defaults to review creation date if none passed)
+        example: '2018-05-20T07:00:30.123Z'
 
   UpdatableReview:
     description: The review entity fields that updates whole entity.