migrate v4 challenge endpoints to v5 challenge endpoints

README.md

@@ -171,6 +171,24 @@ To migrate the existing data from DynamoDB to ES, run the following script
 npm run db-to-es
 ```
 
+#### Store v5 challenge id for current records
+
+Submission API started off using the legacy challenge ids. With the v5 upgrade to the challenge api, we now need to make use of the v5 challenge ids. We have thus created a script to update existing `challengeId` attribute on submissions to v5 and store the older challenge ids in the `legacyChallengeId` attribute.
+
+To update the existing challengeId data on submissions in DynamoDB to v5 challengeId, set the following env variables:
+
+```bash
+SUBMISSION_TABLE_NAME // Table name of the submission records. Defaults to 'Submission'
+UPDATE_V5_CHALLENGE_BATCH_SIZE // Number of records that are updated simultaneously. Defaults to 250
+```
+
+
+and then run the following script
+
+```
+npm run update-to-v5-challengeId
+```
+
 #### Swagger UI
 
 Swagger UI will be served at `http://localhost:3000/docs`

config/default.js

@@ -21,8 +21,8 @@ module.exports = {
   BUSAPI_URL: process.env.BUSAPI_URL || 'https://api.topcoder-dev.com/v5',
   KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC || 'error.notification',
   KAFKA_AGGREGATE_TOPIC: process.env.KAFKA_AGGREGATE_TOPIC || 'submission.notification.aggregate',
-  CHALLENGEAPI_URL: process.env.CHALLENGEAPI_URL || 'https://api.topcoder-dev.com/v4/challenges',
   CHALLENGEAPI_V5_URL: process.env.CHALLENGEAPI_V5_URL || 'https://api.topcoder-dev.com/v5/challenges',
+  RESOURCEAPI_V5_BASE_URL: process.env.RESOURCEAPI_V5_BASE_URL || 'https://api.topcoder-dev.com/v5',
   AUTH0_URL: process.env.AUTH0_URL, // Auth0 credentials for Submission Service
   AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE || 'https://www.topcoder.com',
   TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
@@ -37,5 +37,7 @@ module.exports = {
   PAGE_SIZE: process.env.PAGE_SIZE || 20,
   MAX_PAGE_SIZE: parseInt(process.env.MAX_PAGE_SIZE) || 100,
   ES_BATCH_SIZE: process.env.ES_BATCH_SIZE || 250,
+  UPDATE_V5_CHALLENGE_BATCH_SIZE: process.env.UPDATE_V5_CHALLENGE_BATCH_SIZE || 250,
+  SUBMISSION_TABLE_NAME: process.env.SUBMISSION_TABLE_NAME || 'Submission',
   AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
 }

config/test.js

@@ -7,7 +7,7 @@ module.exports = {
   LOG_LEVEL: 'info',
   WEB_SERVER_PORT: 3010,
   AUTH_SECRET: 'mysecret',
-  VALID_ISSUERS: '["https://api.topcoder.com"]',
+  VALID_ISSUERS: process.env.VALID_ISSUERS ? process.env.VALID_ISSUERS.replace(/\\"/g, '') : '["https://api.topcoder.com","https://topcoder-dev.auth0.com/"]',
   API_VERSION: process.env.API_VERSION || '/api/v5',
   aws: {
     AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used by the application
@@ -16,14 +16,14 @@ module.exports = {
     S3_BUCKET: process.env.S3_BUCKET_TEST || 'tc-testing-submissions' // S3 Bucket to which submissions need to be uploaded
   },
   BUSAPI_EVENTS_URL: 'https://api.topcoder-dev.com/v5/bus/events',
-  CHALLENGEAPI_URL: 'https://api.topcoder-dev.com/v4/challenges',
+  BUSAPI_URL: 'https://api.topcoder-dev.com/v5',
+  CHALLENGEAPI_V5_URL: 'https://api.topcoder-dev.com/v5/challenges',
   esConfig: {
     ES_INDEX: process.env.ES_INDEX_TEST || 'submission-test',
     ES_TYPE: process.env.ES_TYPE_TEST || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
   },
   AUTH0_URL: process.env.AUTH0_URL, // Auth0 credentials for Submission Service
-  AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE || 'https://www.topcoder.com',
-  TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
+  AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
   AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
   USER_TOKEN: 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIl0sImlzcyI6Imh0dHBzOi8vYXBpLnRvcGNvZGVyLmNvbSIsImhhbmRsZSI6IlNoYXJhdGhrdW1hcjkyIiwiZXhwIjo1NTUzMDE5OTI1OSwidXNlcklkIjoiNDA0OTMwNTAiLCJpYXQiOjE1MzAxOTg2NTksImVtYWlsIjoiU2hhcmF0aGt1bWFyOTJAdG9wY29kZXIuY29tIiwianRpIjoiYzNhYzYwOGEtNTZiZS00NWQwLThmNmEtMzFmZTk0Yjk1NjFjIn0.2gtNJwhcv7MYc-muX3Nv-B0RdWbhMRl7-xrwFUsLazM',

package.json

@@ -14,6 +14,7 @@
     "delete-index": "node scripts/deleteIndex.js",
     "init-es": "node scripts/loadES.js",
     "db-to-es": "node scripts/migrateFromDBToES.js",
+    "update-to-v5-challengeId": "node scripts/updateToV5ChallengeId.js",
     "test": "mocha test/unit/*.test.js --require test/unit/prepare.js --exit",
     "e2e": "mocha test/e2e/*.test.js --require test/e2e/prepare.js --exit",
     "cov": "nyc --reporter=html --reporter=text mocha test/unit/*.test.js --require test/unit/prepare.js --exit",

scripts/createIndex.js

@@ -18,6 +18,7 @@ co(function * createIndex () {
     properties: {
       resource: { type: 'keyword' },
       challengeId: { type: 'keyword' },
+      legacyChallengeId: { type: 'keyword' },
       memberId: { type: 'keyword' },
       type: { type: 'keyword' },
       isFileSubmission: { type: 'boolean' },

scripts/updateToV5ChallengeId.js

@@ -0,0 +1,78 @@
+/**
+ * Store v5 challenge id for current records
+ */
+
+const _ = require('lodash')
+const co = require('co')
+const config = require('config')
+const logger = require('../src/common/logger')
+const dbhelper = require('../src/common/dbhelper')
+const helper = require('../src/common/helper')
+
+/**
+ * Update Submission's challenge id to v5
+ * @param {Object} submission The submission record
+ * @returns {Promise}
+ */
+function * updateRecord (submission) {
+  const v5challengeId = yield helper.getV5ChallengeId(submission.challengeId)
+  const record = {
+    TableName: 'Submission',
+    Key: {
+      id: submission.id
+    },
+    UpdateExpression: `set challengeId = :c, legacyChallengeId = :l`,
+    ExpressionAttributeValues: {
+      ':c': v5challengeId,
+      ':l': submission.challengeId
+    }
+  }
+  if (!v5challengeId) {
+    logger.warn(`the challengeId: ${submission.challengeId} is not having a v5 challengeId`)
+
+    return
+  } else if (v5challengeId === submission.challengeId) {
+    logger.info(`the challengeId: ${submission.challengeId} is already a v5 challengeId`)
+  }
+
+  yield dbhelper.updateRecord(record)
+}
+
+/*
+ * Update all submission's challenge id to v5
+ * @returns {Promise}
+ */
+function * updateRecords () {
+  const tableName = config.SUBMISSION_TABLE_NAME
+  let promises = []
+  const params = {
+    TableName: tableName
+  }
+  // Process until all the records from DB is fetched
+  while (true) {
+    const records = yield dbhelper.scanRecords(params)
+    const totalRecords = records.Items.length
+    logger.debug(`Number of ${tableName}s fetched from DB - ${totalRecords}. More fetch iterations may follow (pagination in progress)`)
+    for (let i = 0; i < totalRecords; i++) {
+      const record = records.Items[i]
+      promises.push(updateRecord(record))
+    }
+    // Continue fetching the remaining records from Database
+    if (typeof records.LastEvaluatedKey !== 'undefined') {
+      params.ExclusiveStartKey = records.LastEvaluatedKey
+    } else {
+      break // If there are no more records to process, exit the loop
+    }
+  }
+  logger.debug(`All records fetched. Proceeding to update them in batches of ${config.UPDATE_V5_CHALLENGE_BATCH_SIZE}`)
+  const paraRecords = _.chunk(promises, config.UPDATE_V5_CHALLENGE_BATCH_SIZE)
+  for (const rs of paraRecords) {
+    yield rs
+  }
+}
+
+co(function * () {
+  yield updateRecords()
+}).catch((err) => {
+  logger.logFullError(err)
+})