Merge pull request #50 from topcoder-platform/dev-wp

[PROD] Release 1.6.0

Author: nkumar-topcoder

.gitignore

@@ -5,3 +5,4 @@ coverage
 .DS_Store
 .env
 api.env
+.eslintrc.y*ml

README.md

@@ -30,6 +30,11 @@ The following parameters can be set in config files or in env variables:
 - `topics.TAAS_RESOURCE_BOOKING_CREATE_TOPIC`: the create resource booking entity Kafka message topic
 - `topics.TAAS_RESOURCE_BOOKING_UPDATE_TOPIC`: the update resource booking entity Kafka message topic
 - `topics.TAAS_RESOURCE_BOOKING_DELETE_TOPIC`: the delete resource booking entity Kafka message topic
+- `topics.TAAS_WORK_PERIOD_CREATE_TOPIC`: the create work period entity Kafka message topic
+- `topics.TAAS_WORK_PERIOD_UPDATE_TOPIC`: the update work period entity Kafka message topic
+- `topics.TAAS_WORK_PERIOD_DELETE_TOPIC`: the delete work period entity Kafka message topic
+- `topics.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC`: the create work period payment entity Kafka message topic
+- `topics.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC`: the update work period payment entity Kafka message topic
 - `esConfig.HOST`: Elasticsearch host
 - `esConfig.AWS_REGION`: The Amazon region to use when using AWS Elasticsearch service
 - `esConfig.ELASTICCLOUD.id`: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this
@@ -38,6 +43,7 @@ The following parameters can be set in config files or in env variables:
 - `esConfig.ES_INDEX_JOB`: the index name for job
 - `esConfig.ES_INDEX_JOB_CANDIDATE`: the index name for job candidate
 - `esConfig.ES_INDEX_RESOURCE_BOOKING`: the index name for resource booking
+- `esConfig.ES_INDEX_WORK_PERIOD`: the index name for work period
 
 - `auth0.AUTH0_URL`: Auth0 URL, used to get TC M2M token
 - `auth0.AUTH0_AUDIENCE`: Auth0 audience, used to get TC M2M token

VERIFICATION.md

@@ -2,7 +2,7 @@
 
 ## Create documents in ES
 
-- Run the following commands to create `Job`, `JobCandidate` and `ResourceBooking` documents in ES.
+- Run the following commands to create `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment` documents in ES.
 
   ``` bash
   # for Job
@@ -11,12 +11,16 @@
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.jobcandidate.create < test/messages/taas.jobcandidate.create.event.json
   # for ResourceBooking
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.create < test/messages/taas.resourcebooking.create.event.json
+  # for WorkPeriod
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.create < test/messages/taas.workperiod.create.event.json
+  # for WorkPeriodPayment
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiodpayment.create < test/messages/taas.workperiodpayment.create.event.json
   ```
 
 - Run `npm run view-data <model-name-here>` to see if documents were created.
 
 ## Update documents in ES
-- Run the following commands to update `Job`, `JobCandidate` and `ResourceBooking` documents in ES.
+- Run the following commands to update `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment` documents in ES.
 
   ``` bash
   # for Job
@@ -25,12 +29,16 @@
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.jobcandidate.update < test/messages/taas.jobcandidate.update.event.json
   # for ResourceBooking
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.update < test/messages/taas.resourcebooking.update.event.json
+  # for WorkPeriod
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.update < test/messages/taas.workperiod.update.event.json
+  # for WorkPeriodPayment
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiodpayment.update < test/messages/taas.workperiodpayment.update.event.json
   ```
 
 - Run `npm run view-data <model-name-here>` to see if documents were updated.
 
 ## Delete documents in ES
-- Run the following commands to delete `Job`, `JobCandidate` and `ResourceBooking` documents in ES.
+- Run the following commands to delete `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES.
 
   ``` bash
   # for Job
@@ -39,6 +47,8 @@
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.jobcandidate.delete < test/messages/taas.jobcandidate.delete.event.json
   # for ResourceBooking
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.delete < test/messages/taas.resourcebooking.delete.event.json
+  # for WorkPeriod
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.delete < test/messages/taas.workperiod.delete.event.json
   ```
 
 - Run `npm run view-data <model-name-here>` to see if documents were deleted.

config/default.js

@@ -16,17 +16,25 @@ module.exports = {
   KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'taas-es-processor',
 
   topics: {
+    // topics for job service
     TAAS_JOB_CREATE_TOPIC: process.env.TAAS_JOB_CREATE_TOPIC || 'taas.job.create',
     TAAS_JOB_UPDATE_TOPIC: process.env.TAAS_JOB_UPDATE_TOPIC || 'taas.job.update',
     TAAS_JOB_DELETE_TOPIC: process.env.TAAS_JOB_DELETE_TOPIC || 'taas.job.delete',
     // topics for jobcandidate service
     TAAS_JOB_CANDIDATE_CREATE_TOPIC: process.env.TAAS_JOB_CANDIDATE_CREATE_TOPIC || 'taas.jobcandidate.create',
     TAAS_JOB_CANDIDATE_UPDATE_TOPIC: process.env.TAAS_JOB_CANDIDATE_UPDATE_TOPIC || 'taas.jobcandidate.update',
     TAAS_JOB_CANDIDATE_DELETE_TOPIC: process.env.TAAS_JOB_CANDIDATE_DELETE_TOPIC || 'taas.jobcandidate.delete',
-    // topics for job service
+    // topics for resource booking service
     TAAS_RESOURCE_BOOKING_CREATE_TOPIC: process.env.TAAS_RESOURCE_BOOKING_CREATE_TOPIC || 'taas.resourcebooking.create',
     TAAS_RESOURCE_BOOKING_UPDATE_TOPIC: process.env.TAAS_RESOURCE_BOOKING_UPDATE_TOPIC || 'taas.resourcebooking.update',
-    TAAS_RESOURCE_BOOKING_DELETE_TOPIC: process.env.TAAS_RESOURCE_BOOKING_DELETE_TOPIC || 'taas.resourcebooking.delete'
+    TAAS_RESOURCE_BOOKING_DELETE_TOPIC: process.env.TAAS_RESOURCE_BOOKING_DELETE_TOPIC || 'taas.resourcebooking.delete',
+    // topics for work period service
+    TAAS_WORK_PERIOD_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_CREATE_TOPIC || 'taas.workperiod.create',
+    TAAS_WORK_PERIOD_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_UPDATE_TOPIC || 'taas.workperiod.update',
+    TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete',
+    // topics for work period payment service
+    TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC || 'taas.workperiodpayment.create',
+    TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC || 'taas.workperiodpayment.update'
   },
 
   esConfig: {
@@ -42,7 +50,8 @@ module.exports = {
 
     ES_INDEX_JOB: process.env.ES_INDEX_JOB || 'job',
     ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate',
-    ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking'
+    ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking',
+    ES_INDEX_WORK_PERIOD: process.env.ES_INDEX_WORK_PERIOD || 'work_period'
   },
 
   auth0: {

local/docker-compose.yml

@@ -12,7 +12,7 @@ services:
       - "9092:9092"
     environment:
       KAFKA_ADVERTISED_HOST_NAME: localhost
-      KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1"
+      KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.workperiod.create:1:1,taas.workperiodpayment.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.workperiod.update:1:1,taas.workperiodpayment.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1,taas.workperiod.delete:1:1"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
   esearch:
     image: elasticsearch:7.7.1

src/app.js

@@ -12,6 +12,8 @@ const helper = require('./common/helper')
 const JobProcessorService = require('./services/JobProcessorService')
 const JobCandidateProcessorService = require('./services/JobCandidateProcessorService')
 const ResourceBookingProcessorService = require('./services/ResourceBookingProcessorService')
+const WorkPeriodProcessorService = require('./services/WorkPeriodProcessorService')
+const WorkPeriodPaymentProcessorService = require('./services/WorkPeriodPaymentProcessorService')
 const Mutex = require('async-mutex').Mutex
 const events = require('events')
 
@@ -38,7 +40,14 @@ const topicServiceMapping = {
   // resource booking
   [config.topics.TAAS_RESOURCE_BOOKING_CREATE_TOPIC]: ResourceBookingProcessorService.processCreate,
   [config.topics.TAAS_RESOURCE_BOOKING_UPDATE_TOPIC]: ResourceBookingProcessorService.processUpdate,
-  [config.topics.TAAS_RESOURCE_BOOKING_DELETE_TOPIC]: ResourceBookingProcessorService.processDelete
+  [config.topics.TAAS_RESOURCE_BOOKING_DELETE_TOPIC]: ResourceBookingProcessorService.processDelete,
+  // work period
+  [config.topics.TAAS_WORK_PERIOD_CREATE_TOPIC]: WorkPeriodProcessorService.processCreate,
+  [config.topics.TAAS_WORK_PERIOD_UPDATE_TOPIC]: WorkPeriodProcessorService.processUpdate,
+  [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete,
+  // work period payment
+  [config.topics.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC]: WorkPeriodPaymentProcessorService.processCreate,
+  [config.topics.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC]: WorkPeriodPaymentProcessorService.processUpdate
 }
 
 // Start kafka consumer

src/bootstrap.js

@@ -6,9 +6,12 @@ global.Promise = require('bluebird')
 
 Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly')
 Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled')
+Joi.resourceBookingStatus = () => Joi.string().valid('assigned', 'closed', 'cancelled')
 Joi.jobCandidateStatus = () => Joi.string().valid('open', 'selected', 'shortlist', 'rejected', 'cancelled', 'interview', 'topcoder-rejected')
 Joi.workload = () => Joi.string().valid('full-time', 'fractional')
 Joi.title = () => Joi.string().max(128)
+Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled')
+Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'cancelled')
 // Empty string is not allowed by Joi by default and must be enabled with allow('').
 // See https://joi.dev/api/?v=17.3.0#string fro details why it's like this.
 // In many cases we would like to allow empty string to make it easier to create UI for editing data.

src/scripts/createIndex.js

@@ -63,11 +63,49 @@ async function createIndex () {
             userId: { type: 'keyword' },
             jobId: { type: 'keyword' },
             status: { type: 'keyword' },
-            startDate: { type: 'date' },
-            endDate: { type: 'date' },
+            startDate: { type: 'date', format: 'yyyy-MM-dd' },
+            endDate: { type: 'date', format: 'yyyy-MM-dd' },
             memberRate: { type: 'float' },
             customerRate: { type: 'float' },
             rateType: { type: 'keyword' },
+            billingAccountId: { type: 'integer' },
+            createdAt: { type: 'date' },
+            createdBy: { type: 'keyword' },
+            updatedAt: { type: 'date' },
+            updatedBy: { type: 'keyword' }
+          }
+        }
+      }
+    },
+    {
+      index: config.get('esConfig.ES_INDEX_WORK_PERIOD'),
+      body: {
+        mappings: {
+          properties: {
+            resourceBookingId: { type: 'keyword' },
+            userHandle: { type: 'keyword' },
+            projectId: { type: 'integer' },
+            userId: { type: 'keyword' },
+            startDate: { type: 'date', format: 'yyyy-MM-dd' },
+            endDate: { type: 'date', format: 'yyyy-MM-dd' },
+            daysWorked: { type: 'integer' },
+            memberRate: { type: 'float' },
+            customerRate: { type: 'float' },
+            paymentStatus: { type: 'keyword' },
+            payments: {
+              type: 'nested',
+              properties: {
+                workPeriodId: { type: 'keyword' },
+                challengeId: { type: 'keyword' },
+                amount: { type: 'float' },
+                status: { type: 'keyword' },
+                billingAccountId: { type: 'integer' },
+                createdAt: { type: 'date' },
+                createdBy: { type: 'keyword' },
+                updatedAt: { type: 'date' },
+                updatedBy: { type: 'keyword' }
+              }
+            },
             createdAt: { type: 'date' },
             createdBy: { type: 'keyword' },
             updatedAt: { type: 'date' },

src/scripts/deleteIndex.js

@@ -11,7 +11,8 @@ async function deleteIndex () {
   const esClient = helper.getESClient()
   const indices = [config.get('esConfig.ES_INDEX_JOB'),
     config.get('esConfig.ES_INDEX_JOB_CANDIDATE'),
-    config.get('esConfig.ES_INDEX_RESOURCE_BOOKING')]
+    config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'),
+    config.get('esConfig.ES_INDEX_WORK_PERIOD')]
   for (const index of indices) {
     await esClient.indices.delete({
       index

src/scripts/view-data.js

@@ -11,7 +11,8 @@ const esClient = helper.getESClient()
 const modelIndexMapping = {
   Job: 'ES_INDEX_JOB',
   JobCandidate: 'ES_INDEX_JOB_CANDIDATE',
-  ResourceBooking: 'ES_INDEX_RESOURCE_BOOKING'
+  ResourceBooking: 'ES_INDEX_RESOURCE_BOOKING',
+  WorkPeriod: 'ES_INDEX_WORK_PERIOD'
 }
 
 async function showESData () {

src/services/ResourceBookingProcessorService.js

@@ -37,16 +37,17 @@ processCreate.schema = {
       projectId: Joi.number().integer().required(),
       userId: Joi.string().uuid().required(),
       jobId: Joi.string().uuid().allow(null),
-      startDate: Joi.date().allow(null),
-      endDate: Joi.date().allow(null),
+      startDate: Joi.string().regex(/^(19|20)\d\d-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$/).allow(null),
+      endDate: Joi.string().regex(/^(19|20)\d\d-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$/).allow(null),
       memberRate: Joi.number().allow(null),
       customerRate: Joi.number().allow(null),
       rateType: Joi.rateType().required(),
       createdAt: Joi.date().required(),
       createdBy: Joi.string().uuid().required(),
       updatedAt: Joi.date().allow(null),
       updatedBy: Joi.string().uuid().allow(null),
-      status: Joi.jobStatus().required()
+      status: Joi.resourceBookingStatus().required(),
+      billingAccountId: Joi.number().allow(null)
     }).required()
   }).required(),
   transactionId: Joi.string().required()

src/services/WorkPeriodPaymentProcessorService.js

@@ -0,0 +1,138 @@
+/**
+ * WorkPeriodPayment Processor Service
+ */
+
+const Joi = require('@hapi/joi')
+const config = require('config')
+const _ = require('lodash')
+const logger = require('../common/logger')
+const helper = require('../common/helper')
+const constants = require('../common/constants')
+
+const esClient = helper.getESClient()
+
+/**
+ * Process create entity message
+ * @param {Object} message the kafka message
+ * @param {String} transactionId
+ */
+async function processCreate (message, transactionId) {
+  const data = message.payload
+  const workPeriod = await esClient.getExtra({
+    index: config.get('esConfig.ES_INDEX_WORK_PERIOD'),
+    id: data.workPeriodId
+  })
+  const payments = _.isArray(workPeriod.body.payments) ? workPeriod.body.payments : []
+  payments.push(data)
+
+  return esClient.updateExtra({
+    index: config.get('esConfig.ES_INDEX_WORK_PERIOD'),
+    id: data.workPeriodId,
+    transactionId,
+    body: {
+      doc: _.assign(workPeriod.body, { payments })
+    },
+    refresh: constants.esRefreshOption
+  })
+}
+
+processCreate.schema = {
+  message: Joi.object().keys({
+    topic: Joi.string().required(),
+    originator: Joi.string().required(),
+    timestamp: Joi.date().required(),
+    'mime-type': Joi.string().required(),
+    payload: Joi.object().keys({
+      id: Joi.string().uuid().required(),
+      workPeriodId: Joi.string().uuid().required(),
+      challengeId: Joi.string().uuid().required(),
+      amount: Joi.number().greater(0).allow(null),
+      status: Joi.workPeriodPaymentStatus().required(),
+      billingAccountId: Joi.number().allow(null),
+      createdAt: Joi.date().required(),
+      createdBy: Joi.string().uuid().required(),
+      updatedAt: Joi.date().allow(null),
+      updatedBy: Joi.string().uuid().allow(null)
+    }).required()
+  }).required(),
+  transactionId: Joi.string().required()
+}
+
+/**
+ * Process update entity message
+ * @param {Object} message the kafka message
+ * @param {String} transactionId
+ */
+async function processUpdate (message, transactionId) {
+  const data = message.payload
+  let workPeriod = await esClient.search({
+    index: config.get('esConfig.ES_INDEX_WORK_PERIOD'),
+    body: {
+      query: {
+        nested: {
+          path: 'payments',
+          query: {
+            match: { 'payments.id': data.id }
+          }
+        }
+      }
+    }
+  })
+  if (!workPeriod.body.hits.total.value) {
+    throw new Error(`id: ${data.id} "WorkPeriodPayments" not found`)
+  }
+  let payments
+  // if WorkPeriodPayment's workPeriodId changed then it must be deleted from the old WorkPeriod
+  // and added to the new WorkPeriod
+  if (workPeriod.body.hits.hits[0]._source.id !== data.workPeriodId) {
+    payments = _.filter(workPeriod.body.hits.hits[0]._source.payments, (payment) => payment.id !== data.id)
+    await esClient.updateExtra({
+      index: config.get('esConfig.ES_INDEX_WORK_PERIOD'),
+      id: workPeriod.body.hits.hits[0]._source.id,
+      transactionId,
+      body: {
+        doc: _.assign(workPeriod.body.hits.hits[0]._source, { payments })
+      }
+    })
+    workPeriod = await esClient.getExtra({
+      index: config.get('esConfig.ES_INDEX_WORK_PERIOD'),
+      id: data.workPeriodId
+    })
+    payments = _.isArray(workPeriod.body.payments) ? workPeriod.body.payments : []
+    payments.push(data)
+    return esClient.updateExtra({
+      index: config.get('esConfig.ES_INDEX_WORK_PERIOD'),
+      id: data.workPeriodId,
+      transactionId,
+      body: {
+        doc: _.assign(workPeriod.body, { payments })
+      }
+    })
+  }
+
+  payments = _.map(workPeriod.body.hits.hits[0]._source.payments, (payment) => {
+    if (payment.id === data.id) {
+      return _.assign(payment, data)
+    }
+    return payment
+  })
+
+  return esClient.updateExtra({
+    index: config.get('esConfig.ES_INDEX_WORK_PERIOD'),
+    id: data.workPeriodId,
+    transactionId,
+    body: {
+      doc: _.assign(workPeriod.body.hits.hits[0]._source, { payments })
+    },
+    refresh: constants.esRefreshOption
+  })
+}
+
+processUpdate.schema = processCreate.schema
+
+module.exports = {
+  processCreate,
+  processUpdate
+}
+
+logger.buildService(module.exports, 'WorkPeriodPaymentProcessorService')