From 63c6ddcb16d28d135b727fbee9e379b1a802c5c7 Mon Sep 17 00:00:00 2001 From: shuffledex Date: Thu, 29 Nov 2018 10:26:24 -0300 Subject: [PATCH 01/12] moving multi mint to token manager --- CLI/commands/multi_mint.js | 197 ---------------------------------- CLI/commands/token_manager.js | 192 ++++++++++++++++++++++++++++++++- 2 files changed, 190 insertions(+), 199 deletions(-) delete mode 100644 CLI/commands/multi_mint.js diff --git a/CLI/commands/multi_mint.js b/CLI/commands/multi_mint.js deleted file mode 100644 index 3147b2754..000000000 --- a/CLI/commands/multi_mint.js +++ /dev/null @@ -1,197 +0,0 @@ -var common = require('./common/common_functions'); -var csv_shared = require('./common/csv_shared'); -var abis = require('./helpers/contract_abis'); -var BigNumber = require('bignumber.js'); - -let distribData = new Array(); -let fullFileData = new Array(); -let badData = new Array(); -let affiliatesFailedArray = new Array(); -let affiliatesKYCInvalidArray = new Array(); - -let securityToken; -let tokenDivisible; - -async function startScript(tokenSymbol, batchSize) { - securityToken = await csv_shared.start(tokenSymbol, batchSize); - - let result_processing = await csv_shared.read('./CLI/data/multi_mint_data.csv', multimint_processing); - distribData = result_processing.distribData; - fullFileData = result_processing.fullFileData; - badData = result_processing.badData; - - tokenDivisible = await securityToken.methods.granularity().call() == 1; - - await saveInBlockchain(); - await finalResults(); -} - -function multimint_processing(csv_line) { - let isAddress = web3.utils.isAddress(csv_line[0]); - let validToken = isValidToken(csv_line[1]); - - if (isAddress && - validToken) { - return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), validToken)] - } else { - return [false, new Array(csv_line[0], csv_line[1])] - } -} - -function isValidToken(token) { - var tokenAmount = parseFloat(token); - if (tokenDivisible) { - return tokenAmount - } else { - if ((tokenAmount % 1 == 0)) { - return tokenAmount; - } - return false - } -} - -async function saveInBlockchain() { - let gtmModules = await securityToken.methods.getModulesByType(3).call(); - - if (gtmModules.length > 0) { - console.log("Minting of tokens is only allowed before the STO get attached"); - process.exit(0); - } - - console.log(` - ----------------------------------------- - ----- Mint the tokens to affiliates ----- - ----------------------------------------- - `); - - for (let i = 0; i < distribData.length; i++) { - try { - let affiliatesVerifiedArray = [], tokensVerifiedArray = []; - - // Splitting the user arrays to be organized by input - for (let j = 0; j < distribData[i].length; j++) { - let investorAccount = distribData[i][j][0]; - let tokenAmount = web3.utils.toWei((distribData[i][j][1]).toString(), "ether"); - let verifiedTransaction = await securityToken.methods.verifyTransfer("0x0000000000000000000000000000000000000000", investorAccount, tokenAmount, web3.utils.fromAscii('')).call(); - if (verifiedTransaction) { - affiliatesVerifiedArray.push(investorAccount); - tokensVerifiedArray.push(tokenAmount); - } else { - let gtmModule = await securityToken.methods.getModulesByName(web3.utils.toHex('GeneralTransferManager')).call(); - let generalTransferManager = new web3.eth.Contract(abis.generalTransferManager(), gtmModule[0]); - let validKYC = (await generalTransferManager.methods.whitelist(Issuer.address).call()).expiryTime > Math.floor(Date.now()/1000); - if (validKYC) { - affiliatesFailedArray.push(investorAccount); - } else { - affiliatesKYCInvalidArray.push(investorAccount); - } - } - } - - let mintMultiAction = await securityToken.methods.mintMulti(affiliatesVerifiedArray, tokensVerifiedArray); - let tx = await common.sendTransaction(mintMultiAction); - console.log(`Batch ${i} - Attempting to send the Minted tokens to affiliates accounts:\n\n`, affiliatesVerifiedArray, "\n\n"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); - console.log("Multi Mint transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - - } catch (err) { - console.log("ERROR", err) - process.exit(0) - } - } - - return; -} - -async function finalResults() { - let totalInvestors = 0; - let updatedInvestors = 0; - let investorObjectLookup = {}; - let investorData_Events = new Array(); - - let event_data = await securityToken.getPastEvents('Minted', {fromBlock: 0, toBlock: 'latest'}, () => {}); - - for (var i = 0; i < event_data.length; i++) { - let combineArray = []; - - let investorAddress_Event = event_data[i].returnValues._to; - let amount_Event = event_data[i].returnValues._value; - let blockNumber = event_data[i].blockNumber; - - combineArray.push(investorAddress_Event); - combineArray.push(amount_Event); - combineArray.push(blockNumber); - - investorData_Events.push(combineArray) - - // We have already recorded it, so this is an update to our object - if (investorObjectLookup.hasOwnProperty(investorAddress_Event)) { - // The block number form the event we are checking is bigger, so we gotta replace it - if (investorObjectLookup[investorAddress_Event].recordedBlockNumber < blockNumber) { - investorObjectLookup[investorAddress_Event] = {amount: amount_Event, recordedBlockNumber: blockNumber}; - updatedInvestors += 1; - } - } else { - investorObjectLookup[investorAddress_Event] = {amount: amount_Event, recordedBlockNumber: blockNumber}; - totalInvestors += 1; - } - } - - let investorAddress_Events = Object.keys(investorObjectLookup) - - console.log(`******************** EVENT LOGS ANALYSIS COMPLETE ********************\n`); - console.log(`A total of ${totalInvestors} affiliated investors get the token\n`); - console.log(`This script in total sent ${fullFileData.length - badData.length - affiliatesFailedArray.length - affiliatesKYCInvalidArray.length} new investors and updated investors to the blockchain.\n`); - console.log(`There were ${badData.length} bad entries that didnt get sent to the blockchain in the script.\n`); - console.log(`There were ${affiliatesKYCInvalidArray.length} accounts with invalid KYC.\n`); - console.log(`There were ${affiliatesFailedArray.length} accounts that didn't get sent to the blockchain as they would fail.\n`); - console.log("************************************************************************************************"); - console.log("OBJECT WITH EVERY USER AND THEIR MINTED TOKEN: \n\n", investorObjectLookup) - console.log("************************************************************************************************"); - console.log("LIST OF ALL INVESTORS WHO GOT THE MINTED TOKENS: \n\n", investorAddress_Events) - - let missingDistribs = [], failedVerificationDistribs = [], invalidKYCDistribs = []; - for (let l = 0; l < fullFileData.length; l++) { - if (affiliatesKYCInvalidArray.includes(fullFileData[l][0])) { - invalidKYCDistribs.push(fullFileData[l]); - } else if (affiliatesFailedArray.includes(fullFileData[l][0])) { - failedVerificationDistribs.push(fullFileData[l]); - } else if (!investorObjectLookup.hasOwnProperty(fullFileData[l][0])) { - missingDistribs.push(fullFileData[l]); - } - } - - if (invalidKYCDistribs.length > 0) { - console.log("**************************************************************************************************************************"); - console.log("The following data arrays have an invalid KYC. Please review if these accounts are whitelisted and their KYC is not expired\n"); - console.log(invalidKYCDistribs); - console.log("**************************************************************************************************************************"); - } - if (failedVerificationDistribs.length > 0) { - console.log("*********************************************************************************************************"); - console.log("-- The following data arrays failed at verifyTransfer. Please review if these accounts are whitelisted --\n"); - console.log(failedVerificationDistribs); - console.log("*********************************************************************************************************"); - } - if (missingDistribs.length > 0) { - console.log("******************************************************************************************"); - console.log("-- No Minted event was found for the following data arrays. Please review them manually --\n"); - console.log(missingDistribs); - console.log("******************************************************************************************"); - } - if (missingDistribs.length == 0 && - failedVerificationDistribs.length == 0 && - invalidKYCDistribs.length == 0) { - console.log("\n**************************************************************************************************************************"); - console.log("All accounts passed through from the CSV were successfully get the tokens, because we were able to read them all from events"); - console.log("****************************************************************************************************************************"); - } - -} - -module.exports = { - executeApp: async (tokenSymbol, batchSize) => { - return startScript(tokenSymbol, batchSize); - } -} \ No newline at end of file diff --git a/CLI/commands/token_manager.js b/CLI/commands/token_manager.js index 6faa7ba34..0bbdad729 100644 --- a/CLI/commands/token_manager.js +++ b/CLI/commands/token_manager.js @@ -2,10 +2,11 @@ const readlineSync = require('readline-sync'); const chalk = require('chalk'); const whitelist = require('./whitelist'); -const multimint = require('./multi_mint'); const stoManager = require('./sto_manager'); const common = require('./common/common_functions'); const gbl = require('./common/global'); +const csv_shared = require('./common/csv_shared'); +const BigNumber = require('bignumber.js'); // Load contract artifacts const contracts = require('./helpers/contract_addresses'); @@ -15,10 +16,19 @@ let securityTokenRegistry; let polyToken; let featureRegistry; let securityToken; +let tokenDivisible; let allModules; let tokenSymbol +/* CSV data control */ +let distribData = new Array(); +let fullFileData = new Array(); +let badData = new Array(); +let affiliatesFailedArray = new Array(); +let affiliatesKYCInvalidArray = new Array(); +/* End CSV data control */ + async function setup() { try { let securityTokenRegistryAddress = await contracts.securityTokenRegistry(); @@ -340,11 +350,189 @@ async function multi_mint_tokens() { console.log(chalk.green(`\nCongratulations! All the affiliates get succssfully whitelisted, Now its time to Mint the tokens\n`)); console.log(chalk.red(`WARNING: `) + `Please make sure all the addresses that get whitelisted are only eligible to hold or get Security token\n`); - await multimint.executeApp(tokenSymbol, 75); + await startCSV(tokenSymbol, 75) console.log(chalk.green(`\nCongratulations! Tokens get successfully Minted and transferred to token holders`)); } /// +async function startCSV(tokenSymbol, batchSize) { + securityToken = await csv_shared.start(tokenSymbol, batchSize); + + let result_processing = await csv_shared.read('./CLI/data/multi_mint_data.csv', multimint_processing); + distribData = result_processing.distribData; + fullFileData = result_processing.fullFileData; + badData = result_processing.badData; + + tokenDivisible = await securityToken.methods.granularity().call() == 1; + + await saveInBlockchain(); + await finalResults(); +} + +function multimint_processing(csv_line) { + let isAddress = web3.utils.isAddress(csv_line[0]); + let validToken = isValidToken(csv_line[1]); + + if (isAddress && + validToken) { + return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), validToken)] + } else { + return [false, new Array(csv_line[0], csv_line[1])] + } +} + +function isValidToken(token) { + var tokenAmount = parseFloat(token); + if (tokenDivisible) { + return tokenAmount + } else { + if ((tokenAmount % 1 == 0)) { + return tokenAmount; + } + return false + } +} + +async function saveInBlockchain() { + let gtmModules = await securityToken.methods.getModulesByType(3).call(); + + if (gtmModules.length > 0) { + console.log("Minting of tokens is only allowed before the STO get attached"); + process.exit(0); + } + + console.log(` + ----------------------------------------- + ----- Mint the tokens to affiliates ----- + ----------------------------------------- + `); + + for (let i = 0; i < distribData.length; i++) { + try { + let affiliatesVerifiedArray = [], tokensVerifiedArray = []; + + // Splitting the user arrays to be organized by input + for (let j = 0; j < distribData[i].length; j++) { + let investorAccount = distribData[i][j][0]; + let tokenAmount = web3.utils.toWei((distribData[i][j][1]).toString(), "ether"); + let verifiedTransaction = await securityToken.methods.verifyTransfer("0x0000000000000000000000000000000000000000", investorAccount, tokenAmount, web3.utils.fromAscii('')).call(); + if (verifiedTransaction) { + affiliatesVerifiedArray.push(investorAccount); + tokensVerifiedArray.push(tokenAmount); + } else { + let gtmModule = await securityToken.methods.getModulesByName(web3.utils.toHex('GeneralTransferManager')).call(); + let generalTransferManager = new web3.eth.Contract(abis.generalTransferManager(), gtmModule[0]); + let validKYC = (await generalTransferManager.methods.whitelist(Issuer.address).call()).expiryTime > Math.floor(Date.now()/1000); + if (validKYC) { + affiliatesFailedArray.push(investorAccount); + } else { + affiliatesKYCInvalidArray.push(investorAccount); + } + } + } + + let mintMultiAction = await securityToken.methods.mintMulti(affiliatesVerifiedArray, tokensVerifiedArray); + let tx = await common.sendTransaction(mintMultiAction); + console.log(`Batch ${i} - Attempting to send the Minted tokens to affiliates accounts:\n\n`, affiliatesVerifiedArray, "\n\n"); + console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); + console.log("Multi Mint transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); + console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); + + } catch (err) { + console.log("ERROR", err) + process.exit(0) + } + } + + return; +} + +async function finalResults() { + let totalInvestors = 0; + let updatedInvestors = 0; + let investorObjectLookup = {}; + let investorData_Events = new Array(); + + let event_data = await securityToken.getPastEvents('Minted', {fromBlock: 0, toBlock: 'latest'}, () => {}); + + for (var i = 0; i < event_data.length; i++) { + let combineArray = []; + + let investorAddress_Event = event_data[i].returnValues._to; + let amount_Event = event_data[i].returnValues._value; + let blockNumber = event_data[i].blockNumber; + + combineArray.push(investorAddress_Event); + combineArray.push(amount_Event); + combineArray.push(blockNumber); + + investorData_Events.push(combineArray) + + // We have already recorded it, so this is an update to our object + if (investorObjectLookup.hasOwnProperty(investorAddress_Event)) { + // The block number form the event we are checking is bigger, so we gotta replace it + if (investorObjectLookup[investorAddress_Event].recordedBlockNumber < blockNumber) { + investorObjectLookup[investorAddress_Event] = {amount: amount_Event, recordedBlockNumber: blockNumber}; + updatedInvestors += 1; + } + } else { + investorObjectLookup[investorAddress_Event] = {amount: amount_Event, recordedBlockNumber: blockNumber}; + totalInvestors += 1; + } + } + + let investorAddress_Events = Object.keys(investorObjectLookup) + + console.log(`******************** EVENT LOGS ANALYSIS COMPLETE ********************\n`); + console.log(`A total of ${totalInvestors} affiliated investors get the token\n`); + console.log(`This script in total sent ${fullFileData.length - badData.length - affiliatesFailedArray.length - affiliatesKYCInvalidArray.length} new investors and updated investors to the blockchain.\n`); + console.log(`There were ${badData.length} bad entries that didnt get sent to the blockchain in the script.\n`); + console.log(`There were ${affiliatesKYCInvalidArray.length} accounts with invalid KYC.\n`); + console.log(`There were ${affiliatesFailedArray.length} accounts that didn't get sent to the blockchain as they would fail.\n`); + console.log("************************************************************************************************"); + console.log("OBJECT WITH EVERY USER AND THEIR MINTED TOKEN: \n\n", investorObjectLookup) + console.log("************************************************************************************************"); + console.log("LIST OF ALL INVESTORS WHO GOT THE MINTED TOKENS: \n\n", investorAddress_Events) + + let missingDistribs = [], failedVerificationDistribs = [], invalidKYCDistribs = []; + for (let l = 0; l < fullFileData.length; l++) { + if (affiliatesKYCInvalidArray.includes(fullFileData[l][0])) { + invalidKYCDistribs.push(fullFileData[l]); + } else if (affiliatesFailedArray.includes(fullFileData[l][0])) { + failedVerificationDistribs.push(fullFileData[l]); + } else if (!investorObjectLookup.hasOwnProperty(fullFileData[l][0])) { + missingDistribs.push(fullFileData[l]); + } + } + + if (invalidKYCDistribs.length > 0) { + console.log("**************************************************************************************************************************"); + console.log("The following data arrays have an invalid KYC. Please review if these accounts are whitelisted and their KYC is not expired\n"); + console.log(invalidKYCDistribs); + console.log("**************************************************************************************************************************"); + } + if (failedVerificationDistribs.length > 0) { + console.log("*********************************************************************************************************"); + console.log("-- The following data arrays failed at verifyTransfer. Please review if these accounts are whitelisted --\n"); + console.log(failedVerificationDistribs); + console.log("*********************************************************************************************************"); + } + if (missingDistribs.length > 0) { + console.log("******************************************************************************************"); + console.log("-- No Minted event was found for the following data arrays. Please review them manually --\n"); + console.log(missingDistribs); + console.log("******************************************************************************************"); + } + if (missingDistribs.length == 0 && + failedVerificationDistribs.length == 0 && + invalidKYCDistribs.length == 0) { + console.log("\n**************************************************************************************************************************"); + console.log("All accounts passed through from the CSV were successfully get the tokens, because we were able to read them all from events"); + console.log("****************************************************************************************************************************"); + } + +} + async function withdrawFromContract(erc20address, value) { let withdrawAction = securityToken.methods.withdrawERC20(erc20address, value); await common.sendTransaction(withdrawAction); From 78292847be7ea82085e56ad2df24314f5c32a63c Mon Sep 17 00:00:00 2001 From: shuffledex Date: Thu, 29 Nov 2018 11:37:27 -0300 Subject: [PATCH 02/12] replace multi mint to token manager method in cli menu --- CLI/commands/token_manager.js | 3 +++ CLI/polymath-cli.js | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CLI/commands/token_manager.js b/CLI/commands/token_manager.js index 0bbdad729..265605b98 100644 --- a/CLI/commands/token_manager.js +++ b/CLI/commands/token_manager.js @@ -829,5 +829,8 @@ module.exports = { executeApp: async function (_tokenSymbol) { await initialize(_tokenSymbol) return executeApp(); + }, + startCSV: async function (tokenSymbol, batchSize) { + return startCSV(tokenSymbol, batchSize); } } diff --git a/CLI/polymath-cli.js b/CLI/polymath-cli.js index 29bea752d..aff3aa4c5 100644 --- a/CLI/polymath-cli.js +++ b/CLI/polymath-cli.js @@ -7,7 +7,6 @@ var st20generator = require('./commands/ST20Generator'); var sto_manager = require('./commands/sto_manager'); var transfer = require('./commands/transfer'); var whitelist = require('./commands/whitelist'); -var multimint = require('./commands/multi_mint'); var accredit = require('./commands/accredit'); var changeNonAccreditedLimit = require('./commands/changeNonAccreditedLimit'); var transfer_ownership = require('./commands/transfer_ownership'); @@ -90,7 +89,7 @@ program .description('Distribute tokens to previously whitelisted investors') .action(async function(tokenSymbol, batchSize) { await gbl.initialize(program.remoteNode); - await multimint.executeApp(tokenSymbol, batchSize); + await token_manager.startCSV(tokenSymbol, batchSize); }); program From 0a3adf2df24122f0180955d329f8682c0068a822 Mon Sep 17 00:00:00 2001 From: shuffledex Date: Thu, 29 Nov 2018 13:45:57 -0300 Subject: [PATCH 03/12] moving accredit to sto manager file --- CLI/commands/accredit.js | 94 ------------------------------------- CLI/commands/sto_manager.js | 92 +++++++++++++++++++++++++++++++++++- CLI/polymath-cli.js | 3 +- 3 files changed, 91 insertions(+), 98 deletions(-) delete mode 100644 CLI/commands/accredit.js diff --git a/CLI/commands/accredit.js b/CLI/commands/accredit.js deleted file mode 100644 index cc550af2e..000000000 --- a/CLI/commands/accredit.js +++ /dev/null @@ -1,94 +0,0 @@ -var common = require('./common/common_functions'); -var csv_shared = require('./common/csv_shared'); -var abis = require('./helpers/contract_abis'); -var BigNumber = require('bignumber.js'); - -let distribData = new Array(); -let fullFileData = new Array(); -let badData = new Array(); - -let securityToken; - -async function startScript(tokenSymbol, batchSize) { - securityToken = await csv_shared.start(tokenSymbol, batchSize); - - let result_processing = await csv_shared.read('./CLI/data/accredited_data.csv', accredit_processing); - distribData = result_processing.distribData; - fullFileData = result_processing.fullFileData; - badData = result_processing.badData; - - await saveInBlockchain(); -}; - -function accredit_processing(csv_line) { - let isAddress = web3.utils.isAddress(csv_line[0]); - let isAccredited = (typeof JSON.parse(csv_line[1].toLowerCase())) == "boolean" ? JSON.parse(csv_line[1].toLowerCase()) : "not-valid"; - - if (isAddress && - (isAccredited != "not-valid")) { - return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), isAccredited)] - } else { - return [false, new Array(csv_line[0], isAccredited)] - } -} - -async function saveInBlockchain() { - let gtmModules; - try { - gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('USDTieredSTO')).call(); - } catch (e) { - console.log("Please attach USDTieredSTO module before launch this action.", e) - process.exit(0) - } - - if (!gtmModules.length) { - console.log("Please attach USDTieredSTO module before launch this action.") - process.exit(0) - } - - let usdTieredSTO = new web3.eth.Contract(abis.usdTieredSTO(), gtmModules[0]); - - console.log(` - -------------------------------------------------------- - ----- Sending accreditation changes to blockchain ----- - -------------------------------------------------------- - `); - - for (let i = 0; i < distribData.length; i++) { - try { - - // Splitting the user arrays to be organized by input - for (let i = 0; i < distribData.length; i++) { - try { - let investorArray = [], isAccreditedArray = []; - - for (let j = 0; j < distribData[i].length; j++) { - investorArray.push(distribData[i][j][0]) - isAccreditedArray.push(distribData[i][j][1]) - } - - let changeAccreditedAction = await usdTieredSTO.methods.changeAccredited(investorArray, isAccreditedArray); - let tx = await common.sendTransaction(changeAccreditedAction); - console.log(`Batch ${i} - Attempting to change accredited accounts:\n\n`, investorArray, "\n\n"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); - console.log("Change accredited transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - } catch (err) { - console.log("ERROR:", err); - } - } - - } catch (err) { - console.log("ERROR", err) - process.exit(0) - } - } - - return; -} - -module.exports = { - executeApp: async (tokenSymbol, batchSize) => { - return startScript(tokenSymbol, batchSize); - } -} \ No newline at end of file diff --git a/CLI/commands/sto_manager.js b/CLI/commands/sto_manager.js index d0fd1c3ea..974d57de4 100644 --- a/CLI/commands/sto_manager.js +++ b/CLI/commands/sto_manager.js @@ -1,11 +1,12 @@ const readlineSync = require('readline-sync'); const chalk = require('chalk'); -const accredit = require('./accredit'); const changeNonAccreditedLimit = require('./changeNonAccreditedLimit'); const contracts = require('./helpers/contract_addresses'); const abis = require('./helpers/contract_abis'); const common = require('./common/common_functions'); const gbl = require('./common/global'); +const csv_shared = require('./common/csv_shared'); +const BigNumber = require('bignumber.js'); /////////////////// // Crowdsale params @@ -18,6 +19,12 @@ let polyToken; let usdToken; let securityToken; +/* CSV variables */ +let distribData = new Array(); +let fullFileData = new Array(); +let badData = new Array(); +/* End CSV variables */ + async function executeApp() { let exit = false; while (!exit) { @@ -690,7 +697,7 @@ async function usdTieredSTO_configure(currentSTO) { await common.sendTransaction(changeAccreditedAction); break; case 2: - await accredit.executeApp(tokenSymbol, 75); + await startCSV(tokenSymbol, 75); break; case 3: let account = readlineSync.question('Enter the address to change non accredited limit: '); @@ -728,6 +735,84 @@ async function usdTieredSTO_configure(currentSTO) { } } +async function startCSV(tokenSymbol, batchSize) { + securityToken = await csv_shared.start(tokenSymbol, batchSize); + + let result_processing = await csv_shared.read('./CLI/data/accredited_data.csv', accredit_processing); + distribData = result_processing.distribData; + fullFileData = result_processing.fullFileData; + badData = result_processing.badData; + + await saveInBlockchain(); +} + +function accredit_processing(csv_line) { + let isAddress = web3.utils.isAddress(csv_line[0]); + let isAccredited = (typeof JSON.parse(csv_line[1].toLowerCase())) == "boolean" ? JSON.parse(csv_line[1].toLowerCase()) : "not-valid"; + + if (isAddress && + (isAccredited != "not-valid")) { + return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), isAccredited)] + } else { + return [false, new Array(csv_line[0], isAccredited)] + } +} + +async function saveInBlockchain() { + let gtmModules; + try { + gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('USDTieredSTO')).call(); + } catch (e) { + console.log("Please attach USDTieredSTO module before launch this action.", e) + process.exit(0) + } + + if (!gtmModules.length) { + console.log("Please attach USDTieredSTO module before launch this action.") + process.exit(0) + } + + let usdTieredSTO = new web3.eth.Contract(abis.usdTieredSTO(), gtmModules[0]); + + console.log(` + -------------------------------------------------------- + ----- Sending accreditation changes to blockchain ----- + -------------------------------------------------------- + `); + + for (let i = 0; i < distribData.length; i++) { + try { + + // Splitting the user arrays to be organized by input + for (let i = 0; i < distribData.length; i++) { + try { + let investorArray = [], isAccreditedArray = []; + + for (let j = 0; j < distribData[i].length; j++) { + investorArray.push(distribData[i][j][0]) + isAccreditedArray.push(distribData[i][j][1]) + } + + let changeAccreditedAction = await usdTieredSTO.methods.changeAccredited(investorArray, isAccreditedArray); + let tx = await common.sendTransaction(changeAccreditedAction); + console.log(`Batch ${i} - Attempting to change accredited accounts:\n\n`, investorArray, "\n\n"); + console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); + console.log("Change accredited transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); + console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); + } catch (err) { + console.log("ERROR:", err); + } + } + + } catch (err) { + console.log("ERROR", err) + process.exit(0) + } + } + + return; +} + async function modfifyTimes(currentSTO) { let times = timesConfigUSDTieredSTO(); let modifyTimesAction = currentSTO.methods.modifyTimes(times.startTime, times.endTime); @@ -893,5 +978,8 @@ module.exports = { addSTOModule: async function (_tokenSymbol, stoConfig) { await initialize(_tokenSymbol); return addSTOModule(stoConfig) + }, + startCSV: async function (tokenSymbol, batchSize) { + return startCSV(tokenSymbol, batchSize); } } \ No newline at end of file diff --git a/CLI/polymath-cli.js b/CLI/polymath-cli.js index aff3aa4c5..34d463bb9 100644 --- a/CLI/polymath-cli.js +++ b/CLI/polymath-cli.js @@ -7,7 +7,6 @@ var st20generator = require('./commands/ST20Generator'); var sto_manager = require('./commands/sto_manager'); var transfer = require('./commands/transfer'); var whitelist = require('./commands/whitelist'); -var accredit = require('./commands/accredit'); var changeNonAccreditedLimit = require('./commands/changeNonAccreditedLimit'); var transfer_ownership = require('./commands/transfer_ownership'); var dividends_manager = require('./commands/dividends_manager'); @@ -152,7 +151,7 @@ program .description('Runs accredit') .action(async function(tokenSymbol, batchSize) { await gbl.initialize(program.remoteNode); - await accredit.executeApp(tokenSymbol, batchSize); + await sto_manager.startCSV(tokenSymbol, batchSize) }); program From de2633d8c382b3cc403b5ae947bc464141719582 Mon Sep 17 00:00:00 2001 From: shuffledex Date: Thu, 29 Nov 2018 15:38:50 -0300 Subject: [PATCH 04/12] moving accreditable and nonAccreditable to STO manager --- CLI/commands/changeNonAccreditedLimit.js | 94 ---------------------- CLI/commands/sto_manager.js | 99 +++++++++++++++++++++--- CLI/polymath-cli.js | 5 +- 3 files changed, 92 insertions(+), 106 deletions(-) delete mode 100644 CLI/commands/changeNonAccreditedLimit.js diff --git a/CLI/commands/changeNonAccreditedLimit.js b/CLI/commands/changeNonAccreditedLimit.js deleted file mode 100644 index 029f2fc70..000000000 --- a/CLI/commands/changeNonAccreditedLimit.js +++ /dev/null @@ -1,94 +0,0 @@ -var common = require('./common/common_functions'); -var csv_shared = require('./common/csv_shared'); -var abis = require('./helpers/contract_abis'); -var BigNumber = require('bignumber.js'); - -let distribData = new Array(); -let fullFileData = new Array(); -let badData = new Array(); - -let securityToken; - -async function startScript(tokenSymbol, batchSize) { - securityToken = await csv_shared.start(tokenSymbol, batchSize); - - let result_processing = await csv_shared.read('./CLI/data/nonAccreditedLimits_data.csv', nonAccredited_processing); - distribData = result_processing.distribData; - fullFileData = result_processing.fullFileData; - badData = result_processing.badData; - - await saveInBlockchain(); -} - -function nonAccredited_processing(csv_line) { - let isAddress = web3.utils.isAddress(csv_line[0]); - let isNumber = !isNaN(csv_line[1]); - - if (isAddress && isNumber) { - return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), csv_line[1])] - } else { - return [false, new Array(csv_line[0], csv_line[1])] - } -} - -async function saveInBlockchain() { - let gtmModules; - try { - gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('USDTieredSTO')).call(); - } catch (e) { - console.log("Please attach USDTieredSTO module before launch this action.", e) - process.exit(0) - } - - if (!gtmModules.length) { - console.log("Please attach USDTieredSTO module before launch this action.") - process.exit(0) - } - - let usdTieredSTO = new web3.eth.Contract(abis.usdTieredSTO(), gtmModules[0]); - - console.log(` - -------------------------------------------------------------- - ----- Sending non accredited limit changes to blockchain ----- - -------------------------------------------------------------- - `); - - for (let i = 0; i < distribData.length; i++) { - try { - - // Splitting the user arrays to be organized by input - for (let i = 0; i < distribData.length; i++) { - try { - let investorArray = [], limitArray = []; - - for (let j = 0; j < distribData[i].length; j++) { - investorArray.push(distribData[i][j][0]); - limitArray.push(web3.utils.toWei(distribData[i][j][1].toString())); - } - - let changeNonAccreditedLimitAction = await usdTieredSTO.methods.changeNonAccreditedLimit(investorArray, limitArray); - let tx = await common.sendTransaction(changeNonAccreditedLimitAction); - console.log(`Batch ${i} - Attempting to change non accredited limits to accounts:\n\n`, investorArray, "\n\n"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); - console.log("Change accredited transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString()), "Ether"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - - } catch (err) { - console.log("ERROR:", err); - } - } - - } catch (err) { - console.log("ERROR", err) - process.exit(0) - } - } - - return; -} - -module.exports = { - executeApp: async (tokenSymbol, batchSize) => { - return startScript(tokenSymbol, batchSize); - } -} \ No newline at end of file diff --git a/CLI/commands/sto_manager.js b/CLI/commands/sto_manager.js index 974d57de4..8cd2baa5e 100644 --- a/CLI/commands/sto_manager.js +++ b/CLI/commands/sto_manager.js @@ -1,6 +1,5 @@ const readlineSync = require('readline-sync'); const chalk = require('chalk'); -const changeNonAccreditedLimit = require('./changeNonAccreditedLimit'); const contracts = require('./helpers/contract_addresses'); const abis = require('./helpers/contract_abis'); const common = require('./common/common_functions'); @@ -697,7 +696,7 @@ async function usdTieredSTO_configure(currentSTO) { await common.sendTransaction(changeAccreditedAction); break; case 2: - await startCSV(tokenSymbol, 75); + await startCSV(tokenSymbol, 75, 'accredited'); break; case 3: let account = readlineSync.question('Enter the address to change non accredited limit: '); @@ -709,7 +708,7 @@ async function usdTieredSTO_configure(currentSTO) { await common.sendTransaction(changeNonAccreditedLimitAction); break; case 4: - await changeNonAccreditedLimit.executeApp(tokenSymbol, 75); + await startCSV(tokenSymbol, 75, 'nonAccredited'); break; case 5: await modfifyTimes(currentSTO); @@ -735,15 +734,41 @@ async function usdTieredSTO_configure(currentSTO) { } } -async function startCSV(tokenSymbol, batchSize) { +async function startCSV(tokenSymbol, batchSize, accreditionType) { + let file, proccessing, saving; + + switch (accreditionType) { + case 'accredited': + file = './CLI/data/accredited_data.csv' + proccessing = accredit_processing + saving = saveInBlockchainAccredited + break; + case 'nonAccredited': + file = './CLI/data/nonAccreditedLimits_data.csv' + proccessing = nonAccredited_processing + saving = saveInBlockchainNonAccredited + break; + } + securityToken = await csv_shared.start(tokenSymbol, batchSize); - let result_processing = await csv_shared.read('./CLI/data/accredited_data.csv', accredit_processing); + let result_processing = await csv_shared.read(file, proccessing); distribData = result_processing.distribData; fullFileData = result_processing.fullFileData; badData = result_processing.badData; - await saveInBlockchain(); + await saving(); +} + +function nonAccredited_processing(csv_line) { + let isAddress = web3.utils.isAddress(csv_line[0]); + let isNumber = !isNaN(csv_line[1]); + + if (isAddress && isNumber) { + return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), csv_line[1])] + } else { + return [false, new Array(csv_line[0], csv_line[1])] + } } function accredit_processing(csv_line) { @@ -758,7 +783,7 @@ function accredit_processing(csv_line) { } } -async function saveInBlockchain() { +async function saveInBlockchainAccredited() { let gtmModules; try { gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('USDTieredSTO')).call(); @@ -813,6 +838,62 @@ async function saveInBlockchain() { return; } +async function saveInBlockchainNonAccredited() { + let gtmModules; + try { + gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('USDTieredSTO')).call(); + } catch (e) { + console.log("Please attach USDTieredSTO module before launch this action.", e) + process.exit(0) + } + + if (!gtmModules.length) { + console.log("Please attach USDTieredSTO module before launch this action.") + process.exit(0) + } + + let usdTieredSTO = new web3.eth.Contract(abis.usdTieredSTO(), gtmModules[0]); + + console.log(` + -------------------------------------------------------------- + ----- Sending non accredited limit changes to blockchain ----- + -------------------------------------------------------------- + `); + + for (let i = 0; i < distribData.length; i++) { + try { + + // Splitting the user arrays to be organized by input + for (let i = 0; i < distribData.length; i++) { + try { + let investorArray = [], limitArray = []; + + for (let j = 0; j < distribData[i].length; j++) { + investorArray.push(distribData[i][j][0]); + limitArray.push(web3.utils.toWei(distribData[i][j][1].toString())); + } + + let changeNonAccreditedLimitAction = await usdTieredSTO.methods.changeNonAccreditedLimit(investorArray, limitArray); + let tx = await common.sendTransaction(changeNonAccreditedLimitAction); + console.log(`Batch ${i} - Attempting to change non accredited limits to accounts:\n\n`, investorArray, "\n\n"); + console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); + console.log("Change accredited transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString()), "Ether"); + console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); + + } catch (err) { + console.log("ERROR:", err); + } + } + + } catch (err) { + console.log("ERROR", err) + process.exit(0) + } + } + + return; +} + async function modfifyTimes(currentSTO) { let times = timesConfigUSDTieredSTO(); let modifyTimesAction = currentSTO.methods.modifyTimes(times.startTime, times.endTime); @@ -979,7 +1060,7 @@ module.exports = { await initialize(_tokenSymbol); return addSTOModule(stoConfig) }, - startCSV: async function (tokenSymbol, batchSize) { - return startCSV(tokenSymbol, batchSize); + startCSV: async function (tokenSymbol, batchSize, accreditionType) { + return startCSV(tokenSymbol, batchSize, accreditionType); } } \ No newline at end of file diff --git a/CLI/polymath-cli.js b/CLI/polymath-cli.js index 34d463bb9..c34a4569d 100644 --- a/CLI/polymath-cli.js +++ b/CLI/polymath-cli.js @@ -7,7 +7,6 @@ var st20generator = require('./commands/ST20Generator'); var sto_manager = require('./commands/sto_manager'); var transfer = require('./commands/transfer'); var whitelist = require('./commands/whitelist'); -var changeNonAccreditedLimit = require('./commands/changeNonAccreditedLimit'); var transfer_ownership = require('./commands/transfer_ownership'); var dividends_manager = require('./commands/dividends_manager'); var transfer_manager = require('./commands/transfer_manager'); @@ -151,7 +150,7 @@ program .description('Runs accredit') .action(async function(tokenSymbol, batchSize) { await gbl.initialize(program.remoteNode); - await sto_manager.startCSV(tokenSymbol, batchSize) + await sto_manager.startCSV(tokenSymbol, batchSize, 'accredited') }); program @@ -160,7 +159,7 @@ program .description('Runs changeNonAccreditedLimit') .action(async function(tokenSymbol, batchSize) { await gbl.initialize(program.remoteNode); - await changeNonAccreditedLimit.executeApp(tokenSymbol, batchSize); + await sto_manager.startCSV(tokenSymbol, batchSize, 'nonAccredited') }); program From 8b964c5fdb7347d9dd67799c0bdfdced756ff853 Mon Sep 17 00:00:00 2001 From: shuffledex Date: Fri, 30 Nov 2018 09:04:15 -0300 Subject: [PATCH 05/12] whitelist to transfer manager --- CLI/commands/sto_manager.js | 4 +- CLI/commands/token_manager.js | 3 +- CLI/commands/transfer_manager.js | 181 +++++++++++++++++++++++++++++- CLI/commands/whitelist.js | 182 ------------------------------- CLI/polymath-cli.js | 5 +- 5 files changed, 184 insertions(+), 191 deletions(-) delete mode 100644 CLI/commands/whitelist.js diff --git a/CLI/commands/sto_manager.js b/CLI/commands/sto_manager.js index 8cd2baa5e..898eb8b45 100644 --- a/CLI/commands/sto_manager.js +++ b/CLI/commands/sto_manager.js @@ -708,7 +708,7 @@ async function usdTieredSTO_configure(currentSTO) { await common.sendTransaction(changeNonAccreditedLimitAction); break; case 4: - await startCSV(tokenSymbol, 75, 'nonAccredited'); + await startCSV(tokenSymbol, 75, 'nonAccreditedLimit'); break; case 5: await modfifyTimes(currentSTO); @@ -743,7 +743,7 @@ async function startCSV(tokenSymbol, batchSize, accreditionType) { proccessing = accredit_processing saving = saveInBlockchainAccredited break; - case 'nonAccredited': + case 'nonAccreditedLimit': file = './CLI/data/nonAccreditedLimits_data.csv' proccessing = nonAccredited_processing saving = saveInBlockchainNonAccredited diff --git a/CLI/commands/token_manager.js b/CLI/commands/token_manager.js index a94c0cc1d..e8556450d 100644 --- a/CLI/commands/token_manager.js +++ b/CLI/commands/token_manager.js @@ -1,7 +1,6 @@ // Libraries for terminal prompts const readlineSync = require('readline-sync'); const chalk = require('chalk'); -const whitelist = require('./whitelist'); const stoManager = require('./sto_manager'); const transferManager = require('./transfer_manager'); const common = require('./common/common_functions'); @@ -347,7 +346,7 @@ async function mintToSingleAddress(_investor, _amount) { } async function multi_mint_tokens() { - await whitelist.executeApp(tokenSymbol, 75); + await transferManager.startCSV(tokenSymbol, 75); console.log(chalk.green(`\nCongratulations! All the affiliates get succssfully whitelisted, Now its time to Mint the tokens\n`)); console.log(chalk.red(`WARNING: `) + `Please make sure all the addresses that get whitelisted are only eligible to hold or get Security token\n`); diff --git a/CLI/commands/transfer_manager.js b/CLI/commands/transfer_manager.js index 44ae64ca1..dc5aba389 100644 --- a/CLI/commands/transfer_manager.js +++ b/CLI/commands/transfer_manager.js @@ -5,13 +5,22 @@ var common = require('./common/common_functions'); var contracts = require('./helpers/contract_addresses'); var abis = require('./helpers/contract_abis'); var gbl = require('./common/global'); -var whitelist = require('./whitelist'); +var transferManager = require('./transfer_manager'); +var csv_shared = require('./common/csv_shared'); +var BigNumber = require('bignumber.js'); // App flow let tokenSymbol; let securityToken; let securityTokenRegistry; let currentTransferManager; +let generalTransferManager; + +/* CSV variables */ +let distribData = new Array(); +let fullFileData = new Array(); +let badData = new Array(); +/* End CSV variables */ async function executeApp() { let exit = false; @@ -373,7 +382,7 @@ async function generalTransferManager() { case 'Modify whitelist from CSV': console.log(chalk.yellow(`Data is going to be read from 'data/whitelist_data.csv'. Be sure this file is updated!`)); if (readlineSync.keyInYNStrict(`Do you want to continue?`)) { - await whitelist.executeApp(tokenSymbl); + await transferManager.startCSV(tokenSymbol); } break; /* @@ -468,6 +477,171 @@ async function generalTransferManager() { } } +async function startCSV(tokenSymbol, batchSize) { + securityToken = await csv_shared.start(tokenSymbol, batchSize); + + let result_processing = await csv_shared.read('./CLI/data/whitelist_data.csv', whitelist_processing); + distribData = result_processing.distribData; + fullFileData = result_processing.fullFileData; + badData = result_processing.badData; + + await saveInBlockchain(); + await finalResults(); +}; + +function whitelist_processing(csv_line) { + let isAddress = web3.utils.isAddress(csv_line[0]); + let sellValid = isValidDate(csv_line[1]) + let buyValid = isValidDate(csv_line[2]) + let kycExpiryDate = isValidDate(csv_line[3]) + let canBuyFromSTO = (typeof JSON.parse(csv_line[4].toLowerCase())) == "boolean" ? JSON.parse(csv_line[4].toLowerCase()) : "not-valid"; + + if (isAddress && + sellValid && + buyValid && + kycExpiryDate && + (canBuyFromSTO != "not-valid")) { + return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), sellValid, buyValid, kycExpiryDate, canBuyFromSTO)] + } else { + return [false, new Array(csv_line[0], sellValid, buyValid, kycExpiryDate, canBuyFromSTO)] + } +} + +function isValidDate(date) { + var matches = /^(\d{1,2})[-\/](\d{1,2})[-\/](\d{4})$/.exec(date); + if (matches == null) return false; + var d = matches[2]; + var m = matches[1] - 1; + var y = matches[3]; + var composedDate = new Date(y, m, d); + var timestampDate = composedDate.getTime() + + // For some reason these timestamps are being recorded +4 hours UTC + if (composedDate.getDate() == d && + composedDate.getMonth() == m && + composedDate.getFullYear() == y) { + return timestampDate / 1000 + } else { + return false + } +} + +async function saveInBlockchain() { + let gtmModules; + try { + gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('GeneralTransferManager')).call(); + } catch (e) { + console.log("Please attach General Transfer module before launch this action.", e) + process.exit(0) + } + + generalTransferManager = new web3.eth.Contract(abis.generalTransferManager(), gtmModules[0]); + + console.log(` + ------------------------------------------------------- + ----- Sending buy/sell restrictions to blockchain ----- + ------------------------------------------------------- + `); + + for (let i = 0; i < distribData.length; i++) { + try { + let investorArray = [], fromTimesArray = [], toTimesArray = [], expiryTimeArray = [], canBuyFromSTOArray = []; + + // Splitting the user arrays to be organized by input + for (let j = 0; j < distribData[i].length; j++) { + investorArray.push(distribData[i][j][0]) + fromTimesArray.push(distribData[i][j][1]) + toTimesArray.push(distribData[i][j][2]) + expiryTimeArray.push(distribData[i][j][3]) + canBuyFromSTOArray.push(distribData[i][j][4]) + } + + let modifyWhitelistMultiAction = await generalTransferManager.methods.modifyWhitelistMulti(investorArray, fromTimesArray, toTimesArray, expiryTimeArray, canBuyFromSTOArray); + let tx = await common.sendTransaction(modifyWhitelistMultiAction); + console.log(`Batch ${i} - Attempting to modifyWhitelist accounts:\n\n`, investorArray, "\n\n"); + console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); + console.log("Whitelist transaxction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); + console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); + + } catch (err) { + console.log("ERROR", err) + process.exit(0) + } + } + + return; +} + +async function finalResults() { + let totalInvestors = 0; + let updatedInvestors = 0; + let investorObjectLookup = {}; + let investorData_Events = new Array(); + let event_data = await generalTransferManager.getPastEvents('ModifyWhitelist', {fromBlock: 0, toBlock: 'latest'}, () => {}); + + for (var i = 0; i < event_data.length; i++) { + let combineArray = []; + + let investorAddress_Event = event_data[i].returnValues._investor + let fromTime_Event = event_data[i].returnValues._fromTime + let toTime_Event = event_data[i].returnValues._toTime + let expiryTime_Event = event_data[i].returnValues._expiryTime + let canBuyFromSTO_Event = event_data[i].returnValues._canBuyFromSTO + let blockNumber = event_data[i].blockNumber + + combineArray.push(investorAddress_Event); + combineArray.push(fromTime_Event); + combineArray.push(toTime_Event); + combineArray.push(expiryTime_Event); + combineArray.push(canBuyFromSTO_Event); + combineArray.push(blockNumber) + + investorData_Events.push(combineArray) + + // We have already recorded it, so this is an update to our object + if (investorObjectLookup.hasOwnProperty(investorAddress_Event)) { + // The block number form the event we are checking is bigger, so we gotta replace it + if (investorObjectLookup[investorAddress_Event].recordedBlockNumber < blockNumber) { + investorObjectLookup[investorAddress_Event] = {fromTime: fromTime_Event, toTime: toTime_Event, expiryTime: expiryTime_Event, canBuyFromSTO: canBuyFromSTO_Event, recordedBlockNumber: blockNumber}; + updatedInvestors += 1; + } + } else { + investorObjectLookup[investorAddress_Event] = {fromTime: fromTime_Event, toTime: toTime_Event, expiryTime: expiryTime_Event, canBuyFromSTO: canBuyFromSTO_Event, recordedBlockNumber: blockNumber}; + totalInvestors += 1; + } + } + + let investorAddress_Events = Object.keys(investorObjectLookup) + + console.log(`******************** EVENT LOGS ANALYSIS COMPLETE ********************\n`); + console.log(`A total of ${totalInvestors} investors have been whitelisted total, all time.\n`); + console.log(`This script in total sent ${fullFileData.length - badData.length} new investors and updated investors to the blockchain.\n`); + console.log(`There were ${badData.length} bad entries that didnt get sent to the blockchain in the script.\n`); + console.log("************************************************************************************************"); + console.log("OBJECT WITH EVERY USER AND THEIR UPDATED TIMES: \n\n", investorObjectLookup) + console.log("************************************************************************************************"); + console.log("LIST OF ALL INVESTORS WHITELISTED: \n\n", investorAddress_Events) + + let missingDistribs = []; + for (let l = 0; l < fullFileData.length; l++) { + if (!investorObjectLookup.hasOwnProperty(fullFileData[l][0])) { + missingDistribs.push(fullFileData[l]) + } + } + + if (missingDistribs.length > 0) { + console.log("************************************************************************************************"); + console.log("-- No LogModifyWhitelist event was found for the following data arrays. Please review them manually --") + console.log(missingDistribs) + console.log("************************************************************************************************"); + } else { + console.log("\n************************************************************************************************"); + console.log("All accounts passed through from the CSV were successfully whitelisted, because we were able to read them all from events") + console.log("************************************************************************************************"); + } + +} + async function manualApprovalTransferManager() { console.log(chalk.blue(`Manual Approval Transfer Manager at ${currentTransferManager.options.address}`), '\n'); @@ -952,5 +1126,8 @@ module.exports = { addTransferManagerModule: async function (_tokenSymbol) { await initialize(_tokenSymbol); return addTransferManagerModule() + }, + startCSV: async function (tokenSymbol, batchSize) { + return startCSV(tokenSymbol, batchSize); } } \ No newline at end of file diff --git a/CLI/commands/whitelist.js b/CLI/commands/whitelist.js deleted file mode 100644 index 7a5767611..000000000 --- a/CLI/commands/whitelist.js +++ /dev/null @@ -1,182 +0,0 @@ -var common = require('./common/common_functions'); -var csv_shared = require('./common/csv_shared'); -var abis = require('./helpers/contract_abis'); -var BigNumber = require('bignumber.js'); - -let distribData = new Array(); -let fullFileData = new Array(); -let badData = new Array(); - -let securityToken; -let generalTransferManager; - -async function startScript(tokenSymbol, batchSize) { - securityToken = await csv_shared.start(tokenSymbol, batchSize); - - let result_processing = await csv_shared.read('./CLI/data/whitelist_data.csv', whitelist_processing); - distribData = result_processing.distribData; - fullFileData = result_processing.fullFileData; - badData = result_processing.badData; - - await saveInBlockchain(); - await finalResults(); -}; - -function whitelist_processing(csv_line) { - let isAddress = web3.utils.isAddress(csv_line[0]); - let sellValid = isValidDate(csv_line[1]) - let buyValid = isValidDate(csv_line[2]) - let kycExpiryDate = isValidDate(csv_line[3]) - let canBuyFromSTO = (typeof JSON.parse(csv_line[4].toLowerCase())) == "boolean" ? JSON.parse(csv_line[4].toLowerCase()) : "not-valid"; - - if (isAddress && - sellValid && - buyValid && - kycExpiryDate && - (canBuyFromSTO != "not-valid")) { - return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), sellValid, buyValid, kycExpiryDate, canBuyFromSTO)] - } else { - return [false, new Array(csv_line[0], sellValid, buyValid, kycExpiryDate, canBuyFromSTO)] - } -} - -function isValidDate(date) { - var matches = /^(\d{1,2})[-\/](\d{1,2})[-\/](\d{4})$/.exec(date); - if (matches == null) return false; - var d = matches[2]; - var m = matches[1] - 1; - var y = matches[3]; - var composedDate = new Date(y, m, d); - var timestampDate = composedDate.getTime() - - // For some reason these timestamps are being recorded +4 hours UTC - if (composedDate.getDate() == d && - composedDate.getMonth() == m && - composedDate.getFullYear() == y) { - return timestampDate / 1000 - } else { - return false - } -} - -async function saveInBlockchain() { - let gtmModules; - try { - gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('GeneralTransferManager')).call(); - } catch (e) { - console.log("Please attach General Transfer module before launch this action.", e) - process.exit(0) - } - - generalTransferManager = new web3.eth.Contract(abis.generalTransferManager(), gtmModules[0]); - - console.log(` - ------------------------------------------------------- - ----- Sending buy/sell restrictions to blockchain ----- - ------------------------------------------------------- - `); - - for (let i = 0; i < distribData.length; i++) { - try { - let investorArray = [], fromTimesArray = [], toTimesArray = [], expiryTimeArray = [], canBuyFromSTOArray = []; - - // Splitting the user arrays to be organized by input - for (let j = 0; j < distribData[i].length; j++) { - investorArray.push(distribData[i][j][0]) - fromTimesArray.push(distribData[i][j][1]) - toTimesArray.push(distribData[i][j][2]) - expiryTimeArray.push(distribData[i][j][3]) - canBuyFromSTOArray.push(distribData[i][j][4]) - } - - let modifyWhitelistMultiAction = await generalTransferManager.methods.modifyWhitelistMulti(investorArray, fromTimesArray, toTimesArray, expiryTimeArray, canBuyFromSTOArray); - let tx = await common.sendTransaction(modifyWhitelistMultiAction); - console.log(`Batch ${i} - Attempting to modifyWhitelist accounts:\n\n`, investorArray, "\n\n"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); - console.log("Whitelist transaxction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - - } catch (err) { - console.log("ERROR", err) - process.exit(0) - } - } - - return; -} - -async function finalResults() { - let totalInvestors = 0; - let updatedInvestors = 0; - let investorObjectLookup = {}; - let investorData_Events = new Array(); - let event_data = await generalTransferManager.getPastEvents('ModifyWhitelist', {fromBlock: 0, toBlock: 'latest'}, () => {}); - - for (var i = 0; i < event_data.length; i++) { - let combineArray = []; - - let investorAddress_Event = event_data[i].returnValues._investor - let fromTime_Event = event_data[i].returnValues._fromTime - let toTime_Event = event_data[i].returnValues._toTime - let expiryTime_Event = event_data[i].returnValues._expiryTime - let canBuyFromSTO_Event = event_data[i].returnValues._canBuyFromSTO - let blockNumber = event_data[i].blockNumber - - combineArray.push(investorAddress_Event); - combineArray.push(fromTime_Event); - combineArray.push(toTime_Event); - combineArray.push(expiryTime_Event); - combineArray.push(canBuyFromSTO_Event); - combineArray.push(blockNumber) - - investorData_Events.push(combineArray) - - // We have already recorded it, so this is an update to our object - if (investorObjectLookup.hasOwnProperty(investorAddress_Event)) { - // The block number form the event we are checking is bigger, so we gotta replace it - if (investorObjectLookup[investorAddress_Event].recordedBlockNumber < blockNumber) { - investorObjectLookup[investorAddress_Event] = {fromTime: fromTime_Event, toTime: toTime_Event, expiryTime: expiryTime_Event, canBuyFromSTO: canBuyFromSTO_Event, recordedBlockNumber: blockNumber}; - updatedInvestors += 1; - } - } else { - investorObjectLookup[investorAddress_Event] = {fromTime: fromTime_Event, toTime: toTime_Event, expiryTime: expiryTime_Event, canBuyFromSTO: canBuyFromSTO_Event, recordedBlockNumber: blockNumber}; - totalInvestors += 1; - } - } - - let investorAddress_Events = Object.keys(investorObjectLookup) - - console.log(`******************** EVENT LOGS ANALYSIS COMPLETE ********************\n`); - console.log(`A total of ${totalInvestors} investors have been whitelisted total, all time.\n`); - console.log(`This script in total sent ${fullFileData.length - badData.length} new investors and updated investors to the blockchain.\n`); - console.log(`There were ${badData.length} bad entries that didnt get sent to the blockchain in the script.\n`); - console.log("************************************************************************************************"); - console.log("OBJECT WITH EVERY USER AND THEIR UPDATED TIMES: \n\n", investorObjectLookup) - console.log("************************************************************************************************"); - console.log("LIST OF ALL INVESTORS WHITELISTED: \n\n", investorAddress_Events) - - let missingDistribs = []; - for (let l = 0; l < fullFileData.length; l++) { - if (!investorObjectLookup.hasOwnProperty(fullFileData[l][0])) { - missingDistribs.push(fullFileData[l]) - } - } - - if (missingDistribs.length > 0) { - console.log("************************************************************************************************"); - console.log("-- No LogModifyWhitelist event was found for the following data arrays. Please review them manually --") - console.log(missingDistribs) - console.log("************************************************************************************************"); - } else { - console.log("\n************************************************************************************************"); - console.log("All accounts passed through from the CSV were successfully whitelisted, because we were able to read them all from events") - console.log("************************************************************************************************"); - } - -} - -module.exports = { - executeApp: async (tokenSymbol, batchSize) => { - return startScript(tokenSymbol, batchSize); - } -} \ No newline at end of file diff --git a/CLI/polymath-cli.js b/CLI/polymath-cli.js index b422d5215..3e2494a3f 100644 --- a/CLI/polymath-cli.js +++ b/CLI/polymath-cli.js @@ -6,7 +6,6 @@ var token_manager = require('./commands/token_manager'); var st20generator = require('./commands/ST20Generator'); var sto_manager = require('./commands/sto_manager'); var transfer = require('./commands/transfer'); -var whitelist = require('./commands/whitelist'); var transfer_ownership = require('./commands/transfer_ownership'); var dividends_manager = require('./commands/dividends_manager'); var transfer_manager = require('./commands/transfer_manager'); @@ -114,7 +113,7 @@ program .description('Mass-update a whitelist of allowed/known investors') .action(async function (tokenSymbol, batchSize) { await gbl.initialize(program.remoteNode); - await whitelist.executeApp(tokenSymbol, batchSize); + await transfer_manager.startCSV(tokenSymbol, batchSize); }); program @@ -160,7 +159,7 @@ program .description('Runs changeNonAccreditedLimit') .action(async function (tokenSymbol, batchSize) { await gbl.initialize(program.remoteNode); - await sto_manager.startCSV(tokenSymbol, batchSize, 'nonAccredited') + await sto_manager.startCSV(tokenSymbol, batchSize, 'nonAccreditedLimit') }); program From 1ad53fc4dc94326e606c7132109afb3586c57712 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 30 Nov 2018 20:22:26 -0300 Subject: [PATCH 06/12] CSV parser --- CLI/commands/helpers/csv.js | 42 +++++++++++++++++++++++++++++++++++++ CLI/commands/sto_manager.js | 28 ++++++++++++++++++------- CLI/package.json | 1 + CLI/yarn.lock | 5 +++++ 4 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 CLI/commands/helpers/csv.js diff --git a/CLI/commands/helpers/csv.js b/CLI/commands/helpers/csv.js new file mode 100644 index 000000000..560adb935 --- /dev/null +++ b/CLI/commands/helpers/csv.js @@ -0,0 +1,42 @@ +const csvParse = require('csv-parse/lib/sync'); +const fs = require('fs'); + +function _cast(obj) { + if (/^(\-|\+)?([1-9]+[0-9]*)$/.test(obj)) { // Int + obj = parseInt(obj); + } + else if (/^[+-]?([0-9]*[.])?[0-9]+$/.test(obj)) { // Float + obj = parseFloat(obj); + } + else if (/^(\d{1,2})[-\/](\d{1,2})[-\/](\d{4})$/.test(obj)) { // Datetime + var matches = /^(\d{1,2})[-\/](\d{1,2})[-\/](\d{4})$/.exec(obj); + var composedDate = new Date(matches[3], matches[1] - 1, matches[2]); + var timestampDate = composedDate.getTime(); + obj = timestampDate / 1000; + } + else if (obj.toLowerCase() === "true" || obj.toLowerCase() === "false") { // Boolean + obj = JSON.parse(obj.toLowerCase()); + } + return obj; +} + +function parse(_csvFilePath, _batchSize) { + // Read file + let input = fs.readFileSync(_csvFilePath); + // Parse csv + let data = csvParse(input, { cast: _cast }); + // Batches + let allBatches = []; + for (let index = 0; index < data.length; index += _batchSize) { + allBatches.push(data.slice(index, index + _batchSize)); + } + // Transform result + let result = []; + let columnsLenght = data[0].length; + for (let index = 0; index < columnsLenght; index++) { + result[index] = allBatches.map(batch => batch.map(record => record[index])); + } + return result; +} + +module.exports = parse; \ No newline at end of file diff --git a/CLI/commands/sto_manager.js b/CLI/commands/sto_manager.js index 898eb8b45..c77bb2485 100644 --- a/CLI/commands/sto_manager.js +++ b/CLI/commands/sto_manager.js @@ -5,6 +5,7 @@ const abis = require('./helpers/contract_abis'); const common = require('./common/common_functions'); const gbl = require('./common/global'); const csv_shared = require('./common/csv_shared'); +const csvParse = require('./helpers/csv'); const BigNumber = require('bignumber.js'); /////////////////// @@ -696,6 +697,17 @@ async function usdTieredSTO_configure(currentSTO) { await common.sendTransaction(changeAccreditedAction); break; case 2: + let batchSize = 75; + let csvFile = fs.readFileSync('./CLI/data/accredited_data.csv'); + let columns = ['address', 'accredited']; + let resultData = []; + let index = 0; + while (true) { + index++; + let data = csvParse(csvFile, { columns: columns, from_line: index, to_line: batchSize * index }); + resultData.push(data); + } + let accredit; await startCSV(tokenSymbol, 75, 'accredited'); break; case 3: @@ -736,7 +748,7 @@ async function usdTieredSTO_configure(currentSTO) { async function startCSV(tokenSymbol, batchSize, accreditionType) { let file, proccessing, saving; - + switch (accreditionType) { case 'accredited': file = './CLI/data/accredited_data.csv' @@ -756,7 +768,7 @@ async function startCSV(tokenSymbol, batchSize, accreditionType) { distribData = result_processing.distribData; fullFileData = result_processing.fullFileData; badData = result_processing.badData; - + await saving(); } @@ -812,12 +824,12 @@ async function saveInBlockchainAccredited() { for (let i = 0; i < distribData.length; i++) { try { let investorArray = [], isAccreditedArray = []; - + for (let j = 0; j < distribData[i].length; j++) { investorArray.push(distribData[i][j][0]) isAccreditedArray.push(distribData[i][j][1]) } - + let changeAccreditedAction = await usdTieredSTO.methods.changeAccredited(investorArray, isAccreditedArray); let tx = await common.sendTransaction(changeAccreditedAction); console.log(`Batch ${i} - Attempting to change accredited accounts:\n\n`, investorArray, "\n\n"); @@ -867,7 +879,7 @@ async function saveInBlockchainNonAccredited() { for (let i = 0; i < distribData.length; i++) { try { let investorArray = [], limitArray = []; - + for (let j = 0; j < distribData[i].length; j++) { investorArray.push(distribData[i][j][0]); limitArray.push(web3.utils.toWei(distribData[i][j][1].toString())); @@ -879,7 +891,7 @@ async function saveInBlockchainNonAccredited() { console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); console.log("Change accredited transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString()), "Ether"); console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - + } catch (err) { console.log("ERROR:", err); } @@ -1063,4 +1075,6 @@ module.exports = { startCSV: async function (tokenSymbol, batchSize, accreditionType) { return startCSV(tokenSymbol, batchSize, accreditionType); } -} \ No newline at end of file +} + + diff --git a/CLI/package.json b/CLI/package.json index 95a391b45..64fb2a221 100644 --- a/CLI/package.json +++ b/CLI/package.json @@ -11,6 +11,7 @@ "dependencies": { "chalk": "^2.4.1", "commander": "^2.16.0", + "csv-parse": "^4.0.1", "ethers": "^4.0.7", "moment": "^2.22.2", "readline-sync": "^1.4.9", diff --git a/CLI/yarn.lock b/CLI/yarn.lock index 4d20bee11..51925f5ef 100644 --- a/CLI/yarn.lock +++ b/CLI/yarn.lock @@ -437,6 +437,11 @@ crypto-browserify@3.12.0: randombytes "^2.0.0" randomfill "^1.0.3" +csv-parse@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.0.1.tgz#4ad438352cbf12d5317d0fb9d588e53473293851" + integrity sha512-ehkwejEj05wwO7Q9JD+YSI6dNMIauHIroNU1RALrmRrqPoZIwRnfBtgq5GkU6i2RxZOJqjo3dtI1NrVSXvaimA== + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" From d0d51933ca9f56af2f38154d9eba266dc863a709 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 30 Nov 2018 20:26:44 -0300 Subject: [PATCH 07/12] Data folder reorganization --- CLI/data/{ => Checkpoint}/dividendsExclusions_data.csv | 0 CLI/data/{ => ST}/multi_mint_data.csv | 0 CLI/data/{ => STO/USDTieredSTO}/accredited_data.csv | 0 CLI/data/{ => STO/USDTieredSTO}/nonAccreditedLimits_data.csv | 0 CLI/data/{ => STO}/capped_sto_data.yml | 0 CLI/data/{ => STO}/usd_tiered_sto_data.yml | 0 CLI/data/{ => Ticker}/ticker_data.csv | 0 CLI/data/{ => Transfer/GTM}/whitelist_data.csv | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename CLI/data/{ => Checkpoint}/dividendsExclusions_data.csv (100%) rename CLI/data/{ => ST}/multi_mint_data.csv (100%) rename CLI/data/{ => STO/USDTieredSTO}/accredited_data.csv (100%) rename CLI/data/{ => STO/USDTieredSTO}/nonAccreditedLimits_data.csv (100%) rename CLI/data/{ => STO}/capped_sto_data.yml (100%) rename CLI/data/{ => STO}/usd_tiered_sto_data.yml (100%) rename CLI/data/{ => Ticker}/ticker_data.csv (100%) rename CLI/data/{ => Transfer/GTM}/whitelist_data.csv (100%) diff --git a/CLI/data/dividendsExclusions_data.csv b/CLI/data/Checkpoint/dividendsExclusions_data.csv similarity index 100% rename from CLI/data/dividendsExclusions_data.csv rename to CLI/data/Checkpoint/dividendsExclusions_data.csv diff --git a/CLI/data/multi_mint_data.csv b/CLI/data/ST/multi_mint_data.csv similarity index 100% rename from CLI/data/multi_mint_data.csv rename to CLI/data/ST/multi_mint_data.csv diff --git a/CLI/data/accredited_data.csv b/CLI/data/STO/USDTieredSTO/accredited_data.csv similarity index 100% rename from CLI/data/accredited_data.csv rename to CLI/data/STO/USDTieredSTO/accredited_data.csv diff --git a/CLI/data/nonAccreditedLimits_data.csv b/CLI/data/STO/USDTieredSTO/nonAccreditedLimits_data.csv similarity index 100% rename from CLI/data/nonAccreditedLimits_data.csv rename to CLI/data/STO/USDTieredSTO/nonAccreditedLimits_data.csv diff --git a/CLI/data/capped_sto_data.yml b/CLI/data/STO/capped_sto_data.yml similarity index 100% rename from CLI/data/capped_sto_data.yml rename to CLI/data/STO/capped_sto_data.yml diff --git a/CLI/data/usd_tiered_sto_data.yml b/CLI/data/STO/usd_tiered_sto_data.yml similarity index 100% rename from CLI/data/usd_tiered_sto_data.yml rename to CLI/data/STO/usd_tiered_sto_data.yml diff --git a/CLI/data/ticker_data.csv b/CLI/data/Ticker/ticker_data.csv similarity index 100% rename from CLI/data/ticker_data.csv rename to CLI/data/Ticker/ticker_data.csv diff --git a/CLI/data/whitelist_data.csv b/CLI/data/Transfer/GTM/whitelist_data.csv similarity index 100% rename from CLI/data/whitelist_data.csv rename to CLI/data/Transfer/GTM/whitelist_data.csv From d47cc3270ce814f5543935828cae62bbef51c3ef Mon Sep 17 00:00:00 2001 From: Victor Date: Sun, 2 Dec 2018 14:47:01 -0300 Subject: [PATCH 08/12] Refactoring and improvements --- CLI/commands/common/common_functions.js | 51 ++++++++++++++++--------- CLI/commands/helpers/csv.js | 17 +++------ 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/CLI/commands/common/common_functions.js b/CLI/commands/common/common_functions.js index 0d21d06f3..a59b9ac21 100644 --- a/CLI/commands/common/common_functions.js +++ b/CLI/commands/common/common_functions.js @@ -42,7 +42,7 @@ function getFinalOptions(options) { async function getGasLimit(options, action) { let block = await web3.eth.getBlock("latest"); let networkGasLimit = block.gasLimit; - let gas = Math.round(options.factor * (await action.estimateGas({ from: options.from.address, value: options.value}))); + let gas = Math.round(options.factor * (await action.estimateGas({ from: options.from.address, value: options.value }))); return (gas > networkGasLimit) ? networkGasLimit : gas; } @@ -65,16 +65,16 @@ async function checkPermissions(action) { module.exports = { convertToDaysRemaining: function (timeRemaining) { var seconds = parseInt(timeRemaining, 10); - + var days = Math.floor(seconds / (3600 * 24)); - seconds -= days * 3600 * 24; - var hrs = Math.floor(seconds / 3600); - seconds -= hrs * 3600; + seconds -= days * 3600 * 24; + var hrs = Math.floor(seconds / 3600); + seconds -= hrs * 3600; var mnts = Math.floor(seconds / 60); - seconds -= mnts * 60; + seconds -= mnts * 60; return (days + " days, " + hrs + " Hrs, " + mnts + " Minutes, " + seconds + " Seconds"); }, - logAsciiBull: function() { + logAsciiBull: function () { console.log(` /######%%, /#( ##########%%%%%, ,%%%. % @@ -103,8 +103,8 @@ module.exports = { options = getFinalOptions(options); let gasLimit = await getGasLimit(options, action); - - console.log(chalk.black.bgYellowBright(`---- Transaction executed: ${action._method.name} - Gas limit provided: ${gasLimit} ----`)); + + console.log(chalk.black.bgYellowBright(`---- Transaction executed: ${action._method.name} - Gas limit provided: ${gasLimit} ----`)); let nonce = await web3.eth.getTransactionCount(options.from.address); let abi = action.encodeABI(); @@ -117,24 +117,24 @@ module.exports = { nonce: nonce, value: web3.utils.toHex(options.value) }; - + const transaction = new Tx(parameter); transaction.sign(Buffer.from(options.from.privateKey.replace('0x', ''), 'hex')); return await web3.eth.sendSignedTransaction('0x' + transaction.serialize().toString('hex')) - .on('transactionHash', function(hash){ - console.log(` + .on('transactionHash', function (hash) { + console.log(` Your transaction is being processed. Please wait... TxHash: ${hash}` - ); - }) - .on('receipt', function(receipt){ - console.log(` + ); + }) + .on('receipt', function (receipt) { + console.log(` Congratulations! The transaction was successfully completed. Gas used: ${receipt.gasUsed} - Gas spent: ${web3.utils.fromWei((new web3.utils.BN(options.gasPrice)).mul(new web3.utils.BN(receipt.gasUsed)))} Ether Review it on Etherscan. TxHash: ${receipt.transactionHash}\n` - ); - }); + ); + }); }, getEventFromLogs: function (jsonInterface, logs, eventName) { let eventJsonInterface = jsonInterface.find(o => o.name === eventName && o.type === 'event'); @@ -145,5 +145,20 @@ module.exports = { let eventJsonInterface = jsonInterface.find(o => o.name === eventName && o.type === 'event'); let filteredLogs = logs.filter(l => l.topics.includes(eventJsonInterface.signature)); return filteredLogs.map(l => web3.eth.abi.decodeLog(eventJsonInterface.inputs, l.data, l.topics.slice(1))); + }, + splitIntoBatches: function (data, batchSize) { + let allBatches = []; + for (let index = 0; index < data.length; index += batchSize) { + allBatches.push(data.slice(index, index + batchSize)); + } + return allBatches; + }, + transposeBatches: function (batches) { + let result = []; + let columns = batches[0][0].length; + for (let index = 0; index < columns; index++) { + result[index] = batches.map(batch => batch.map(record => record[index])); + } + return result; } }; diff --git a/CLI/commands/helpers/csv.js b/CLI/commands/helpers/csv.js index 560adb935..a36dcec15 100644 --- a/CLI/commands/helpers/csv.js +++ b/CLI/commands/helpers/csv.js @@ -1,5 +1,6 @@ const csvParse = require('csv-parse/lib/sync'); const fs = require('fs'); +const web3Utils = require('web3-utils'); function _cast(obj) { if (/^(\-|\+)?([1-9]+[0-9]*)$/.test(obj)) { // Int @@ -16,6 +17,8 @@ function _cast(obj) { } else if (obj.toLowerCase() === "true" || obj.toLowerCase() === "false") { // Boolean obj = JSON.parse(obj.toLowerCase()); + } else if (web3Utils.isAddress(obj)) { + obj = web3Utils.toChecksumAddress(obj); } return obj; } @@ -25,18 +28,8 @@ function parse(_csvFilePath, _batchSize) { let input = fs.readFileSync(_csvFilePath); // Parse csv let data = csvParse(input, { cast: _cast }); - // Batches - let allBatches = []; - for (let index = 0; index < data.length; index += _batchSize) { - allBatches.push(data.slice(index, index + _batchSize)); - } - // Transform result - let result = []; - let columnsLenght = data[0].length; - for (let index = 0; index < columnsLenght; index++) { - result[index] = allBatches.map(batch => batch.map(record => record[index])); - } - return result; + + return data; } module.exports = parse; \ No newline at end of file From 1fecc9a3941de0e47b9208a66ed50334a9d099eb Mon Sep 17 00:00:00 2001 From: Victor Date: Sun, 2 Dec 2018 15:18:31 -0300 Subject: [PATCH 09/12] STO manager refactoring for batch operations --- CLI/commands/common/constants.js | 15 +- CLI/commands/sto_manager.js | 238 ++++++++----------------------- CLI/polymath-cli.js | 18 --- 3 files changed, 66 insertions(+), 205 deletions(-) diff --git a/CLI/commands/common/constants.js b/CLI/commands/common/constants.js index c3e2b796c..bb9c8f69c 100644 --- a/CLI/commands/common/constants.js +++ b/CLI/commands/common/constants.js @@ -7,22 +7,22 @@ module.exports = Object.freeze({ BURN: 5 }, DURATION: { - seconds: function(val) { + seconds: function (val) { return val }, - minutes: function(val) { + minutes: function (val) { return val * this.seconds(60) }, - hours: function(val) { + hours: function (val) { return val * this.minutes(60) }, - days: function(val) { + days: function (val) { return val * this.hours(24) }, - weeks: function(val) { + weeks: function (val) { return val * this.days(7) }, - years: function(val) { + years: function (val) { return val * this.days(365) } }, @@ -30,5 +30,6 @@ module.exports = Object.freeze({ ETH: 0, POLY: 1, DAI: 2 - } + }, + DEFAULT_BATCH_SIZE: 75 }); \ No newline at end of file diff --git a/CLI/commands/sto_manager.js b/CLI/commands/sto_manager.js index c77bb2485..8e2a347f0 100644 --- a/CLI/commands/sto_manager.js +++ b/CLI/commands/sto_manager.js @@ -8,6 +8,11 @@ const csv_shared = require('./common/csv_shared'); const csvParse = require('./helpers/csv'); const BigNumber = require('bignumber.js'); +/////////////////// +// Constants +const ACCREDIT_DATA_CSV = './CLI/data/STO/USDTieredSTO/accredited_data.csv'; +const NON_ACCREDIT_LIMIT_DATA_CSV = './CLI/data/STO/USDTieredSTO/nonAccreditedLimits_data.csv' + /////////////////// // Crowdsale params let tokenSymbol; @@ -19,12 +24,6 @@ let polyToken; let usdToken; let securityToken; -/* CSV variables */ -let distribData = new Array(); -let fullFileData = new Array(); -let badData = new Array(); -/* End CSV variables */ - async function executeApp() { let exit = false; while (!exit) { @@ -697,18 +696,7 @@ async function usdTieredSTO_configure(currentSTO) { await common.sendTransaction(changeAccreditedAction); break; case 2: - let batchSize = 75; - let csvFile = fs.readFileSync('./CLI/data/accredited_data.csv'); - let columns = ['address', 'accredited']; - let resultData = []; - let index = 0; - while (true) { - index++; - let data = csvParse(csvFile, { columns: columns, from_line: index, to_line: batchSize * index }); - resultData.push(data); - } - let accredit; - await startCSV(tokenSymbol, 75, 'accredited'); + await changeAccreditedInBatch(currentSTO); break; case 3: let account = readlineSync.question('Enter the address to change non accredited limit: '); @@ -716,11 +704,10 @@ async function usdTieredSTO_configure(currentSTO) { let accounts = [account]; let limits = [web3.utils.toWei(limit)]; let changeNonAccreditedLimitAction = currentSTO.methods.changeNonAccreditedLimit(accounts, limits); - // 2 GAS? await common.sendTransaction(changeNonAccreditedLimitAction); break; case 4: - await startCSV(tokenSymbol, 75, 'nonAccreditedLimit'); + await changeNonAccreditedLimitsInBatch(currentSTO); break; case 5: await modfifyTimes(currentSTO); @@ -746,164 +733,60 @@ async function usdTieredSTO_configure(currentSTO) { } } -async function startCSV(tokenSymbol, batchSize, accreditionType) { - let file, proccessing, saving; - - switch (accreditionType) { - case 'accredited': - file = './CLI/data/accredited_data.csv' - proccessing = accredit_processing - saving = saveInBlockchainAccredited - break; - case 'nonAccreditedLimit': - file = './CLI/data/nonAccreditedLimits_data.csv' - proccessing = nonAccredited_processing - saving = saveInBlockchainNonAccredited - break; - } - - securityToken = await csv_shared.start(tokenSymbol, batchSize); - - let result_processing = await csv_shared.read(file, proccessing); - distribData = result_processing.distribData; - fullFileData = result_processing.fullFileData; - badData = result_processing.badData; - - await saving(); -} - -function nonAccredited_processing(csv_line) { - let isAddress = web3.utils.isAddress(csv_line[0]); - let isNumber = !isNaN(csv_line[1]); - - if (isAddress && isNumber) { - return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), csv_line[1])] - } else { - return [false, new Array(csv_line[0], csv_line[1])] - } -} - -function accredit_processing(csv_line) { - let isAddress = web3.utils.isAddress(csv_line[0]); - let isAccredited = (typeof JSON.parse(csv_line[1].toLowerCase())) == "boolean" ? JSON.parse(csv_line[1].toLowerCase()) : "not-valid"; - - if (isAddress && - (isAccredited != "not-valid")) { - return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), isAccredited)] - } else { - return [false, new Array(csv_line[0], isAccredited)] - } -} - -async function saveInBlockchainAccredited() { - let gtmModules; - try { - gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('USDTieredSTO')).call(); - } catch (e) { - console.log("Please attach USDTieredSTO module before launch this action.", e) - process.exit(0) - } - - if (!gtmModules.length) { - console.log("Please attach USDTieredSTO module before launch this action.") - process.exit(0) - } - - let usdTieredSTO = new web3.eth.Contract(abis.usdTieredSTO(), gtmModules[0]); - - console.log(` - -------------------------------------------------------- - ----- Sending accreditation changes to blockchain ----- - -------------------------------------------------------- - `); - - for (let i = 0; i < distribData.length; i++) { - try { - - // Splitting the user arrays to be organized by input - for (let i = 0; i < distribData.length; i++) { - try { - let investorArray = [], isAccreditedArray = []; - - for (let j = 0; j < distribData[i].length; j++) { - investorArray.push(distribData[i][j][0]) - isAccreditedArray.push(distribData[i][j][1]) - } - - let changeAccreditedAction = await usdTieredSTO.methods.changeAccredited(investorArray, isAccreditedArray); - let tx = await common.sendTransaction(changeAccreditedAction); - console.log(`Batch ${i} - Attempting to change accredited accounts:\n\n`, investorArray, "\n\n"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); - console.log("Change accredited transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - } catch (err) { - console.log("ERROR:", err); - } - } - - } catch (err) { - console.log("ERROR", err) - process.exit(0) - } +async function changeAccreditedInBatch(currentSTO) { + let csvFilePath = readlineSync.question(`Enter the path for csv data file (${ACCREDIT_DATA_CSV}): `, { + defaultInput: ACCREDIT_DATA_CSV + }); + let batchSize = readlineSync.question(`Enter the max number of records per transaction or batch size (${gbl.constants.DEFAULT_BATCH_SIZE}): `, { + limit: function (input) { + return parseInt(input) > 0; + }, + limitMessage: 'Must be greater than 0', + defaultInput: gbl.constants.DEFAULT_BATCH_SIZE + }); + let parsedData = csvParse(csvFilePath); + let validData = parsedData.filter(row => web3.utils.isAddress(row[0]) && typeof row[1] === 'boolean'); + let invalidRows = parsedData.filter(row => !validData.includes(row)); + if (invalidRows.length > 0) { + console.log(chalk.red(`The following lines from csv file are not valid: ${invalidRows.map(r => parsedData.indexOf(r) + 1).join(',')}`)); + } + let batches = common.splitIntoBatches(validData, batchSize); + let [investorArray, isAccreditedArray] = common.transposeBatches(batches); + for (let batch = 0; batch < batches.length; batch++) { + console.log(`Batch ${batch + 1} - Attempting to change accredited accounts:\n\n`, investorArray[batch], '\n'); + let action = await currentSTO.methods.changeAccredited(investorArray[batch], isAccreditedArray[batch]); + let receipt = await common.sendTransaction(action); + console.log(chalk.green('Change accredited transaction was successful.')); + console.log(`${receipt.gasUsed} gas used. Spent: ${web3.utils.fromWei((new web3.utils.BN(receipt.gasUsed)).mul(new web3.utils.BN(defaultGasPrice)))} ETH`); } - - return; } -async function saveInBlockchainNonAccredited() { - let gtmModules; - try { - gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('USDTieredSTO')).call(); - } catch (e) { - console.log("Please attach USDTieredSTO module before launch this action.", e) - process.exit(0) - } - - if (!gtmModules.length) { - console.log("Please attach USDTieredSTO module before launch this action.") - process.exit(0) - } - - let usdTieredSTO = new web3.eth.Contract(abis.usdTieredSTO(), gtmModules[0]); - - console.log(` - -------------------------------------------------------------- - ----- Sending non accredited limit changes to blockchain ----- - -------------------------------------------------------------- - `); - - for (let i = 0; i < distribData.length; i++) { - try { - - // Splitting the user arrays to be organized by input - for (let i = 0; i < distribData.length; i++) { - try { - let investorArray = [], limitArray = []; - - for (let j = 0; j < distribData[i].length; j++) { - investorArray.push(distribData[i][j][0]); - limitArray.push(web3.utils.toWei(distribData[i][j][1].toString())); - } - - let changeNonAccreditedLimitAction = await usdTieredSTO.methods.changeNonAccreditedLimit(investorArray, limitArray); - let tx = await common.sendTransaction(changeNonAccreditedLimitAction); - console.log(`Batch ${i} - Attempting to change non accredited limits to accounts:\n\n`, investorArray, "\n\n"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); - console.log("Change accredited transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString()), "Ether"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - - } catch (err) { - console.log("ERROR:", err); - } - } - - } catch (err) { - console.log("ERROR", err) - process.exit(0) - } +async function changeNonAccreditedLimitsInBatch(currentSTO) { + let csvFilePath = readlineSync.question(`Enter the path for csv data file (${NON_ACCREDIT_LIMIT_DATA_CSV}): `, { + defaultInput: NON_ACCREDIT_LIMIT_DATA_CSV + }); + let batchSize = readlineSync.question(`Enter the max number of records per transaction or batch size (${gbl.constants.DEFAULT_BATCH_SIZE}): `, { + limit: function (input) { + return parseInt(input) > 0; + }, + limitMessage: 'Must be greater than 0', + defaultInput: gbl.constants.DEFAULT_BATCH_SIZE + }); + let parsedData = csvParse(csvFilePath); + let validData = parsedData.filter(row => web3.utils.isAddress(row[0]) && !isNaN(row[1])); + let invalidRows = parsedData.filter(row => !validData.includes(row)); + if (invalidRows.length > 0) { + console.log(chalk.red(`The following lines from csv file are not valid: ${invalidRows.map(r => parsedData.indexOf(r) + 1).join(',')}`)); + } + let batches = common.splitIntoBatches(validData, batchSize); + let [investorArray, limitArray] = common.transposeBatches(batches); + for (let batch = 0; batch < batches.length; batch++) { + console.log(`Batch ${batch + 1} - Attempting to change accredited accounts:\n\n`, investorArray[batch], '\n'); + let action = await currentSTO.methods.changeNonAccreditedLimit(investorArray[batch], limitArray[batch]); + let receipt = await common.sendTransaction(action); + console.log(chalk.green('Change non accredited limits transaction was successful.')); + console.log(`${receipt.gasUsed} gas used. Spent: ${web3.utils.fromWei((new web3.utils.BN(receipt.gasUsed)).mul(new web3.utils.BN(defaultGasPrice)))} ETH`); } - - return; } async function modfifyTimes(currentSTO) { @@ -1071,10 +954,5 @@ module.exports = { addSTOModule: async function (_tokenSymbol, stoConfig) { await initialize(_tokenSymbol); return addSTOModule(stoConfig) - }, - startCSV: async function (tokenSymbol, batchSize, accreditionType) { - return startCSV(tokenSymbol, batchSize, accreditionType); } -} - - +} \ No newline at end of file diff --git a/CLI/polymath-cli.js b/CLI/polymath-cli.js index 3e2494a3f..524c167cb 100644 --- a/CLI/polymath-cli.js +++ b/CLI/polymath-cli.js @@ -144,24 +144,6 @@ program await contract_manager.executeApp(); }); -program - .command('accredit [batchSize]') - .alias('a') - .description('Runs accredit') - .action(async function (tokenSymbol, batchSize) { - await gbl.initialize(program.remoteNode); - await sto_manager.startCSV(tokenSymbol, batchSize, 'accredited') - }); - -program - .command('nonAccreditedLimit [batchSize]') - .alias('nal') - .description('Runs changeNonAccreditedLimit') - .action(async function (tokenSymbol, batchSize) { - await gbl.initialize(program.remoteNode); - await sto_manager.startCSV(tokenSymbol, batchSize, 'nonAccreditedLimit') - }); - program .command('strMigrator [toStrAddress] [fromTrAddress] [fromStrAddress]') .alias('str') From dde9bf0c2bfddf8c9cd0fae7c8b2b382628ecc24 Mon Sep 17 00:00:00 2001 From: Victor Date: Sun, 2 Dec 2018 16:03:43 -0300 Subject: [PATCH 10/12] Transfer manager refactoring for batches --- CLI/commands/sto_manager.js | 2 - CLI/commands/transfer_manager.js | 293 +++++++++---------------------- CLI/polymath-cli.js | 9 - 3 files changed, 82 insertions(+), 222 deletions(-) diff --git a/CLI/commands/sto_manager.js b/CLI/commands/sto_manager.js index 8e2a347f0..d5bd58a4d 100644 --- a/CLI/commands/sto_manager.js +++ b/CLI/commands/sto_manager.js @@ -4,9 +4,7 @@ const contracts = require('./helpers/contract_addresses'); const abis = require('./helpers/contract_abis'); const common = require('./common/common_functions'); const gbl = require('./common/global'); -const csv_shared = require('./common/csv_shared'); const csvParse = require('./helpers/csv'); -const BigNumber = require('bignumber.js'); /////////////////// // Constants diff --git a/CLI/commands/transfer_manager.js b/CLI/commands/transfer_manager.js index dc5aba389..ca2f4ee06 100644 --- a/CLI/commands/transfer_manager.js +++ b/CLI/commands/transfer_manager.js @@ -1,26 +1,21 @@ -var readlineSync = require('readline-sync'); -var chalk = require('chalk'); -var moment = require('moment'); -var common = require('./common/common_functions'); -var contracts = require('./helpers/contract_addresses'); -var abis = require('./helpers/contract_abis'); -var gbl = require('./common/global'); -var transferManager = require('./transfer_manager'); -var csv_shared = require('./common/csv_shared'); -var BigNumber = require('bignumber.js'); +const readlineSync = require('readline-sync'); +const chalk = require('chalk'); +const moment = require('moment'); +const common = require('./common/common_functions'); +const contracts = require('./helpers/contract_addresses'); +const abis = require('./helpers/contract_abis'); +const gbl = require('./common/global'); +const csvParse = require('./helpers/csv'); + +/////////////////// +// Constants +const WHITELIST_DATA_CSV = './CLI/data/Transfer/GTM/whitelist_data.csv'; // App flow let tokenSymbol; let securityToken; let securityTokenRegistry; let currentTransferManager; -let generalTransferManager; - -/* CSV variables */ -let distribData = new Array(); -let fullFileData = new Array(); -let badData = new Array(); -/* End CSV variables */ async function executeApp() { let exit = false; @@ -334,7 +329,7 @@ async function generalTransferManager() { console.log(`- Allow all burn transfers: ${displayAllowAllBurnTransfers ? `YES` : `NO`}`); // ------------------ - let options = ['Modify whitelist', 'Modify whitelist from CSV', /*'Modify Whitelist Signed',*/ + let options = ['Check whitelist', 'Modify whitelist', 'Modify whitelist from CSV', /*'Modify Whitelist Signed',*/ `Change issuance address`, 'Change signing address']; if (displayAllowAllTransfers) { options.push('Disallow all transfers'); @@ -361,6 +356,19 @@ async function generalTransferManager() { let optionSelected = options[index]; console.log('Selected:', index != -1 ? optionSelected : 'Return', '\n'); switch (optionSelected) { + case 'Check whitelist': + let investorToCheck = readlineSync.question('Enter the address you want to check: ', { + limit: function (input) { + return web3.utils.isAddress(input); + }, + limitMessage: "Must be a valid address" + }); + let timeRestriction = await currentTransferManager.methods.whitelist(investorToCheck).call(); + console.log(`Sale lockup: ${moment.unix(timeRestriction.fromTime).format('MMMM Do YYYY, HH:mm:ss')}`); + console.log(`Buy lockup: ${moment.unix(timeRestriction.toTime).format('MMMM Do YYYY, HH:mm:ss')}`); + console.log(`KYC expiry time: ${moment.unix(timeRestriction.expiryTime).format('MMMM Do YYYY, HH:mm:ss')}`); + console.log(`Restricted investor: ${timeRestriction.canBuyFromSTO ? 'YES' : 'NO'} `); + break; case 'Modify whitelist': let investor = readlineSync.question('Enter the address to whitelist: ', { limit: function (input) { @@ -369,10 +377,10 @@ async function generalTransferManager() { limitMessage: "Must be a valid address" }); let now = Math.floor(Date.now() / 1000); - let fromTime = readlineSync.questionInt(`Enter the time (Unix Epoch time) when the sale lockup period ends and the investor can freely sell his tokens (now = ${now}): `, { defaultInput: now }); - let toTime = readlineSync.questionInt(`Enter the time (Unix Epoch time) when the purchase lockup period ends and the investor can freely purchase tokens from others (now = ${now}): `, { defaultInput: now }); + let fromTime = readlineSync.questionInt(`Enter the time(Unix Epoch time) when the sale lockup period ends and the investor can freely sell his tokens(now = ${now}): `, { defaultInput: now }); + let toTime = readlineSync.questionInt(`Enter the time(Unix Epoch time) when the purchase lockup period ends and the investor can freely purchase tokens from others(now = ${now}): `, { defaultInput: now }); let oneHourFromNow = Math.floor(Date.now() / 1000 + 3600); - let expiryTime = readlineSync.questionInt(`Enter the time till investors KYC will be validated (after that investor need to do re-KYC) (1 hour from now = ${oneHourFromNow}): `, { defaultInput: oneHourFromNow }); + let expiryTime = readlineSync.questionInt(`Enter the time till investors KYC will be validated(after that investor need to do re - KYC) (1 hour from now = ${oneHourFromNow}): `, { defaultInput: oneHourFromNow }); let canBuyFromSTO = readlineSync.keyInYNStrict('Is the investor a restricted investor?'); let modifyWhitelistAction = currentTransferManager.methods.modifyWhitelist(investor, fromTime, toTime, expiryTime, canBuyFromSTO); let modifyWhitelistReceipt = await common.sendTransaction(modifyWhitelistAction); @@ -380,10 +388,7 @@ async function generalTransferManager() { console.log(chalk.green(`${modifyWhitelistEvent._investor} has been whitelisted sucessfully!`)); break; case 'Modify whitelist from CSV': - console.log(chalk.yellow(`Data is going to be read from 'data/whitelist_data.csv'. Be sure this file is updated!`)); - if (readlineSync.keyInYNStrict(`Do you want to continue?`)) { - await transferManager.startCSV(tokenSymbol); - } + await modifyWhitelistInBatch(); break; /* case 'Modify Whitelist Signed': @@ -403,7 +408,7 @@ async function generalTransferManager() { let modifyWhitelistSignedAction = currentTransferManager.methods.modifyWhitelistSigned(investorSigned, fromTimeSigned, toTimeSigned, expiryTimeSigned, canBuyFromSTOSigned); let modifyWhitelistSignedReceipt = await common.sendTransaction(Issuer, modifyWhitelistSignedAction, defaultGasPrice); let modifyWhitelistSignedEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, modifyWhitelistSignedReceipt.logs, 'ModifyWhitelist'); - console.log(chalk.green(`${modifyWhitelistSignedEvent._investor} has been whitelisted sucessfully!`)); + console.log(chalk.green(`${ modifyWhitelistSignedEvent._investor } has been whitelisted sucessfully!`)); break; */ case 'Change issuance address': @@ -477,173 +482,42 @@ async function generalTransferManager() { } } -async function startCSV(tokenSymbol, batchSize) { - securityToken = await csv_shared.start(tokenSymbol, batchSize); - - let result_processing = await csv_shared.read('./CLI/data/whitelist_data.csv', whitelist_processing); - distribData = result_processing.distribData; - fullFileData = result_processing.fullFileData; - badData = result_processing.badData; - - await saveInBlockchain(); - await finalResults(); -}; - -function whitelist_processing(csv_line) { - let isAddress = web3.utils.isAddress(csv_line[0]); - let sellValid = isValidDate(csv_line[1]) - let buyValid = isValidDate(csv_line[2]) - let kycExpiryDate = isValidDate(csv_line[3]) - let canBuyFromSTO = (typeof JSON.parse(csv_line[4].toLowerCase())) == "boolean" ? JSON.parse(csv_line[4].toLowerCase()) : "not-valid"; - - if (isAddress && - sellValid && - buyValid && - kycExpiryDate && - (canBuyFromSTO != "not-valid")) { - return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), sellValid, buyValid, kycExpiryDate, canBuyFromSTO)] - } else { - return [false, new Array(csv_line[0], sellValid, buyValid, kycExpiryDate, canBuyFromSTO)] - } -} - -function isValidDate(date) { - var matches = /^(\d{1,2})[-\/](\d{1,2})[-\/](\d{4})$/.exec(date); - if (matches == null) return false; - var d = matches[2]; - var m = matches[1] - 1; - var y = matches[3]; - var composedDate = new Date(y, m, d); - var timestampDate = composedDate.getTime() - - // For some reason these timestamps are being recorded +4 hours UTC - if (composedDate.getDate() == d && - composedDate.getMonth() == m && - composedDate.getFullYear() == y) { - return timestampDate / 1000 - } else { - return false - } -} - -async function saveInBlockchain() { - let gtmModules; - try { - gtmModules = await securityToken.methods.getModulesByName(web3.utils.toHex('GeneralTransferManager')).call(); - } catch (e) { - console.log("Please attach General Transfer module before launch this action.", e) - process.exit(0) - } - - generalTransferManager = new web3.eth.Contract(abis.generalTransferManager(), gtmModules[0]); - - console.log(` - ------------------------------------------------------- - ----- Sending buy/sell restrictions to blockchain ----- - ------------------------------------------------------- - `); - - for (let i = 0; i < distribData.length; i++) { - try { - let investorArray = [], fromTimesArray = [], toTimesArray = [], expiryTimeArray = [], canBuyFromSTOArray = []; - - // Splitting the user arrays to be organized by input - for (let j = 0; j < distribData[i].length; j++) { - investorArray.push(distribData[i][j][0]) - fromTimesArray.push(distribData[i][j][1]) - toTimesArray.push(distribData[i][j][2]) - expiryTimeArray.push(distribData[i][j][3]) - canBuyFromSTOArray.push(distribData[i][j][4]) - } - - let modifyWhitelistMultiAction = await generalTransferManager.methods.modifyWhitelistMulti(investorArray, fromTimesArray, toTimesArray, expiryTimeArray, canBuyFromSTOArray); - let tx = await common.sendTransaction(modifyWhitelistMultiAction); - console.log(`Batch ${i} - Attempting to modifyWhitelist accounts:\n\n`, investorArray, "\n\n"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); - console.log("Whitelist transaxction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - - } catch (err) { - console.log("ERROR", err) - process.exit(0) - } - } - - return; -} - -async function finalResults() { - let totalInvestors = 0; - let updatedInvestors = 0; - let investorObjectLookup = {}; - let investorData_Events = new Array(); - let event_data = await generalTransferManager.getPastEvents('ModifyWhitelist', {fromBlock: 0, toBlock: 'latest'}, () => {}); - - for (var i = 0; i < event_data.length; i++) { - let combineArray = []; - - let investorAddress_Event = event_data[i].returnValues._investor - let fromTime_Event = event_data[i].returnValues._fromTime - let toTime_Event = event_data[i].returnValues._toTime - let expiryTime_Event = event_data[i].returnValues._expiryTime - let canBuyFromSTO_Event = event_data[i].returnValues._canBuyFromSTO - let blockNumber = event_data[i].blockNumber - - combineArray.push(investorAddress_Event); - combineArray.push(fromTime_Event); - combineArray.push(toTime_Event); - combineArray.push(expiryTime_Event); - combineArray.push(canBuyFromSTO_Event); - combineArray.push(blockNumber) - - investorData_Events.push(combineArray) - - // We have already recorded it, so this is an update to our object - if (investorObjectLookup.hasOwnProperty(investorAddress_Event)) { - // The block number form the event we are checking is bigger, so we gotta replace it - if (investorObjectLookup[investorAddress_Event].recordedBlockNumber < blockNumber) { - investorObjectLookup[investorAddress_Event] = {fromTime: fromTime_Event, toTime: toTime_Event, expiryTime: expiryTime_Event, canBuyFromSTO: canBuyFromSTO_Event, recordedBlockNumber: blockNumber}; - updatedInvestors += 1; - } - } else { - investorObjectLookup[investorAddress_Event] = {fromTime: fromTime_Event, toTime: toTime_Event, expiryTime: expiryTime_Event, canBuyFromSTO: canBuyFromSTO_Event, recordedBlockNumber: blockNumber}; - totalInvestors += 1; - } - } - - let investorAddress_Events = Object.keys(investorObjectLookup) - - console.log(`******************** EVENT LOGS ANALYSIS COMPLETE ********************\n`); - console.log(`A total of ${totalInvestors} investors have been whitelisted total, all time.\n`); - console.log(`This script in total sent ${fullFileData.length - badData.length} new investors and updated investors to the blockchain.\n`); - console.log(`There were ${badData.length} bad entries that didnt get sent to the blockchain in the script.\n`); - console.log("************************************************************************************************"); - console.log("OBJECT WITH EVERY USER AND THEIR UPDATED TIMES: \n\n", investorObjectLookup) - console.log("************************************************************************************************"); - console.log("LIST OF ALL INVESTORS WHITELISTED: \n\n", investorAddress_Events) - - let missingDistribs = []; - for (let l = 0; l < fullFileData.length; l++) { - if (!investorObjectLookup.hasOwnProperty(fullFileData[l][0])) { - missingDistribs.push(fullFileData[l]) - } +async function modifyWhitelistInBatch() { + let csvFilePath = readlineSync.question(`Enter the path for csv data file(${WHITELIST_DATA_CSV}): `, { + defaultInput: WHITELIST_DATA_CSV + }); + let batchSize = readlineSync.question(`Enter the max number of records per transaction or batch size(${gbl.constants.DEFAULT_BATCH_SIZE}): `, { + limit: function (input) { + return parseInt(input) > 0; + }, + limitMessage: 'Must be greater than 0', + defaultInput: gbl.constants.DEFAULT_BATCH_SIZE + }); + let parsedData = csvParse(csvFilePath); + let validData = parsedData.filter(row => + web3.utils.isAddress(row[0]) && + moment.unix(row[1]).isValid() && + moment.unix(row[2]).isValid() && + moment.unix(row[3]).isValid() && + typeof row[4] === 'boolean' + ); + let invalidRows = parsedData.filter(row => !validData.includes(row)); + if (invalidRows.length > 0) { + console.log(chalk.red(`The following lines from csv file are not valid: ${invalidRows.map(r => parsedData.indexOf(r) + 1).join(',')} `)); } - - if (missingDistribs.length > 0) { - console.log("************************************************************************************************"); - console.log("-- No LogModifyWhitelist event was found for the following data arrays. Please review them manually --") - console.log(missingDistribs) - console.log("************************************************************************************************"); - } else { - console.log("\n************************************************************************************************"); - console.log("All accounts passed through from the CSV were successfully whitelisted, because we were able to read them all from events") - console.log("************************************************************************************************"); + let batches = common.splitIntoBatches(validData, batchSize); + let [investorArray, fromTimesArray, toTimesArray, expiryTimeArray, canBuyFromSTOArray] = common.transposeBatches(batches); + for (let batch = 0; batch < batches.length; batch++) { + console.log(`Batch ${batch + 1} - Attempting to change accredited accounts: \n\n`, investorArray[batch], '\n'); + let action = await currentTransferManager.methods.modifyWhitelistMulti(investorArray[batch], fromTimesArray[batch], toTimesArray[batch], expiryTimeArray[batch], canBuyFromSTOArray[batch]); + let receipt = await common.sendTransaction(action); + console.log(chalk.green('Whitelist transaction was successful.')); + console.log(`${receipt.gasUsed} gas used.Spent: ${web3.utils.fromWei((new web3.utils.BN(receipt.gasUsed)).mul(new web3.utils.BN(defaultGasPrice)))} ETH`); } - } async function manualApprovalTransferManager() { - console.log(chalk.blue(`Manual Approval Transfer Manager at ${currentTransferManager.options.address}`), '\n'); + console.log(chalk.blue(`Manual Approval Transfer Manager at ${currentTransferManager.options.address} `), '\n'); let options = ['Check manual approval', 'Add manual approval', 'Revoke manual approval', 'Check manual blocking', 'Add manual blocking', 'Revoke manual blocking']; @@ -671,8 +545,8 @@ async function manualApprovalTransferManager() { let manualApproval = await getManualApproval(from, to); if (manualApproval) { console.log(`Manual approval found!`); - console.log(`Allowance: ${web3.utils.fromWei(manualApproval.allowance)}`); - console.log(`Expiry time: ${moment.unix(manualApproval.expiryTime).format('MMMM Do YYYY, HH:mm:ss')};`) + console.log(`Allowance: ${web3.utils.fromWei(manualApproval.allowance)} `); + console.log(`Expiry time: ${moment.unix(manualApproval.expiryTime).format('MMMM Do YYYY, HH:mm:ss')}; `) } else { console.log(chalk.yellow(`There are no manual approvals from ${from} to ${to}.`)); } @@ -693,13 +567,13 @@ async function manualApprovalTransferManager() { if (!await getManualApproval(from, to)) { let allowance = readlineSync.question('Enter the amount of tokens which will be approved: '); let oneHourFromNow = Math.floor(Date.now() / 1000 + 3600); - let expiryTime = readlineSync.questionInt(`Enter the time (Unix Epoch time) until which the transfer is allowed (1 hour from now = ${oneHourFromNow}): `, { defaultInput: oneHourFromNow }); + let expiryTime = readlineSync.questionInt(`Enter the time(Unix Epoch time) until which the transfer is allowed(1 hour from now = ${oneHourFromNow}): `, { defaultInput: oneHourFromNow }); let addManualApprovalAction = currentTransferManager.methods.addManualApproval(from, to, web3.utils.toWei(allowance), expiryTime); let addManualApprovalReceipt = await common.sendTransaction(addManualApprovalAction); let addManualApprovalEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, addManualApprovalReceipt.logs, 'AddManualApproval'); console.log(chalk.green(`Manual approval has been added successfully!`)); } else { - console.log(chalk.red(`A manual approval already exists from ${from} to ${to}. Revoke it first if you want to add a new one.`)); + console.log(chalk.red(`A manual approval already exists from ${from} to ${to}.Revoke it first if you want to add a new one.`)); } break; case 'Revoke manual approval': @@ -741,7 +615,7 @@ async function manualApprovalTransferManager() { let manualBlocking = await getManualBlocking(from, to); if (manualBlocking) { console.log(`Manual blocking found!`); - console.log(`Expiry time: ${moment.unix(manualBlocking).format('MMMM Do YYYY, HH:mm:ss')};`) + console.log(`Expiry time: ${moment.unix(manualBlocking).format('MMMM Do YYYY, HH:mm:ss')}; `) } else { console.log(chalk.yellow(`There are no manual blockings from ${from} to ${to}.`)); } @@ -761,13 +635,13 @@ async function manualApprovalTransferManager() { }); if (!await getManualBlocking(from, to)) { let oneHourFromNow = Math.floor(Date.now() / 1000 + 3600); - let expiryTime = readlineSync.questionInt(`Enter the time (Unix Epoch time) until which the transfer is blocked (1 hour from now = ${oneHourFromNow}): `, { defaultInput: oneHourFromNow }); + let expiryTime = readlineSync.questionInt(`Enter the time(Unix Epoch time) until which the transfer is blocked(1 hour from now = ${oneHourFromNow}): `, { defaultInput: oneHourFromNow }); let addManualBlockingAction = currentTransferManager.methods.addManualBlocking(from, to, expiryTime); let addManualBlockingReceipt = await common.sendTransaction(addManualBlockingAction); let addManualBlockingEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, addManualBlockingReceipt.logs, 'AddManualBlocking'); console.log(chalk.green(`Manual blocking has been added successfully!`)); } else { - console.log(chalk.red(`A manual blocking already exists from ${from} to ${to}. Revoke it first if you want to add a new one.`)); + console.log(chalk.red(`A manual blocking already exists from ${from} to ${to}.Revoke it first if you want to add a new one.`)); } break; case 'Revoke manual blocking': @@ -818,7 +692,7 @@ async function getManualBlocking(_from, _to) { } async function singleTradeVolumeRestrictionTM() { - console.log(chalk.blue(`Single Trade Volume Restriction Transfer Manager at ${currentTransferManager.options.address}`)); + console.log(chalk.blue(`Single Trade Volume Restriction Transfer Manager at ${currentTransferManager.options.address} `)); console.log(); // Show current data @@ -831,9 +705,9 @@ async function singleTradeVolumeRestrictionTM() { } let displayAllowPrimaryIssuance = await currentTransferManager.methods.allowPrimaryIssuance().call(); - console.log(`- Limit type: ${displayIsInPercentage ? `Percentage` : `Tokens`}`); - console.log(`- Default transfer limit: ${displayGlobalTransferLimit} ${displayIsInPercentage ? `%` : `${tokenSymbol}`}`); - console.log(`- Allow primary issuance: ${displayAllowPrimaryIssuance ? `YES` : `NO`}`); + console.log(`- Limit type: ${displayIsInPercentage ? `Percentage` : `Tokens`} `); + console.log(`- Default transfer limit: ${displayGlobalTransferLimit} ${displayIsInPercentage ? `%` : `${tokenSymbol}`} `); + console.log(`- Allow primary issuance: ${displayAllowPrimaryIssuance ? `YES` : `NO`} `); // ------------------ let options = []; @@ -895,7 +769,7 @@ async function singleTradeVolumeRestrictionTM() { let changeTransferLimitToTokensReceipt = await common.sendTransaction(changeTransferLimitToTokensAction); let changeTransferLimitToTokensEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, changeTransferLimitToTokensReceipt.logs, 'GlobalTransferLimitInTokensSet'); console.log(chalk.green(`Transfer limit has been set to tokens sucessfully!`)); - console.log(chalk.green(`The default transfer limit is ${web3.utils.fromWei(changeTransferLimitToTokensEvent._amount)} ${tokenSymbol}`)); + console.log(chalk.green(`The default transfer limit is ${web3.utils.fromWei(changeTransferLimitToTokensEvent._amount)} ${tokenSymbol} `)); break; case 'Change transfer limit to percentage': let newDefaultLimitInPercentage = toWeiPercentage(readlineSync.question('Enter the percentage for default limit: ', { @@ -908,7 +782,7 @@ async function singleTradeVolumeRestrictionTM() { let changeTransferLimitToPercentageReceipt = await common.sendTransaction(changeTransferLimitToPercentageAction); let changeTransferLimitToPercentageEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, changeTransferLimitToPercentageReceipt.logs, 'GlobalTransferLimitInPercentageSet'); console.log(chalk.green(`Transfer limit has been set to tokens sucessfully!`)); - console.log(chalk.green(`The default transfer limit is ${fromWeiPercentage(changeTransferLimitToPercentageEvent._percentage)} %`)); + console.log(chalk.green(`The default transfer limit is ${fromWeiPercentage(changeTransferLimitToPercentageEvent._percentage)} % `)); break; case 'Change default percentage limit': let defaultLimitInPercentage = toWeiPercentage(readlineSync.question('Enter the percentage for default limit: ', { @@ -920,7 +794,7 @@ async function singleTradeVolumeRestrictionTM() { let changeGlobalLimitInPercentageAction = currentTransferManager.methods.changeGlobalLimitInPercentage(defaultLimitInPercentage); let changeGlobalLimitInPercentageReceipt = await common.sendTransaction(changeGlobalLimitInPercentageAction); let changeGlobalLimitInPercentageEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, changeGlobalLimitInPercentageReceipt.logs, 'GlobalTransferLimitInPercentageSet'); - console.log(chalk.green(`The default transfer limit is ${fromWeiPercentage(changeGlobalLimitInPercentageEvent._percentage)} %`)); + console.log(chalk.green(`The default transfer limit is ${fromWeiPercentage(changeGlobalLimitInPercentageEvent._percentage)} % `)); break; case 'Change default tokens limit': let defaultLimitInTokens = web3.utils.toWei(readlineSync.question('Enter the amount of tokens for default limit: ', { @@ -932,7 +806,7 @@ async function singleTradeVolumeRestrictionTM() { let changeGlobalLimitInTokensAction = currentTransferManager.methods.changeGlobalLimitInTokens(defaultLimitInTokens); let changeGlobalLimitInTokensReceipt = await common.sendTransaction(changeGlobalLimitInTokensAction); let changeGlobalLimitInTokensEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, changeGlobalLimitInTokensReceipt.logs, 'GlobalTransferLimitInTokensSet'); - console.log(chalk.green(`The default transfer limit is ${web3.utils.fromWei(changeGlobalLimitInTokensEvent._amount)} ${tokenSymbol}`)); + console.log(chalk.green(`The default transfer limit is ${web3.utils.fromWei(changeGlobalLimitInTokensEvent._amount)} ${tokenSymbol} `)); break; case 'Set percentage transfer limit per account': let percentageAccount = readlineSync.question('Enter the wallet: ', { @@ -950,7 +824,7 @@ async function singleTradeVolumeRestrictionTM() { let setTransferLimitInPercentageAction = currentTransferManager.methods.setTransferLimitInPercentage(percentageAccount, accountLimitInPercentage); let setTransferLimitInPercentageReceipt = await common.sendTransaction(setTransferLimitInPercentageAction); let setTransferLimitInPercentageEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, setTransferLimitInPercentageReceipt.logs, 'TransferLimitInPercentageSet'); - console.log(chalk.green(`The transfer limit for ${setTransferLimitInPercentageEvent._wallet} is ${fromWeiPercentage(setTransferLimitInPercentageEvent._percentage)} %`)); + console.log(chalk.green(`The transfer limit for ${setTransferLimitInPercentageEvent._wallet} is ${fromWeiPercentage(setTransferLimitInPercentageEvent._percentage)} % `)); break; case 'Set tokens transfer limit per account': let tokensAccount = readlineSync.question('Enter the wallet: ', { @@ -968,7 +842,7 @@ async function singleTradeVolumeRestrictionTM() { let setTransferLimitInTokensAction = currentTransferManager.methods.setTransferLimitInTokens(tokensAccount, accountLimitInTokens); let setTransferLimitInTokensReceipt = await common.sendTransaction(setTransferLimitInTokensAction); let setTransferLimitInTokensEvent = common.getEventFromLogs(currentTransferManager._jsonInterface, setTransferLimitInTokensReceipt.logs, 'TransferLimitInTokensSet'); - console.log(chalk.green(`The transfer limit for ${setTransferLimitInTokensEvent._wallet} is ${web3.utils.fromWei(setTransferLimitInTokensEvent._amount)} ${tokenSymbol}`)); + console.log(chalk.green(`The transfer limit for ${setTransferLimitInTokensEvent._wallet} is ${web3.utils.fromWei(setTransferLimitInTokensEvent._amount)} ${tokenSymbol} `)); break; case 'Remove percentage transfer limit per account': let percentageAccountToRemove = readlineSync.question('Enter the wallet to remove: ', { @@ -1007,8 +881,8 @@ function signData(tmAddress, investorAddress, fromTime, toTime, expiryTime, rest ) .slice(2); packedData = new Buffer(packedData, "hex"); - packedData = Buffer.concat([new Buffer(`\x19Ethereum Signed Message:\n${packedData.length.toString()}`), packedData]); - packedData = web3.sha3(`0x${packedData.toString("hex")}`, { encoding: "hex" }); + packedData = Buffer.concat([new Buffer(`\x19Ethereum Signed Message: \n${ packedData.length.toString() } `), packedData]); + packedData = web3.sha3(`0x${ packedData.toString("hex") } `, { encoding: "hex" }); return ethUtil.ecsign(new Buffer(packedData.slice(2), "hex"), new Buffer(pk, "hex")); } */ @@ -1098,7 +972,7 @@ async function selectToken() { return { symbol: tokenData[0], address: t }; })); let options = tokenDataArray.map(function (t) { - return `${t.symbol} - Deployed at ${t.address}`; + return `${t.symbol} - Deployed at ${t.address} `; }); options.push('Enter token symbol manually'); @@ -1126,8 +1000,5 @@ module.exports = { addTransferManagerModule: async function (_tokenSymbol) { await initialize(_tokenSymbol); return addTransferManagerModule() - }, - startCSV: async function (tokenSymbol, batchSize) { - return startCSV(tokenSymbol, batchSize); } } \ No newline at end of file diff --git a/CLI/polymath-cli.js b/CLI/polymath-cli.js index 524c167cb..0ffba10dd 100644 --- a/CLI/polymath-cli.js +++ b/CLI/polymath-cli.js @@ -107,15 +107,6 @@ program await transfer_ownership.executeApp(contractAddress, transferTo); }); -program - .command('whitelist [batchSize]') - .alias('w') - .description('Mass-update a whitelist of allowed/known investors') - .action(async function (tokenSymbol, batchSize) { - await gbl.initialize(program.remoteNode); - await transfer_manager.startCSV(tokenSymbol, batchSize); - }); - program .command('dividends_manager [dividendsType]') .alias('dm') From e4252109678f58f54fb03e57802490f763a50209 Mon Sep 17 00:00:00 2001 From: Victor Date: Mon, 3 Dec 2018 10:06:11 -0300 Subject: [PATCH 11/12] Fix --- CLI/commands/common/common_functions.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CLI/commands/common/common_functions.js b/CLI/commands/common/common_functions.js index a59b9ac21..c19834037 100644 --- a/CLI/commands/common/common_functions.js +++ b/CLI/commands/common/common_functions.js @@ -155,9 +155,11 @@ module.exports = { }, transposeBatches: function (batches) { let result = []; - let columns = batches[0][0].length; - for (let index = 0; index < columns; index++) { - result[index] = batches.map(batch => batch.map(record => record[index])); + if (batches.length > 0 && batches[0].length > 0) { + let columns = batches[0][0].length; + for (let index = 0; index < columns; index++) { + result[index] = batches.map(batch => batch.map(record => record[index])); + } } return result; } From ee54b6d907d3f88d2a577cb12da17ad067f355fe Mon Sep 17 00:00:00 2001 From: Victor Date: Mon, 3 Dec 2018 10:07:05 -0300 Subject: [PATCH 12/12] Multi mint in batches --- CLI/commands/common/constants.js | 3 +- CLI/commands/sto_manager.js | 2 +- CLI/commands/token_manager.js | 252 ++++++++----------------------- CLI/commands/transfer_manager.js | 6 +- CLI/polymath-cli.js | 18 +-- 5 files changed, 75 insertions(+), 206 deletions(-) diff --git a/CLI/commands/common/constants.js b/CLI/commands/common/constants.js index bb9c8f69c..30a3bf699 100644 --- a/CLI/commands/common/constants.js +++ b/CLI/commands/common/constants.js @@ -31,5 +31,6 @@ module.exports = Object.freeze({ POLY: 1, DAI: 2 }, - DEFAULT_BATCH_SIZE: 75 + DEFAULT_BATCH_SIZE: 75, + ADDRESS_ZERO: '0x0000000000000000000000000000000000000000' }); \ No newline at end of file diff --git a/CLI/commands/sto_manager.js b/CLI/commands/sto_manager.js index d5bd58a4d..b66f3d80e 100644 --- a/CLI/commands/sto_manager.js +++ b/CLI/commands/sto_manager.js @@ -779,7 +779,7 @@ async function changeNonAccreditedLimitsInBatch(currentSTO) { let batches = common.splitIntoBatches(validData, batchSize); let [investorArray, limitArray] = common.transposeBatches(batches); for (let batch = 0; batch < batches.length; batch++) { - console.log(`Batch ${batch + 1} - Attempting to change accredited accounts:\n\n`, investorArray[batch], '\n'); + console.log(`Batch ${batch + 1} - Attempting to change non accredited limit to accounts:\n\n`, investorArray[batch], '\n'); let action = await currentSTO.methods.changeNonAccreditedLimit(investorArray[batch], limitArray[batch]); let receipt = await common.sendTransaction(action); console.log(chalk.green('Change non accredited limits transaction was successful.')); diff --git a/CLI/commands/token_manager.js b/CLI/commands/token_manager.js index e8556450d..e10f81467 100644 --- a/CLI/commands/token_manager.js +++ b/CLI/commands/token_manager.js @@ -5,8 +5,10 @@ const stoManager = require('./sto_manager'); const transferManager = require('./transfer_manager'); const common = require('./common/common_functions'); const gbl = require('./common/global'); -const csv_shared = require('./common/csv_shared'); -const BigNumber = require('bignumber.js'); +const csvParse = require('./helpers/csv'); + +// Constants +const MULTIMINT_DATA_CSV = './CLI/data/ST/multi_mint_data.csv'; // Load contract artifacts const contracts = require('./helpers/contract_addresses'); @@ -16,19 +18,10 @@ let securityTokenRegistry; let polyToken; let featureRegistry; let securityToken; -let tokenDivisible; let allModules; let tokenSymbol -/* CSV data control */ -let distribData = new Array(); -let fullFileData = new Array(); -let badData = new Array(); -let affiliatesFailedArray = new Array(); -let affiliatesKYCInvalidArray = new Array(); -/* End CSV data control */ - async function setup() { try { let securityTokenRegistryAddress = await contracts.securityTokenRegistry(); @@ -289,7 +282,7 @@ async function listInvestorsAtCheckpoint(checkpointId) { } async function mintTokens() { - let options = ['Modify whitelist', 'Mint tokens to a single address', `Modify whitelist and mint tokens using data from 'whitelist_data.csv' and 'multi_mint_data.csv'`]; + let options = ['Modify whitelist', 'Mint tokens to a single address', `Mint tokens to multiple addresses from CSV`]; let index = readlineSync.keyInSelect(options, 'What do you want to do?', { cancel: 'Return' }); let selected = index == -1 ? 'Return' : options[index]; console.log('Selected:', selected); @@ -313,8 +306,9 @@ async function mintTokens() { let amount = readlineSync.question(`Enter the amount of tokens to mint: `); await mintToSingleAddress(receiver, amount); break; - case `Modify whitelist and mint tokens using data from 'whitelist_data.csv' and 'multi_mint_data.csv'`: - await multi_mint_tokens(); + case `Mint tokens to multiple addresses from CSV`: + console.log(chalk.yellow(`Investors should be previously whitelisted.`)); + await multiMint(); break; } } @@ -345,192 +339,67 @@ async function mintToSingleAddress(_investor, _amount) { } } -async function multi_mint_tokens() { - await transferManager.startCSV(tokenSymbol, 75); - console.log(chalk.green(`\nCongratulations! All the affiliates get succssfully whitelisted, Now its time to Mint the tokens\n`)); - console.log(chalk.red(`WARNING: `) + `Please make sure all the addresses that get whitelisted are only eligible to hold or get Security token\n`); - - await startCSV(tokenSymbol, 75) - console.log(chalk.green(`\nCongratulations! Tokens get successfully Minted and transferred to token holders`)); -} -/// - -async function startCSV(tokenSymbol, batchSize) { - securityToken = await csv_shared.start(tokenSymbol, batchSize); - - let result_processing = await csv_shared.read('./CLI/data/multi_mint_data.csv', multimint_processing); - distribData = result_processing.distribData; - fullFileData = result_processing.fullFileData; - badData = result_processing.badData; - - tokenDivisible = await securityToken.methods.granularity().call() == 1; - - await saveInBlockchain(); - await finalResults(); -} - -function multimint_processing(csv_line) { - let isAddress = web3.utils.isAddress(csv_line[0]); - let validToken = isValidToken(csv_line[1]); - - if (isAddress && - validToken) { - return [true, new Array(web3.utils.toChecksumAddress(csv_line[0]), validToken)] +async function multiMint(_csvFilePath, _batchSize) { + let csvFilePath; + if (typeof _csvFilePath !== 'undefined') { + csvFilePath = _csvFilePath; } else { - return [false, new Array(csv_line[0], csv_line[1])] + csvFilePath = readlineSync.question(`Enter the path for csv data file (${MULTIMINT_DATA_CSV}): `, { + defaultInput: MULTIMINT_DATA_CSV + }); } -} - -function isValidToken(token) { - var tokenAmount = parseFloat(token); - if (tokenDivisible) { - return tokenAmount + let batchSize; + if (typeof _batchSize !== 'undefined') { + batchSize = _batchSize; } else { - if ((tokenAmount % 1 == 0)) { - return tokenAmount; - } - return false - } -} - -async function saveInBlockchain() { - let gtmModules = await securityToken.methods.getModulesByType(3).call(); - - if (gtmModules.length > 0) { - console.log("Minting of tokens is only allowed before the STO get attached"); - process.exit(0); - } - - console.log(` - ----------------------------------------- - ----- Mint the tokens to affiliates ----- - ----------------------------------------- - `); - - for (let i = 0; i < distribData.length; i++) { - try { - let affiliatesVerifiedArray = [], tokensVerifiedArray = []; - - // Splitting the user arrays to be organized by input - for (let j = 0; j < distribData[i].length; j++) { - let investorAccount = distribData[i][j][0]; - let tokenAmount = web3.utils.toWei((distribData[i][j][1]).toString(), "ether"); - let verifiedTransaction = await securityToken.methods.verifyTransfer("0x0000000000000000000000000000000000000000", investorAccount, tokenAmount, web3.utils.fromAscii('')).call(); - if (verifiedTransaction) { - affiliatesVerifiedArray.push(investorAccount); - tokensVerifiedArray.push(tokenAmount); - } else { - let gtmModule = await securityToken.methods.getModulesByName(web3.utils.toHex('GeneralTransferManager')).call(); - let generalTransferManager = new web3.eth.Contract(abis.generalTransferManager(), gtmModule[0]); - let validKYC = (await generalTransferManager.methods.whitelist(Issuer.address).call()).expiryTime > Math.floor(Date.now()/1000); - if (validKYC) { - affiliatesFailedArray.push(investorAccount); - } else { - affiliatesKYCInvalidArray.push(investorAccount); - } - } - } - - let mintMultiAction = await securityToken.methods.mintMulti(affiliatesVerifiedArray, tokensVerifiedArray); - let tx = await common.sendTransaction(mintMultiAction); - console.log(`Batch ${i} - Attempting to send the Minted tokens to affiliates accounts:\n\n`, affiliatesVerifiedArray, "\n\n"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------"); - console.log("Multi Mint transaction was successful.", tx.gasUsed, "gas used. Spent:", web3.utils.fromWei(BigNumber(tx.gasUsed * defaultGasPrice).toString(), "ether"), "Ether"); - console.log("---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------\n\n"); - - } catch (err) { - console.log("ERROR", err) - process.exit(0) - } - } - - return; -} - -async function finalResults() { - let totalInvestors = 0; - let updatedInvestors = 0; - let investorObjectLookup = {}; - let investorData_Events = new Array(); - - let event_data = await securityToken.getPastEvents('Minted', {fromBlock: 0, toBlock: 'latest'}, () => {}); - - for (var i = 0; i < event_data.length; i++) { - let combineArray = []; - - let investorAddress_Event = event_data[i].returnValues._to; - let amount_Event = event_data[i].returnValues._value; - let blockNumber = event_data[i].blockNumber; - - combineArray.push(investorAddress_Event); - combineArray.push(amount_Event); - combineArray.push(blockNumber); - - investorData_Events.push(combineArray) - - // We have already recorded it, so this is an update to our object - if (investorObjectLookup.hasOwnProperty(investorAddress_Event)) { - // The block number form the event we are checking is bigger, so we gotta replace it - if (investorObjectLookup[investorAddress_Event].recordedBlockNumber < blockNumber) { - investorObjectLookup[investorAddress_Event] = {amount: amount_Event, recordedBlockNumber: blockNumber}; - updatedInvestors += 1; - } + batchSize = readlineSync.question(`Enter the max number of records per transaction or batch size (${gbl.constants.DEFAULT_BATCH_SIZE}): `, { + limit: function (input) { + return parseInt(input) > 0; + }, + limitMessage: 'Must be greater than 0', + defaultInput: gbl.constants.DEFAULT_BATCH_SIZE + }); + } + let parsedData = csvParse(csvFilePath); + let tokenDivisible = await securityToken.methods.granularity().call() == 1; + let validData = parsedData.filter(row => + web3.utils.isAddress(row[0]) && + (!isNaN(row[1]) && (tokenDivisible || parseFloat(row[1]) % 1 == 0)) + ); + let invalidRows = parsedData.filter(row => !validData.includes(row)); + if (invalidRows.length > 0) { + console.log(chalk.red(`The following lines from csv file are not valid: ${invalidRows.map(r => parsedData.indexOf(r) + 1).join(',')} `)); + } + let verifiedData = []; + let unverifiedData = []; + for (const row of validData) { + let investorAccount = row[0]; + let tokenAmount = web3.utils.toWei(row[1].toString()); + let verifiedTransaction = await securityToken.methods.verifyTransfer(gbl.constants.ADDRESS_ZERO, investorAccount, tokenAmount, web3.utils.fromAscii('')).call(); + if (verifiedTransaction) { + verifiedData.push(row); } else { - investorObjectLookup[investorAddress_Event] = {amount: amount_Event, recordedBlockNumber: blockNumber}; - totalInvestors += 1; + unverifiedData.push(row); } } - let investorAddress_Events = Object.keys(investorObjectLookup) - - console.log(`******************** EVENT LOGS ANALYSIS COMPLETE ********************\n`); - console.log(`A total of ${totalInvestors} affiliated investors get the token\n`); - console.log(`This script in total sent ${fullFileData.length - badData.length - affiliatesFailedArray.length - affiliatesKYCInvalidArray.length} new investors and updated investors to the blockchain.\n`); - console.log(`There were ${badData.length} bad entries that didnt get sent to the blockchain in the script.\n`); - console.log(`There were ${affiliatesKYCInvalidArray.length} accounts with invalid KYC.\n`); - console.log(`There were ${affiliatesFailedArray.length} accounts that didn't get sent to the blockchain as they would fail.\n`); - console.log("************************************************************************************************"); - console.log("OBJECT WITH EVERY USER AND THEIR MINTED TOKEN: \n\n", investorObjectLookup) - console.log("************************************************************************************************"); - console.log("LIST OF ALL INVESTORS WHO GOT THE MINTED TOKENS: \n\n", investorAddress_Events) - - let missingDistribs = [], failedVerificationDistribs = [], invalidKYCDistribs = []; - for (let l = 0; l < fullFileData.length; l++) { - if (affiliatesKYCInvalidArray.includes(fullFileData[l][0])) { - invalidKYCDistribs.push(fullFileData[l]); - } else if (affiliatesFailedArray.includes(fullFileData[l][0])) { - failedVerificationDistribs.push(fullFileData[l]); - } else if (!investorObjectLookup.hasOwnProperty(fullFileData[l][0])) { - missingDistribs.push(fullFileData[l]); - } + let batches = common.splitIntoBatches(verifiedData, batchSize); + let [investorArray, amountArray] = common.transposeBatches(batches); + for (let batch = 0; batch < batches.length; batch++) { + console.log(`Batch ${batch + 1} - Attempting to mint tokens to accounts: \n\n`, investorArray[batch], '\n'); + amountArray[batch] = amountArray[batch].map(a => web3.utils.toWei(a.toString())); + let action = await securityToken.methods.mintMulti(investorArray[batch], amountArray[batch]); + let receipt = await common.sendTransaction(action); + console.log(chalk.green('Multi mint transaction was successful.')); + console.log(`${receipt.gasUsed} gas used.Spent: ${web3.utils.fromWei((new web3.utils.BN(receipt.gasUsed)).mul(new web3.utils.BN(defaultGasPrice)))} ETH`); } - if (invalidKYCDistribs.length > 0) { - console.log("**************************************************************************************************************************"); - console.log("The following data arrays have an invalid KYC. Please review if these accounts are whitelisted and their KYC is not expired\n"); - console.log(invalidKYCDistribs); - console.log("**************************************************************************************************************************"); - } - if (failedVerificationDistribs.length > 0) { + if (unverifiedData.length > 0) { console.log("*********************************************************************************************************"); - console.log("-- The following data arrays failed at verifyTransfer. Please review if these accounts are whitelisted --\n"); - console.log(failedVerificationDistribs); + console.log('The following data arrays failed at verifyTransfer. Please review if these accounts are whitelisted\n'); + console.log(chalk.red(unverifiedData.map(d => `${d[0]}, ${d[1]}`).join('\n'))); console.log("*********************************************************************************************************"); } - if (missingDistribs.length > 0) { - console.log("******************************************************************************************"); - console.log("-- No Minted event was found for the following data arrays. Please review them manually --\n"); - console.log(missingDistribs); - console.log("******************************************************************************************"); - } - if (missingDistribs.length == 0 && - failedVerificationDistribs.length == 0 && - invalidKYCDistribs.length == 0) { - console.log("\n**************************************************************************************************************************"); - console.log("All accounts passed through from the CSV were successfully get the tokens, because we were able to read them all from events"); - console.log("****************************************************************************************************************************"); - } - } async function withdrawFromContract(erc20address, value) { @@ -824,10 +693,11 @@ async function selectToken() { module.exports = { executeApp: async function (_tokenSymbol) { - await initialize(_tokenSymbol) + await initialize(_tokenSymbol); return executeApp(); }, - startCSV: async function (tokenSymbol, batchSize) { - return startCSV(tokenSymbol, batchSize); + multiMint: async function (_tokenSymbol, _csvPath, _batchSize) { + await initialize(_tokenSymbol); + return multiMint(_csvPath, _batchSize); } } diff --git a/CLI/commands/transfer_manager.js b/CLI/commands/transfer_manager.js index ca2f4ee06..ee42b2fa4 100644 --- a/CLI/commands/transfer_manager.js +++ b/CLI/commands/transfer_manager.js @@ -483,10 +483,10 @@ async function generalTransferManager() { } async function modifyWhitelistInBatch() { - let csvFilePath = readlineSync.question(`Enter the path for csv data file(${WHITELIST_DATA_CSV}): `, { + let csvFilePath = readlineSync.question(`Enter the path for csv data file (${WHITELIST_DATA_CSV}): `, { defaultInput: WHITELIST_DATA_CSV }); - let batchSize = readlineSync.question(`Enter the max number of records per transaction or batch size(${gbl.constants.DEFAULT_BATCH_SIZE}): `, { + let batchSize = readlineSync.question(`Enter the max number of records per transaction or batch size (${gbl.constants.DEFAULT_BATCH_SIZE}): `, { limit: function (input) { return parseInt(input) > 0; }, @@ -508,7 +508,7 @@ async function modifyWhitelistInBatch() { let batches = common.splitIntoBatches(validData, batchSize); let [investorArray, fromTimesArray, toTimesArray, expiryTimeArray, canBuyFromSTOArray] = common.transposeBatches(batches); for (let batch = 0; batch < batches.length; batch++) { - console.log(`Batch ${batch + 1} - Attempting to change accredited accounts: \n\n`, investorArray[batch], '\n'); + console.log(`Batch ${batch + 1} - Attempting to modify whitelist to accounts: \n\n`, investorArray[batch], '\n'); let action = await currentTransferManager.methods.modifyWhitelistMulti(investorArray[batch], fromTimesArray[batch], toTimesArray[batch], expiryTimeArray[batch], canBuyFromSTOArray[batch]); let receipt = await common.sendTransaction(action); console.log(chalk.green('Whitelist transaction was successful.')); diff --git a/CLI/polymath-cli.js b/CLI/polymath-cli.js index 0ffba10dd..2ce101a4d 100644 --- a/CLI/polymath-cli.js +++ b/CLI/polymath-cli.js @@ -74,19 +74,17 @@ program .command('token_manager') .alias('stm') .option('-t, --securityToken ', 'Selects a ST to manage') + .option('-m, --multiMint ', 'Distribute tokens to previously whitelisted investors') + .option('-b, --batchSize ', 'Max number of records per transaction') .description('Manage your Security Tokens, mint tokens, add modules and change config') .action(async function (cmd) { await gbl.initialize(program.remoteNode); - await token_manager.executeApp(cmd.securityToken); - }); - -program - .command('multi_mint [batchSize]') - .alias('mi') - .description('Distribute tokens to previously whitelisted investors') - .action(async function (tokenSymbol, batchSize) { - await gbl.initialize(program.remoteNode); - await token_manager.startCSV(tokenSymbol, batchSize); + if (cmd.multiMint) { + let batchSize = cmd.batchSize ? cmd.batchSize : gbl.constants.DEFAULT_BATCH_SIZE; + await token_manager.multiMint(cmd.securityToken, cmd.multiMint, batchSize); + } else { + await token_manager.executeApp(cmd.securityToken); + } }); program