From f0d6ea95c8117340f8cf7add066edecc6ea52f0c Mon Sep 17 00:00:00 2001 From: Nicholas Cottrell Date: Tue, 16 Jul 2019 14:46:12 +0200 Subject: [PATCH 1/5] Create getMongoDataLite.js --- getMongoData/getMongoDataLite.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 getMongoData/getMongoDataLite.js diff --git a/getMongoData/getMongoDataLite.js b/getMongoData/getMongoDataLite.js new file mode 100644 index 00000000..3e453d25 --- /dev/null +++ b/getMongoData/getMongoDataLite.js @@ -0,0 +1,20 @@ +print("================================"); +print("DB Info"); +print("================================"); +db._adminCommand("listDatabases").databases.forEach( + function (d) { + mdb = db.getSiblingDB(d.name); + printjson(mdb.stats()); +}); +print("================================"); +print("Coll Info"); +print("================================"); +db.getMongo().getDBNames().forEach(function (name) { + var mdb = db.getSiblingDB(name); + print("\n\n\n======== DB: " + name + "========"); + printjson(mdb.stats()); + mdb.getCollectionNames().forEach(function(coll) { + print("\nCollection: " + mdb.getCollection(coll)); + printjson(mdb.getCollection(coll).stats()); + }); +}); From ff44dd9c10f2bc66d124a86f86b1af56397b9a3f Mon Sep 17 00:00:00 2001 From: Nic Cottrell Date: Wed, 17 Jul 2019 14:42:34 +0200 Subject: [PATCH 2/5] Switched to JSON output for easier processing --- getMongoData/getMongoDataLite.js | 29 ++++++++++++------- getMongoData/processLite.js | 49 ++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 11 deletions(-) create mode 100644 getMongoData/processLite.js diff --git a/getMongoData/getMongoDataLite.js b/getMongoData/getMongoDataLite.js index 3e453d25..2edbdfce 100644 --- a/getMongoData/getMongoDataLite.js +++ b/getMongoData/getMongoDataLite.js @@ -1,20 +1,27 @@ -print("================================"); -print("DB Info"); -print("================================"); +// +// Launch like: +// mongo --quiet < getMongoDataLite.js > out.json +// + +print("{\n\"dbstats\": ["); db._adminCommand("listDatabases").databases.forEach( - function (d) { + function (d, idx, array) { mdb = db.getSiblingDB(d.name); + // print(d.name + ": ") printjson(mdb.stats()); + if (idx < array.length - 1) print(",") }); -print("================================"); -print("Coll Info"); -print("================================"); -db.getMongo().getDBNames().forEach(function (name) { +print("]\n, \n\"collstats\": {"); +db.getMongo().getDBNames().forEach(function (name, idx, array) { var mdb = db.getSiblingDB(name); - print("\n\n\n======== DB: " + name + "========"); + print("\"" + name + "\": { \n\t\t\"stats\":"); printjson(mdb.stats()); - mdb.getCollectionNames().forEach(function(coll) { - print("\nCollection: " + mdb.getCollection(coll)); + print("\n\t, \"collections\": [") + mdb.getCollectionNames().forEach(function(coll, idx2, array2) { printjson(mdb.getCollection(coll).stats()); + if (idx2 < array2.length - 1) print(","); }); + print("\t]\n}") + if (idx < array.length - 1) print(","); }); +print("\n\t}\n}"); diff --git a/getMongoData/processLite.js b/getMongoData/processLite.js new file mode 100644 index 00000000..0d2b9b89 --- /dev/null +++ b/getMongoData/processLite.js @@ -0,0 +1,49 @@ +/** +* Launch with `node processLite.js` +*/ + +var fs = require('fs'); +var json = JSON.parse(fs.readFileSync('out.json', 'utf8')); + +dbs = [] +json.dbstats.forEach(function(element) { + if (element.db != 'admin' && element.db != 'config') { // TODO: what about `local`? + dbs.push(element.db); + } +}); +console.log(`DBs: ${dbs}`); +console.log(`DB Count: ${dbs.length}`); + +collections = [] // collection names +// collection via DB and collection (and assert they are the same) +objectsViaDb = 0 +bytesViaDb = 0 // data size +indexSizeTotal = 0 +storageSizeTotal = 0 + +objectsViaColls = 0 // total objects in ALL databases/collections +bytesViaColls = 0 + +Object.keys(json.collstats).forEach(function(key) { + if (key != 'admin' && key != 'config') { + collections.push(key); + var elem = json.collstats[key]; + // collection via db.stats() + objectsViaDb += elem.stats.objects; + bytesViaDb += elem.stats.dataSize; + indexSizeTotal += elem.stats.indexSize; + storageSizeTotal += elem.stats.storageSize; + // same for getCollectionNames + elem.collections.forEach(function(coll){ + objectsViaColls += coll.count; + bytesViaColls += coll.size; + }); +} +}); + +console.log(`objectsViaDb: ${objectsViaDb}`); +console.log(`objectsViaColls: ${objectsViaColls}`); +console.log(`bytesViaDb: ${bytesViaDb}`); +console.log(`bytesViaColls: ${bytesViaColls}`); +console.log(`indexSizeTotal: ${indexSizeTotal}`); +console.log(`storageSizeTotal: ${storageSizeTotal}`); From eebd580dfbeb62baeb8a8b3da27b1c9db2a5af65 Mon Sep 17 00:00:00 2001 From: Nic Cottrell Date: Wed, 17 Jul 2019 15:29:25 +0200 Subject: [PATCH 3/5] Added CLI options and optimized collection logic --- getMongoData/getMongoDataLite.js | 12 ++-------- getMongoData/processLite.js | 38 +++++++++++++++++--------------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/getMongoData/getMongoDataLite.js b/getMongoData/getMongoDataLite.js index 2edbdfce..1b777f5e 100644 --- a/getMongoData/getMongoDataLite.js +++ b/getMongoData/getMongoDataLite.js @@ -3,15 +3,7 @@ // mongo --quiet < getMongoDataLite.js > out.json // -print("{\n\"dbstats\": ["); -db._adminCommand("listDatabases").databases.forEach( - function (d, idx, array) { - mdb = db.getSiblingDB(d.name); - // print(d.name + ": ") - printjson(mdb.stats()); - if (idx < array.length - 1) print(",") -}); -print("]\n, \n\"collstats\": {"); +print("{"); db.getMongo().getDBNames().forEach(function (name, idx, array) { var mdb = db.getSiblingDB(name); print("\"" + name + "\": { \n\t\t\"stats\":"); @@ -24,4 +16,4 @@ db.getMongo().getDBNames().forEach(function (name, idx, array) { print("\t]\n}") if (idx < array.length - 1) print(","); }); -print("\n\t}\n}"); +print("\n}"); diff --git a/getMongoData/processLite.js b/getMongoData/processLite.js index 0d2b9b89..fc245d4c 100644 --- a/getMongoData/processLite.js +++ b/getMongoData/processLite.js @@ -2,18 +2,13 @@ * Launch with `node processLite.js` */ +var args = process.argv.slice(2); +var filename = args.length > 0 ? args[0] : 'out.json'; + var fs = require('fs'); -var json = JSON.parse(fs.readFileSync('out.json', 'utf8')); +var json = JSON.parse(fs.readFileSync(filename, 'utf8')); dbs = [] -json.dbstats.forEach(function(element) { - if (element.db != 'admin' && element.db != 'config') { // TODO: what about `local`? - dbs.push(element.db); - } -}); -console.log(`DBs: ${dbs}`); -console.log(`DB Count: ${dbs.length}`); - collections = [] // collection names // collection via DB and collection (and assert they are the same) objectsViaDb = 0 @@ -24,10 +19,10 @@ storageSizeTotal = 0 objectsViaColls = 0 // total objects in ALL databases/collections bytesViaColls = 0 -Object.keys(json.collstats).forEach(function(key) { +Object.keys(json).forEach(function(key) { if (key != 'admin' && key != 'config') { - collections.push(key); - var elem = json.collstats[key]; + dbs.push(key); + var elem = json[key]; // collection via db.stats() objectsViaDb += elem.stats.objects; bytesViaDb += elem.stats.dataSize; @@ -35,15 +30,22 @@ Object.keys(json.collstats).forEach(function(key) { storageSizeTotal += elem.stats.storageSize; // same for getCollectionNames elem.collections.forEach(function(coll){ + collections.push(coll.ns); objectsViaColls += coll.count; bytesViaColls += coll.size; }); } }); -console.log(`objectsViaDb: ${objectsViaDb}`); -console.log(`objectsViaColls: ${objectsViaColls}`); -console.log(`bytesViaDb: ${bytesViaDb}`); -console.log(`bytesViaColls: ${bytesViaColls}`); -console.log(`indexSizeTotal: ${indexSizeTotal}`); -console.log(`storageSizeTotal: ${storageSizeTotal}`); +console.log(`{`) +console.log(`\tdbs: "${dbs}",`); +console.log(`\tdbCount: ${dbs.length},`); +console.log(`\tcolls: "${collections}",`); +console.log(`\tcollCount: ${collections.length},`); +console.log(`\tobjectsViaDb: ${objectsViaDb},`); +console.log(`\tobjectsViaColls: ${objectsViaColls},`); +console.log(`\tbytesViaDb: ${bytesViaDb},`); +console.log(`\tbytesViaColls: ${bytesViaColls},`); +console.log(`\tindexSizeTotal: ${indexSizeTotal},`); +console.log(`\tstorageSizeTotal: ${storageSizeTotal},`); +console.log(`}`) From 258747f39dbe92e3b67016dae90555f0905062a2 Mon Sep 17 00:00:00 2001 From: Nic Cottrell Date: Fri, 19 Jul 2019 22:20:50 +0200 Subject: [PATCH 4/5] Improved reporting --- getMongoData/processLite.js | 48 +++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/getMongoData/processLite.js b/getMongoData/processLite.js index fc245d4c..50266e44 100644 --- a/getMongoData/processLite.js +++ b/getMongoData/processLite.js @@ -1,7 +1,8 @@ /** -* Launch with `node processLite.js` +* Launch with `node processLite.js` or `node processList.js ` */ +var markdown = true; // include markdown summary table var args = process.argv.slice(2); var filename = args.length > 0 ? args[0] : 'out.json'; @@ -19,6 +20,10 @@ storageSizeTotal = 0 objectsViaColls = 0 // total objects in ALL databases/collections bytesViaColls = 0 +if (markdown) { + console.log(`| Database | dataSize | storageSize | indexSize |`) + console.log(`| ---- | ---: | ---: | ----: |`) +} Object.keys(json).forEach(function(key) { if (key != 'admin' && key != 'config') { dbs.push(key); @@ -26,8 +31,13 @@ Object.keys(json).forEach(function(key) { // collection via db.stats() objectsViaDb += elem.stats.objects; bytesViaDb += elem.stats.dataSize; + // console.log(`dataSize ${key}: ${(elem.stats.dataSize/1024/1024/1024).toFixed(1)}GB`) indexSizeTotal += elem.stats.indexSize; + // console.log(`indexSize ${key}: ${(elem.stats.indexSize/1024/1024/1024).toFixed(1)}GB`) storageSizeTotal += elem.stats.storageSize; + // console.log(`storageSize ${key}: ${(elem.stats.storageSize/1024/1024/1024).toFixed(1)}GB`) + if (markdown) + console.log(`| ${key} | ${(elem.stats.dataSize/1024/1024/1024).toFixed(1)} | ${(elem.stats.storageSize/1024/1024/1024).toFixed(1)} | ${(elem.stats.indexSize/1024/1024/1024).toFixed(1)} | `) // same for getCollectionNames elem.collections.forEach(function(coll){ collections.push(coll.ns); @@ -37,15 +47,27 @@ Object.keys(json).forEach(function(key) { } }); -console.log(`{`) -console.log(`\tdbs: "${dbs}",`); -console.log(`\tdbCount: ${dbs.length},`); -console.log(`\tcolls: "${collections}",`); -console.log(`\tcollCount: ${collections.length},`); -console.log(`\tobjectsViaDb: ${objectsViaDb},`); -console.log(`\tobjectsViaColls: ${objectsViaColls},`); -console.log(`\tbytesViaDb: ${bytesViaDb},`); -console.log(`\tbytesViaColls: ${bytesViaColls},`); -console.log(`\tindexSizeTotal: ${indexSizeTotal},`); -console.log(`\tstorageSizeTotal: ${storageSizeTotal},`); -console.log(`}`) +console.log(`{\n` + + `\tdbs: ${JSON.stringify(dbs)},\n`+ + `\tdbCount: ${dbs.length},\n`+ + `\tcolls: ${JSON.stringify(collections)},\n`+ + `\tcollCount: ${collections.length},\n`+ + `\tobjectsViaDb: ${objectsViaDb},\n`+ + `\tobjectsViaColls: ${objectsViaColls},\n`+ + `\tbytesViaDb: {\n`+ + `\t\traw: ${bytesViaDb},\n`+ + `\t\tgb: ${bytesViaDb/1024/1024/1024}\n`+ + `\t},\n` + + `\tbytesViaColls: {\n`+ + `\t\traw: ${bytesViaColls},\n`+ + `\t\tgb: ${(bytesViaColls/1024/1024).toFixed(0)}\n`+ + `\t},\n` + + `\tindexSizeTotal: {\n`+ + `\t\traw: ${indexSizeTotal},\n`+ + `\t\tgb: ${(bytesViaColls/1024/1024).toFixed(0)},\n`+ + `\t},\n` + + `\tstorageSizeTotal: {\n`+ + `\t\traw: ${storageSizeTotal},\n`+ + `\t\tgb: ${(storageSizeTotal/1024/1024).toFixed(0)},\n`+ + `\t},\n` + + `}`) From 97ca85880fe7c1605f53c9dbd12b34122d0a2a20 Mon Sep 17 00:00:00 2001 From: Nic Cottrell Date: Mon, 22 Jul 2019 11:02:41 +0200 Subject: [PATCH 5/5] Fixed regex cleanup of all occurrences --- getMongoData/processLite.js | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/getMongoData/processLite.js b/getMongoData/processLite.js index 50266e44..0ca95925 100644 --- a/getMongoData/processLite.js +++ b/getMongoData/processLite.js @@ -6,23 +6,31 @@ var markdown = true; // include markdown summary table var args = process.argv.slice(2); var filename = args.length > 0 ? args[0] : 'out.json'; +console.log(`Reading data from ${filename}`); + var fs = require('fs'); -var json = JSON.parse(fs.readFileSync(filename, 'utf8')); +var content = fs.readFileSync(filename, 'utf8'); + +content = content.replace(/Timestamp\(.*\)/g, '1'); +content = content.replace(/NumberLong\((.*)\)/g, '$1'); +content = content.replace(/ObjectId\((.*)\)/g, '$1'); + +var json = JSON.parse(content); -dbs = [] -collections = [] // collection names +dbs = []; +collections = []; // collection names // collection via DB and collection (and assert they are the same) -objectsViaDb = 0 -bytesViaDb = 0 // data size -indexSizeTotal = 0 -storageSizeTotal = 0 +objectsViaDb = 0; +bytesViaDb = 0; // data size +indexSizeTotal = 0; +storageSizeTotal = 0; -objectsViaColls = 0 // total objects in ALL databases/collections -bytesViaColls = 0 +objectsViaColls = 0; // total objects in ALL databases/collections +bytesViaColls = 0; if (markdown) { - console.log(`| Database | dataSize | storageSize | indexSize |`) - console.log(`| ---- | ---: | ---: | ----: |`) + console.log(`| Database | dataSize | storageSize | indexSize |`); + console.log(`| ---- | ---: | ---: | ----: |`); } Object.keys(json).forEach(function(key) { if (key != 'admin' && key != 'config') { @@ -39,7 +47,7 @@ Object.keys(json).forEach(function(key) { if (markdown) console.log(`| ${key} | ${(elem.stats.dataSize/1024/1024/1024).toFixed(1)} | ${(elem.stats.storageSize/1024/1024/1024).toFixed(1)} | ${(elem.stats.indexSize/1024/1024/1024).toFixed(1)} | `) // same for getCollectionNames - elem.collections.forEach(function(coll){ + elem.collections.forEach(function(coll) { collections.push(coll.ns); objectsViaColls += coll.count; bytesViaColls += coll.size; @@ -70,4 +78,4 @@ console.log(`{\n` + `\t\traw: ${storageSizeTotal},\n`+ `\t\tgb: ${(storageSizeTotal/1024/1024).toFixed(0)},\n`+ `\t},\n` + - `}`) + `}`);