From dc4859f561dfae078d4529d8cfa4405ac4daf4c9 Mon Sep 17 00:00:00 2001 From: Peter Shin Date: Thu, 4 Feb 2016 08:18:19 -0800 Subject: [PATCH] Logs support. Added /logs endpoint with basic logger and LoggerAdapter. --- package.json | 3 +- spec/FileLoggerAdapter.spec.js | 64 +++++++ spec/LoggerController.spec.js | 55 ++++++ src/Adapters/Logger/FileLoggerAdapter.js | 225 +++++++++++++++++++++++ src/Adapters/Logger/LoggerAdapter.js | 17 ++ src/Controllers/LoggerController.js | 78 ++++++++ src/index.js | 9 +- 7 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 spec/FileLoggerAdapter.spec.js create mode 100644 spec/LoggerController.spec.js create mode 100644 src/Adapters/Logger/FileLoggerAdapter.js create mode 100644 src/Adapters/Logger/LoggerAdapter.js create mode 100644 src/Controllers/LoggerController.js diff --git a/package.json b/package.json index 521381f0ee..974679633e 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "node-gcm": "^0.14.0", "parse": "^1.7.0", "randomstring": "^1.1.3", - "request": "^2.65.0" + "request": "^2.65.0", + "winston": "^2.1.1" }, "devDependencies": { "babel-cli": "^6.5.1", diff --git a/spec/FileLoggerAdapter.spec.js b/spec/FileLoggerAdapter.spec.js new file mode 100644 index 0000000000..4466e087f2 --- /dev/null +++ b/spec/FileLoggerAdapter.spec.js @@ -0,0 +1,64 @@ +var FileLoggerAdapter = require('../src/Adapters/Logger/FileLoggerAdapter').FileLoggerAdapter; +var Parse = require('parse/node').Parse; +var request = require('request'); +var fs = require('fs'); + +var LOGS_FOLDER = './test_logs/'; + +var deleteFolderRecursive = function(path) { + if( fs.existsSync(path) ) { + fs.readdirSync(path).forEach(function(file,index){ + var curPath = path + "/" + file; + if(fs.lstatSync(curPath).isDirectory()) { // recurse + deleteFolderRecursive(curPath); + } else { // delete file + fs.unlinkSync(curPath); + } + }); + fs.rmdirSync(path); + } +}; + +describe('info logs', () => { + + afterEach((done) => { + deleteFolderRecursive(LOGS_FOLDER); + done(); + }); + + it("Verify INFO logs", (done) => { + var fileLoggerAdapter = new FileLoggerAdapter({ + logsFolder: LOGS_FOLDER + }); + fileLoggerAdapter.info('testing info logs', () => { + fileLoggerAdapter.query({ + size: 1, + level: 'info' + }, (results) => { + expect(results[0].message).toEqual('testing info logs'); + done(); + }); + }); + }); +}); + +describe('error logs', () => { + + afterEach((done) => { + deleteFolderRecursive(LOGS_FOLDER); + done(); + }); + + it("Verify ERROR logs", (done) => { + var fileLoggerAdapter = new FileLoggerAdapter(); + fileLoggerAdapter.error('testing error logs', () => { + fileLoggerAdapter.query({ + size: 1, + level: 'error' + }, (results) => { + expect(results[0].message).toEqual('testing error logs'); + done(); + }); + }); + }); +}); diff --git a/spec/LoggerController.spec.js b/spec/LoggerController.spec.js new file mode 100644 index 0000000000..f23004ab64 --- /dev/null +++ b/spec/LoggerController.spec.js @@ -0,0 +1,55 @@ +var LoggerController = require('../src/Controllers/LoggerController').LoggerController; +var FileLoggerAdapter = require('../src/Adapters/Logger/FileLoggerAdapter').FileLoggerAdapter; + +describe('LoggerController', () => { + it('can check valid master key of request', (done) => { + // Make mock request + var request = { + auth: { + isMaster: true + }, + query: {} + }; + + var loggerController = new LoggerController(new FileLoggerAdapter()); + + expect(() => { + loggerController.handleGET(request); + }).not.toThrow(); + done(); + }); + + it('can check invalid construction of controller', (done) => { + // Make mock request + var request = { + auth: { + isMaster: true + }, + query: {} + }; + + var loggerController = new LoggerController(); + + expect(() => { + loggerController.handleGET(request); + }).toThrow(); + done(); + }); + + it('can check invalid master key of request', (done) => { + // Make mock request + var request = { + auth: { + isMaster: false + }, + query: {} + }; + + var loggerController = new LoggerController(new FileLoggerAdapter()); + + expect(() => { + loggerController.handleGET(request); + }).toThrow(); + done(); + }); +}); diff --git a/src/Adapters/Logger/FileLoggerAdapter.js b/src/Adapters/Logger/FileLoggerAdapter.js new file mode 100644 index 0000000000..4edc412289 --- /dev/null +++ b/src/Adapters/Logger/FileLoggerAdapter.js @@ -0,0 +1,225 @@ +// Logger +// +// Wrapper around Winston logging library with custom query +// +// expected log entry to be in the shape of: +// {"level":"info","message":"{ '0': 'Your Message' }","timestamp":"2016-02-04T05:59:27.412Z"} +// +import { LoggerAdapter } from './LoggerAdapter'; +import winston from 'winston'; +import fs from 'fs'; +import { Parse } from 'parse/node'; + +const MILLISECONDS_IN_A_DAY = 24 * 60 * 60 * 1000; +const CACHE_TIME = 1000 * 60; + +let LOGS_FOLDER = './logs/'; + +if (typeof process !== 'undefined' && process.env.NODE_ENV === 'test') { + LOGS_FOLDER = './test_logs/' +} + +let currentDate = new Date(); + +let simpleCache = { + timestamp: null, + from: null, + until: null, + order: null, + data: [], + level: 'info', +}; + +// returns Date object rounded to nearest day +let _getNearestDay = (date) => { + return new Date(date.getFullYear(), date.getMonth(), date.getDate()); +} + +// returns Date object of previous day +let _getPrevDay = (date) => { + return new Date(date - MILLISECONDS_IN_A_DAY); +} + +// returns the iso formatted file name +let _getFileName = () => { + return _getNearestDay(currentDate).toISOString() +} + +// check for valid cache when both from and util match. +// cache valid for up to 1 minute +let _hasValidCache = (from, until, level) => { + if (String(from) === String(simpleCache.from) && + String(until) === String(simpleCache.until) && + new Date() - simpleCache.timestamp < CACHE_TIME && + level === simpleCache.level) { + return true; + } + return false; +} + +// renews transports to current date +let _renewTransports = ({infoLogger, errorLogger, logsFolder}) => { + if (infoLogger) { + infoLogger.add(winston.transports.File, { + filename: logsFolder + _getFileName() + '.info', + name: 'info-file', + level: 'info' + }); + } + if (errorLogger) { + errorLogger.add(winston.transports.File, { + filename: logsFolder + _getFileName() + '.error', + name: 'error-file', + level: 'error' + }); + } +}; + +// check that log entry has valid time stamp based on query +let _isValidLogEntry = (from, until, entry) => { + var _entry = JSON.parse(entry), + timestamp = new Date(_entry.timestamp); + return timestamp >= from && timestamp <= until + ? true + : false +}; + +// ensure that file name is up to date +let _verifyTransports = ({infoLogger, errorLogger, logsFolder}) => { + if (_getNearestDay(currentDate) !== _getNearestDay(new Date())) { + currentDate = new Date(); + if (infoLogger) { + infoLogger.remove('info-file'); + } + if (errorLogger) { + errorLogger.remove('error-file'); + } + _renewTransports({infoLogger, errorLogger, logsFolder}); + } +} + +export class FileLoggerAdapter extends LoggerAdapter { + constructor(options = {}) { + super(); + + this._logsFolder = options.logsFolder || LOGS_FOLDER; + + // check logs folder exists + if (!fs.existsSync(this._logsFolder)) { + fs.mkdirSync(this._logsFolder); + } + + this._errorLogger = new (winston.Logger)({ + exitOnError: false, + transports: [ + new (winston.transports.File)({ + filename: this._logsFolder + _getFileName() + '.error', + name: 'error-file', + level: 'error' + }) + ] + }); + + this._infoLogger = new (winston.Logger)({ + exitOnError: false, + transports: [ + new (winston.transports.File)({ + filename: this._logsFolder + _getFileName() + '.info', + name: 'info-file', + level: 'info' + }) + ] + }); + } + + info() { + _verifyTransports({infoLogger: this._infoLogger, logsFolder: this._logsFolder}); + return this._infoLogger.info.apply(undefined, arguments); + } + + error() { + _verifyTransports({errorLogger: this._errorLogger, logsFolder: this._logsFolder}); + return this._errorLogger.error.apply(undefined, arguments); + } + + // custom query as winston is currently limited + query(options, callback) { + if (!options) { + options = {}; + } + // defaults to 7 days prior + let from = options.from || new Date(Date.now() - (7 * MILLISECONDS_IN_A_DAY)); + let until = options.until || new Date(); + let size = options.size || 10; + let order = options.order || 'desc'; + let level = options.level || 'info'; + let roundedUntil = _getNearestDay(until); + let roundedFrom = _getNearestDay(from); + + if (_hasValidCache(roundedFrom, roundedUntil, level)) { + let logs = []; + if (order !== simpleCache.order) { + // reverse order of data + simpleCache.data.forEach((entry) => { + logs.unshift(entry); + }); + } else { + logs = simpleCache.data; + } + callback(logs.slice(0, size)); + return; + } + + let curDate = roundedUntil; + let curSize = 0; + let method = order === 'desc' ? 'push' : 'unshift'; + let files = []; + let promises = []; + + // current a batch call, all files with valid dates are read + while (curDate >= from) { + files[method](this._logsFolder + curDate.toISOString() + '.' + level); + curDate = _getPrevDay(curDate); + } + + // read each file and split based on newline char. + // limitation is message cannot contain newline + // TODO: strip out delimiter from logged message + files.forEach(function(file, i) { + let promise = new Parse.Promise(); + fs.readFile(file, 'utf8', function(err, data) { + if (err) { + promise.resolve([]); + } else { + let results = data.split('\n').filter((value) => { + return value.trim() !== ''; + }); + promise.resolve(results); + } + }); + promises[method](promise); + }); + + Parse.Promise.when(promises).then((results) => { + let logs = []; + results.forEach(function(logEntries, i) { + logEntries.forEach(function(entry) { + if (_isValidLogEntry(from, until, entry)) { + logs[method](JSON.parse(entry)); + } + }); + }); + simpleCache = { + timestamp: new Date(), + from: roundedFrom, + until: roundedUntil, + data: logs, + order, + level, + }; + callback(logs.slice(0, size)); + }); + } +} + +export default FileLoggerAdapter; diff --git a/src/Adapters/Logger/LoggerAdapter.js b/src/Adapters/Logger/LoggerAdapter.js new file mode 100644 index 0000000000..b1fe31b8ab --- /dev/null +++ b/src/Adapters/Logger/LoggerAdapter.js @@ -0,0 +1,17 @@ +// Logger Adapter +// +// Allows you to change the logger mechanism +// +// Adapter classes must implement the following functions: +// * info(obj1 [, obj2, .., objN]) +// * error(obj1 [, obj2, .., objN]) +// * query(options, callback) +// Default is FileLoggerAdapter.js + +export class LoggerAdapter { + info() {} + error() {} + query(options, callback) {} +} + +export default LoggerAdapter; diff --git a/src/Controllers/LoggerController.js b/src/Controllers/LoggerController.js new file mode 100644 index 0000000000..d0b8bb28cf --- /dev/null +++ b/src/Controllers/LoggerController.js @@ -0,0 +1,78 @@ +import { Parse } from 'parse/node'; +import PromiseRouter from '../PromiseRouter'; +import rest from '../rest'; + +const Promise = Parse.Promise; +const INFO = 'info'; +const ERROR = 'error'; +const MILLISECONDS_IN_A_DAY = 24 * 60 * 60 * 1000; + +// only allow request with master key +let enforceSecurity = (auth) => { + if (!auth || !auth.isMaster) { + throw new Parse.Error( + Parse.Error.OPERATION_FORBIDDEN, + 'Clients aren\'t allowed to perform the ' + + 'get' + ' operation on logs.' + ); + } +} + +// check that date input is valid +let isValidDateTime = (date) => { + if (!date || isNaN(Number(date))) { + return false; + } +} + +export class LoggerController { + + constructor(loggerAdapter) { + this._loggerAdapter = loggerAdapter; + } + + // Returns a promise for a {response} object. + // query params: + // level (optional) Level of logging you want to query for (info || error) + // from (optional) Start time for the search. Defaults to 1 week ago. + // until (optional) End time for the search. Defaults to current time. + // order (optional) Direction of results returned, either “asc” or “desc”. Defaults to “desc”. + // size (optional) Number of rows returned by search. Defaults to 10 + handleGET(req) { + if (!this._loggerAdapter) { + throw new Parse.Error(Parse.Error.PUSH_MISCONFIGURED, + 'Logger adapter is not availabe'); + } + + let promise = new Parse.Promise(); + let from = (isValidDateTime(req.query.from) && new Date(req.query.from)) || + new Date(Date.now() - 7 * MILLISECONDS_IN_A_DAY); + let until = (isValidDateTime(req.query.until) && new Date(req.query.until)) || new Date(); + let size = Number(req.query.size) || 10; + let order = req.query.order || 'desc'; + let level = req.query.level || INFO; + enforceSecurity(req.auth); + this._loggerAdapter.query({ + from, + until, + size, + order, + level, + }, (result) => { + promise.resolve({ + response: result + }); + }); + return promise; + } + + getExpressRouter() { + let router = new PromiseRouter(); + router.route('GET','/logs', (req) => { + return this.handleGET(req); + }); + return router; + } +} + +export default LoggerController; diff --git a/src/index.js b/src/index.js index c29934000b..4458f8c07f 100644 --- a/src/index.js +++ b/src/index.js @@ -21,6 +21,9 @@ import { PushController } from './Controllers/PushController'; import { ClassesRouter } from './Routers/ClassesRouter'; import { InstallationsRouter } from './Routers/InstallationsRouter'; +import { FileLoggerAdapter } from './Adapters/Logger/FileLoggerAdapter'; +import { LoggerController } from './Controllers/LoggerController'; + // Mutate the Parse object to add the Cloud Code handlers addParseCloud(); @@ -69,6 +72,9 @@ function ParseServer(args) { pushAdapter = new ParsePushAdapter(pushConfig) } + // Make logger adapter + let loggerAdapter = args.loggerAdapter || new FileLoggerAdapter(); + if (args.databaseURI) { DatabaseAdapter.setAppDatabaseURI(args.appId, args.databaseURI); } @@ -136,7 +142,8 @@ function ParseServer(args) { new InstallationsRouter().getExpressRouter(), require('./functions'), require('./schemas'), - new PushController(pushAdapter).getExpressRouter() + new PushController(pushAdapter).getExpressRouter(), + new LoggerController(loggerAdapter).getExpressRouter() ]; if (process.env.PARSE_EXPERIMENTAL_CONFIG_ENABLED || process.env.TESTING) { routers.push(require('./global_config'));