check pm2 status before monitoring

This commit is contained in:
Tjatse 2015-12-25 17:49:52 +08:00
parent 31e65020b5
commit 0374f26d30
8 changed files with 213 additions and 171 deletions

View File

@ -29,8 +29,9 @@ function Monitor(options) {
this._init(options);
};
Monitor.ACCEPT_KEYS = ['pm2', 'refresh', 'statsd', 'node', 'log', 'daemonize', 'max_restarts'];
Monitor.ACCEPT_KEYS = ['pm2', 'refresh', 'statsd', 'node', 'log', 'daemonize', 'max_restarts', 'port'];
Monitor.DEF_CONF_FILE = 'pm2-gui.ini';
Monitor.PM2_DAEMON_PROPS = ['DAEMON_RPC_PORT', 'DAEMON_PUB_PORT', 'PM2_LOG_FILE_PATH'];
/**
* Resolve home path.
@ -68,27 +69,30 @@ Monitor.prototype._init = function (options) {
Log(options.log);
// Load PM2 config.
var pm2ConfPath = path.join(options.pm2, 'conf.js');
var pm2ConfPath = path.join(options.pm2, 'conf.js'),
fbMsg = '';
try {
options.pm2Conf = require(pm2ConfPath)(options.pm2);
if (!options.pm2Conf) {
throw new Error(404);
}
} catch (err) {
var fbMsg = 'Can not load PM2 config, the file "' + pm2ConfPath + '" does not exist or empty, fallback to auto-load by pm2 home.';
fbMsg = 'Can not load PM2 config, the file "' + pm2ConfPath + '" does not exist or empty, fallback to auto-load by pm2 home. ';
console.warn(fbMsg);
options.pm2Conf = {
DAEMON_RPC_PORT: path.resolve(options.pm2, 'rpc.sock'),
DAEMON_PUB_PORT: path.resolve(options.pm2, 'pub.sock'),
PM2_LOG_FILE_PATH: path.resolve(options.pm2, 'pm2.log')
};
for (var pm2ConfProp in options.pm2Conf) {
if (!fs.existsSync(options.pm2Conf[pm2ConfProp])) {
throw new Error(fbMsg + ' But file ' + options.pm2Conf[pm2ConfProp] + ' can not found.');
}
}
}
Monitor.PM2_DAEMON_PROPS.forEach(function (prop) {
var val = options.pm2Conf[prop];
if (!val || !fs.existsSync(val)) {
throw new Error(fbMsg + 'Unfortunately ' + (val || prop) + ' can not found, please makesure that your pm2 is running and the home path is correct.');
}
});
// Bind socket.io server to context.
if (options.sockio) {
this._sockio = options.sockio;
@ -129,12 +133,19 @@ Monitor.prototype.run = function () {
Monitor.prototype.quit = function () {
console.debug('Closing pm2 pub emitter socket.');
this.pm2Sock && this.pm2Sock.close();
console.debug('Closing socket.io server.');
this._sockio.close();
console.debug('Destroying tails.');
this._killTailProcess();
};
/**
* Monitor dashboard.
*/
Monitor.prototype.dashboard = function () {
Log({
level: 1000
});
// Socket.io server.
var port = this.options.port;
this._sockio = require('socket.io')();
@ -153,26 +164,27 @@ Monitor.prototype.dashboard = function () {
* @private
*/
Monitor.prototype._connectSysSock = function (socket) {
var self = this;
// Still has one client connects to server at least.
this._noClient = false;
socket.on('disconnect', function () {
// Check connecting client.
this._noClient = this._sockio.of(conf.NSP.SYS).sockets.length == 0;
}.bind(this));
self._noClient = self._sockio.of(conf.NSP.SYS).sockets.length == 0;
});
// Trigger actions of process.
socket.on('action', function (action, id) {
console.info('[' + id + ']', action, 'sending to pm2 daemon...');
pm.action(this.options.pm2Conf.DAEMON_RPC_PORT, action, id, function (err, forceRefresh) {
pm.action(self.options.pm2Conf.DAEMON_RPC_PORT, action, id, function (err, forceRefresh) {
if (err) {
console.error(action, err.message);
return socket.emit('action', id, err.message);
}
console.info('[' + id + ']', action, 'completed!');
forceRefresh && this._throttleRefresh();
}.bind(this));
}.bind(this));
forceRefresh && self._throttleRefresh();
});
});
// Get PM2 version and return it to client.
this._pm2Ver(socket);
@ -184,6 +196,7 @@ Monitor.prototype._connectSysSock = function (socket) {
// Grep system states once and again.
(this._status != 'R') && this._nextTick(this.options.refresh || 5000);
console.info('SYS socket connected!');
};
/**
@ -203,36 +216,6 @@ Monitor.prototype._connectLogSock = function (socket) {
self._broadcast.call(self, 'log', data, conf.NSP.LOG);
}
// Clean up `tail` after socket disconnected.
function killTail(pm_id) {
self._tails[pm_id].forEach(function (tail) {
try {
tail.kill('SIGTERM');
} catch (err) {}
});
delete self._tails[pm_id];
console.info('[' + pm_id + ']', 'tail destroyed!');
}
function killTailProcess(pm_id) {
if (!isNaN(pm_id)) {
return killTail(pm_id);
}
var socks = self._sockio.of(conf.NSP.LOG).sockets,
canNotBeDeleted = {};
if (socks && socks.length > 0) {
socks.forEach(function (sock) {
canNotBeDeleted[sock._pm_id] = 1;
});
}
for (var pm_id in self._tails) {
if (!canNotBeDeleted[pm_id]) {
killTail(pm_id);
}
}
}
function startTailProcess(pm_id, keepANSI) {
socket._pm_id = pm_id;
@ -272,9 +255,10 @@ Monitor.prototype._connectLogSock = function (socket) {
});
}
socket.on('disconnect', killTailProcess);
socket.on('tail_kill', killTailProcess);
socket.on('disconnect', self._killTailProcess.bind(self));
socket.on('tail_kill', self._killTailProcess.bind(self));
socket.on('tail', startTailProcess);
console.info('LOG socket connected!');
};
/**
@ -346,6 +330,7 @@ Monitor.prototype._connectProcSock = function (socket) {
socket.on('disconnect', killObserver);
socket.on('proc', runObserver);
console.info('PROC socket connected!');
};
/**
@ -360,7 +345,7 @@ Monitor.prototype._nextTick = function (tick, continuously) {
}
// Running
this._status = 'R';
console.debug('monitor heartbeat', tick);
console.debug('monitor heartbeat per', tick + 'ms');
// Grep system state
this._systemStat(function () {
// If there still has any client, grep again after `tick` ms.
@ -399,8 +384,9 @@ Monitor.prototype._systemStat = function (cb) {
* @private
*/
Monitor.prototype._observePM2 = function () {
var pm2Daemon = this.options.pm2Conf.DAEMON_PUB_PORT;
console.info('Connecting to pm2 daemon:', pm2Daemon);
this.pm2Sock = pm.sub(this.options.pm2Conf.DAEMON_PUB_PORT, function (data) {
this.pm2Sock = pm.sub(pm2Daemon, function (data) {
console.info('sub-emitter', chalk.magenta(data.event), data.process.name + '-' + data.process.pm_id);
this._throttleRefresh();
}, this);
@ -463,7 +449,10 @@ Monitor.prototype._refreshProcs = function () {
* @private
*/
Monitor.prototype._pm2Ver = function (socket) {
pm.version(this.options.pm2Conf.DAEMON_RPC_PORT, function (err, version) {
var pm2RPC = this.options.pm2Conf.DAEMON_RPC_PORT;
console.info('Fetch pm2 version:', pm2RPC);
pm.version(pm2RPC, function (err, version) {
console.log(err, version);
socket.emit('pm2_ver', (err || !version) ? '0.0.0' : version);
});
};
@ -488,3 +477,38 @@ Monitor.prototype._broadcast = function (event, data, nsp) {
}
});*/
};
/**
* Destroy tails.
* @param {Number} pm_id
* @return {[type]}
*/
Monitor.prototype._killTailProcess = function (pm_id) {
var self = this;
function killTail(id) {
self._tails[id].forEach(function (tail) {
try {
tail.kill('SIGTERM');
} catch (err) {}
});
delete self._tails[id];
console.info('[' + id + ']', 'tail destroyed!');
}
if (!isNaN(pm_id)) {
return killTail(pm_id);
}
var socks = self._sockio.of(conf.NSP.LOG).sockets,
canNotBeDeleted = {};
if (socks && socks.length > 0) {
socks.forEach(function (sock) {
canNotBeDeleted[sock._pm_id] = 1;
});
}
for (var pm_id in self._tails) {
if (!canNotBeDeleted[pm_id]) {
killTail(pm_id);
}
}
}

View File

@ -57,7 +57,7 @@ stat.cpuUsage = function (fn, context) {
setTimeout(function (ctx, stat1) {
var stat2 = ctx.cpuInfo(),
perc = 100 * (1 - (stat2.idle - stat1.idle) / (stat2.total - stat1.total));
fn.call(context, perc.toFixed(2));
fn.call(context, null, perc.toFixed(2));
}, 1000, this, this.cpuInfo());
};

View File

@ -10,9 +10,10 @@ module.exports = function (options) {
consoled = {},
lev = options.level;
if ((typeof lev == 'string' && !(lev = hacks[lev])) || (isFinite(lev) && (lev < 0 || lev > hacks.length))) {
if ((typeof lev == 'string' && typeof (lev = hacks.indexOf(lev)) == 'undefined') || (isFinite(lev) && (lev < 0 || lev > hacks.length))) {
options.level = 0;
}
options.level = !isNaN(lev) || 0;
hacks.forEach(function (method) {
if (method == 'debug') {

View File

@ -28,7 +28,7 @@ global.action = function(method, path, func){
var _cwd = path.resolve(__dirname, '../../', 'web/routes');
// initialize.
module.exports = function(server, log){
module.exports = function(server){
fs.readdirSync(_cwd).forEach(function(f){
if (path.extname(f) != '.js') {
return;
@ -39,10 +39,6 @@ module.exports = function(server, log){
});
routes.forEach(function(route){
route.path = route.path.replace(/\/+/g, '/');
log.i('hook', chalk.bold.green(route.method.toUpperCase()), chalk.underline.grey(route.path));
server[route.method](route.path, function(req, res, next){
req.log = log;
next();
}, route.fn);
server[route.method](route.path, route.fn);
});
};

View File

@ -8,4 +8,4 @@ daemonize = false
dir = ./logs
prefix = true
date = false
level = debug
level = log

View File

@ -1,71 +1,66 @@
var Monitor = require('./lib/monitor'),
chalk = require('chalk'),
var chalk = require('chalk'),
path = require('path'),
fs = require('fs'),
_ = require('lodash'),
Log = require('./lib/util/log');
socketIO = require('socket.io'),
Monitor = require('./lib/monitor'),
Log = require('./lib/util/log'),
Web = require('./web/index');
exports.start = function (options) {
process.title = 'pm2-gui slave';
options = options || {};
var confFile = options.confFile;
if (!confFile) {
confFile = './pm2-gui.ini';
if (process.argv.length > 2) {
confFile = process.argv[2];
}
confFile = path.resolve(__dirname, confFile);
if (!fs.existsSync(confFile)) {
console.error(chalk.bold(confFile), 'does not exist!');
return process.exit(0);
}
if (path.basename(process.mainModule.filename, '.js') == 'pm2-gui') {
var cmd, file;
if (process.argv.length > 2) {
cmd = process.argv[2];
}
var monitor = Monitor({
confFile: confFile
if (process.argv.length > 3) {
file = process.argv[3];
}
cmd = cmd || 'start';
switch (cmd) {
case 'start':
startWebServer(file);
break;
case 'mon':
dashboard(file);
break;
default:
break;
}
}
exports.startWebServer = startWebServer;
exports.dashboard = dashboard;
exports.exitGraceful = exitGraceful;
function startWebServer(confFile) {
var monitor = slave({
confFile: confFile
}),
options = monitor.options;
options.port = options.port || 8088;
var server = Web({
middleware: function (req, res, next) {
req._config = options;
next();
},
port: options.port
});
Log(monitor.options.log);
console.log(chalk.cyan(
'\n' +
'█▀▀█ █▀▄▀█ █▀█ ░░ █▀▀▀ █░░█ ░▀░\n' +
'█░░█ █░▀░█ ░▄▀ ▀▀ █░▀█ █░░█ ▀█▀\n' +
'█▀▀▀ ▀░░░▀ █▄▄ ░░ ▀▀▀▀ ░▀▀▀ ▀▀▀\n'));
monitor._sockio = socketIO(server);
monitor.run();
process.on('SIGTERM', shutdown);
process.on('SIGINT', shutdown);
process.on('SIGHUP', restart);
process.on('uncaughtException', caughtException);
process.on('exit', exports.exitGraceful)
function shutdown(code, signal) {
console.info('Shutting down....');
monitor.quit();
console.info('Both', chalk.bold('pm2-emitter'), 'and', chalk.bold('statsd dgram'), 'sockets are closed.');
console.info('Shutdown complete!');
exports.exitGraceful(code, '-f');
}
function restart() {
if (process.send) {
process.send({
action: 'restart'
});
} else {
console.error('No IPC found, could not restart monitor, shutting down.');
shutdown(1);
}
}
function caughtException(err) {
console.error(err.stack);
shutdown(1);
}
console.info('Web server is listening on 0.0.0.0:' + options.port);
};
exports.exitGraceful = function exit(code, signal) {
function dashboard(confFile) {
var monitor = slave({
confFile: confFile
});
monitor.dashboard();
};
function exitGraceful(code, signal) {
code = code || 0;
if (signal != '-f') {
console.debug('Slave has exited, code: ' + code + ', signal: ' + (signal || 'NULL'));
@ -91,6 +86,59 @@ exports.exitGraceful = function exit(code, signal) {
});
tryToExit();
};
if (path.basename(process.mainModule.filename, '.js') == 'pm2-ant') {
exports.start();
}
function slave(options) {
process.title = 'pm2-gui slave';
options = options || {};
var confFile = options.confFile;
if (!confFile) {
confFile = path.resolve(__dirname, './pm2-gui.ini');
if (!fs.existsSync(confFile)) {
console.error(chalk.bold(confFile), 'does not exist!');
return process.exit(0);
}
}
var monitor = Monitor({
confFile: confFile
});
Log(monitor.options.log);
console.log(chalk.cyan(
'\n' +
'█▀▀█ █▀▄▀█ █▀█ ░░ ▒█▀▀█ ▒█░▒█ ▀█▀\n' +
'█░░█ █░▀░█ ░▄▀ ▀▀ ▒█░▄▄ ▒█░▒█ ▒█░\n' +
'█▀▀▀ ▀░░░▀ █▄▄ ░░ ▒█▄▄█ ░▀▄▄▀ ▄█▄\n'));
process.on('SIGTERM', shutdown);
process.on('SIGINT', shutdown);
process.on('SIGHUP', restart);
process.on('uncaughtException', caughtException);
process.on('exit', exitGraceful)
function shutdown(code, signal) {
console.info('Shutting down....');
monitor.quit();
console.info('Shutdown complete!');
exitGraceful(code, '-f');
}
function restart() {
if (process.send) {
process.send({
action: 'restart'
});
} else {
console.error('No IPC found, could not restart monitor, shutting down.');
shutdown(1);
}
}
function caughtException(err) {
console.error(err.stack);
shutdown(1);
}
return monitor;
};

View File

@ -1,45 +1,30 @@
var express = require('express'),
swig = require('swig'),
path = require('path'),
chalk = require('chalk'),
Monitor = require('../lib/monitor'),
Debug = require('../lib/util/debug'),
session = require('express-session');
session = require('express-session'),
swig = require('swig'),
path = require('path'),
http = require('http'),
Monitor = require('../lib/monitor'),
router = require('../lib/util/router');
module.exports = runServer;
function runServer(debug){
function runServer(options) {
var app = express();
// all environments
app.set('view engine', 'html');
app.set('views', path.join(__dirname, 'views'));
app.engine('html', swig.renderFile);
app.use(express.static(path.join(__dirname, 'public')));
app.use(session({
secret : 'pm2@gui',
resave : false,
secret: 'pm2@gui',
resave: false,
saveUninitialized: true
}));
var log = Debug(({namespace: 'pm2-gui', debug: !!debug}));
// router
require('../lib/util/router')(app, log);
var server = require('http').Server(app);
var io = require('socket.io')(server);
try {
var mon = Monitor({
sockio: io,
debug : !!debug
});
var port = mon.config('port');
server.listen(port);
log.i('http', 'Web server of', chalk.bold.underline('Unitech/PM2'), 'is listening on port', chalk.bold(port));
mon.run();
}catch(err){
log.e(chalk.red(err.message));
if (options.middleware) {
app.use(options.middleware);
}
}
router(app);
var server = http.Server(app);
server.listen(options.port);
return server;
}

View File

@ -1,22 +1,14 @@
var Monitor = require('../../lib/mon'),
crypto = require('crypto');
// Authorize
// Authorization
action(function auth(req, res){
var authCode = req.session['auth_code'],
storedCode = Monitor().config('password');
if (!storedCode || (storedCode == authCode)) {
if (!req._config.password || (req._config.password === req.session['password'])) {
return res.redirect('/');
}
res.render('auth', {title: 'Authorize'});
res.render('auth', {title: 'Authorization'});
});
// Index
action(function(req, res){
var authCode = req.session['auth_code'],
storedCode = Monitor().config('password');
if (storedCode && storedCode != authCode) {
if (req._config.password && (req._config.password !== req.session['password'])) {
return res.redirect('/auth');
}
res.render('index', {title: 'Monitor'});
@ -25,16 +17,12 @@ action(function(req, res){
// API
action(function auth_api(req, res){
if (!req.query || !req.query.pwd) {
return res.json({error: 'Authorize failed, password is required!'});
return res.json({error: 'Authorization failed, password is required!'});
}
var mon = Monitor(),
md5 = crypto.createHash('md5');
md5.update(req.query.pwd);
encryptedPwd = md5.digest('hex');
if (encryptedPwd == mon.config('password')) {
req.session['auth_code'] = encryptedPwd;
if (req.query.pwd === req._config.password) {
req.session['password'] = req.query.pwd;
return res.json({status: 200});
}
return res.json({error: 'Authorize failed, password is incorrect.'});
return res.json({error: 'Authorization failed, password is incorrect.'});
});