diff --git a/.gitignore b/.gitignore index 00bd4da..080625b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ reports target docs logs +log cluster-cache-domain cluster-cache-persist ports diff --git a/.npmignore b/.npmignore index bbb769f..ac7d04a 100644 --- a/.npmignore +++ b/.npmignore @@ -1,7 +1,11 @@ *.iml .travis.yml +.idea node_modules test +examples +log +pids cluster-cache-domain cluster-cache-persist ports \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index e216e48..bafc11b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ language: node_js node_js: - - 0.8 + - 0.10 + +branches: + only: + - cluster3 before_script: - "npm install" diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 0b83ae3..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,14 +0,0 @@ -Copyright 2012 eBay Software Foundation - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - diff --git a/Makefile b/Makefile deleted file mode 100644 index 805684e..0000000 --- a/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -all: clean install test - -clean: - -rm -fr node_modules - -install: - npm install;\ - npm link; - -.PHONY : test -test: - export NODE_PATH=./node_modules;\ - node_modules/nodeunit/bin/nodeunit test/cluster-test.js - -test-debug: - export NODE_PATH=./node_modules;\ - node --debug-brk node_modules/nodeunit/bin/nodeunit test - -test-part: - export NODE_PATH=./node_modules;\ - node_modules/nodeunit/bin/nodeunit test--reporter dot --output ../../reports - -unpublish: - npm --registry $(REGISTRY) unpublish - -publish: - npm --registry $(REGISTRY) publish - -refresh: - npm --registry $(REGISTRY) publish --force diff --git a/README.md b/README.md index 484095a..71c22b2 100644 --- a/README.md +++ b/README.md @@ -1,258 +1,640 @@ -## What is cluster2 +cluster2 +=============== + +This is a completely overhaul, not expected to be backward compatible, but the features should cover the most popular while some changes are on their way: + +## simplification + +You'll see that we've simplified the api a great deal, no more cluster class, instance to worry, a single #listen method to take all dancing parts. +And those configurable pieces mostly have reasonable defaults, and could be easily set from command line arguments. For example, `--port=8080`, `--cache.enable` etc. +Also we've adopted Promise A+ (when.js). style to replace the callbacks, we like it for the fewer level of nested code, a lot. +You'll also find some redundant features like: multiple app/port support, ecv on workers, none cluster mode, all removed to keep code compact. + +* **`cluster`** + +```javascript +var listen = require('cluster2').listen; + +listen({ + + 'noWorkers': 1, //default number of cpu cores + 'createServer': require('http').createServer, + 'app': app, //your express app + 'port': 9090, //express app listening port + 'configureApp': function(app){ + //register your routes, middlewares to the app, must return value or promise + return app; + }, + 'warmUp': function(app, address){ + //warm up your application, must return value or promise + return app; + }, + 'warmUpPort': 9093, //the port to do warmup, after which, server will be stopped, and restarted on the actual port + 'debug': { //node-inspector integration + 'webPort': 9092, //node-inspector web listening port + 'saveLiveEdit': true + }, + 'ecv': { + 'mode': 'control', + 'root': '/ecv' + }, + 'cache': { + 'enable': true, //check cache section + 'mode': 'standalone' //default creates a standalone worker specific to run as cache manager, otherwise use master to run + }, + 'gc': { + 'monitor': true, //will reflect the gc (incremental, full) in heartbeat, this conflicts with socket.io somehow + 'idle-noitification': false, //for performance reason, we'll disable node idle notification to v8 by default + 'explicit': false //yet impl, meant to expose gc() as a global function + }, + 'monCreateServer': require('http').createServer, //tell master what mon app server should be created as + 'monConfigureApp': function(monApp){//could overwrite with your own monitor app, and configure it + return monApp; + }, + 'monApp': monApp, + 'monPort': 9091, //monitoring app listening port + 'maxAge': 3600, //worker's max life in seconds, default is 3 days + 'heartbeatInterval': 5000 //heartbeat interval in MS +}) +.then(function(resolved){ + //cluster started + //resolved is an object which embeds server, app, port, etc. + //this is quite useful, you must understand that both master and workers will get to here (due to fork) + //if it's worker, it means the warmup of that worker process has been finished (and listening already happen of course) + //if it's master, it means all of the workers (noWorkers) asked to be created initially have all been warmed up + //and implicitly, the ecv if enabled, will return 200 to whoever polls for the status +}) +.otherwise(function(error){ + //cluster start error +}); +//the major change is the return of promise and the much simplified #listen (as all options pushed to construction) + +``` + +## application flow + +For this cluster2 to work perfect, you might need to accept some of the assumption we made of your application flow. It exists to make your life easier, so as for our middleware registration to work as expected. +The flow is as the following: + +* master starts `listen` +* master configures `monApp` with given `monConfigureApp` +* master starts `caching` service if enabled +* master creats server using `monCreateServer` and takes in configured `monApp` +* master starts server on the `monPort` and wait for `listening` event +* master starts forking workers +* worker starts `listen` +* worker configures `app` with given `configureApp` +* worker creates server using `createServer` and takes in configured `app` +* worker starts server on `warmUpPort` and wait for `listening` event +* worker receives `listening` event and starts `warmup` +* worker waits for `warmup` to complete and stops the warmup server +* worker starts server on actual `port` and wait for `listening` event +* worker receives `listening` event and notify master that it's ready to serve traffic +* worker resolves the `promise` returned by `listen` +* master receives notifications from all workers then mark up `ecv` +* master then resolves the `promise` returned by `listen` + +A few key points: +* The abstract pattern is the same for master & worker (different in what's done in each step): **listen** -> **configure app** -> **create server** -> **warmup** -> **start listening** -> **resolve promise** +* Caching service starts early, so that you could start using cache whether in master or worker, after **configure app** +* WarmUp is added as an explicit step to allow application to be performant when the traffic is on. +* Configure app, and warmup could return value (app) or promise which resolves to the app. + +A clear flow as above allows users to inject their middleware, routes, warm up their application in a deterministic manner. +And we could leverage this, so that we could safely register middleware like tps collection in front of users'. This makes testing much easier too, +as the promise won't be resolved till the server actually starts, no more timed waiting, event emitting etc. You're good to request anything by then. + +## emitter + +Cluster used to be an emitter itself, which isn't very much helpful, and forced event register/emit to be delayed till the cluster instance is created. +Even if it's created, accessing the instance from different modules require the instance to be passed down, or global, neither looks appealing. +The new cluster-emitter is designed to work with cluster not cluster2 instance at all (in fact, we eliminated the cluster2 instance as you see the api above) +The emitter also makes communications between worker & master (or reverse) as simple as a normal EventEmitter. + +```javascript +var emitter = require('cluster2/emitter'); + +emitter.on('event', function callback(){ + //an event callback +}); + +emitter.once('event', function callbackOnce(){ + //another event callback +}); + +emitter.removeListener('event', callback); +emitter.removeListener('event', callbackOnce); +emitter.removeAllListeners('event'); + +emitter.emit('event', 'arg0', 'arg1'); +//it varies in master and worker runtime, in master it's the same as saying +emitter.emitTo(['self'].concat(_.map(cluster.workers, function(w){return w.process.pid;})), ['event', 'arg0', 'arg1']); +//as this indicates, the master's emit target by default is everybody, master itself and all active workers +//and in worker runtime, it's intepreted as worker itself and master +emitter.emitTo(['self', 'master'], ['event', 'arg0', 'arg1']); +//you don't have to use the different `emitTo` method unless you have a different targets set from the default explained above. +//but in cause you need, it's also simplified as: +emitter.to(['master']).emit('event', 'arg0', 'arg1'); +//use to method to scope the target differently, the value should be an array of pids, or 'master', or 'self' + +``` + +## ecv + +ECV is a preserved feature, but we've simplified that too. Most of the use cases we've seen doesn't really need an ECV for each worker process, in fact +that could be very confusing. To let tools view the cluster as an entirety, ECV is to run only in master runtime, it still supports the 'monitor' vs. 'control' mode. + +```javascript + +//ecv control could be used as such +var enable = require('cluster2/ecv').enable; + +enable(app); + +//more use cases just let cluster2 enables it by passing configurations to the #listen +var listen = require('cluster2').listen; + +listen({ + + 'noWorkers': 1, //default number of cpu cores + 'createServer': require('http').createServer, + 'app': app, + 'port': 9090, + 'monPort': 9091, + 'debug': { //node-inspector integration + 'webPort': 9092, + 'saveLiveEdit': true + }, + 'ecv': { + 'mode': 'control',//could be 'monitor' or 'control' + 'root': '/ecv', + 'markUp': '/ecv/markUp', + 'markDown': '/ecv/markDown' + }, + 'heartbeatInterval': 5000 //heartbeat rate +}); + +//alternatively + +listen({ + + 'noWorkers': 1, //default number of cpu cores + 'createServer': require('http').createServer, + 'app': app, + 'port': 9090, + 'monPort': 9091, + 'debug': { //node-inspector integration + 'webPort': 9092, + 'saveLiveEdit': true + }, + 'ecv': { + 'mode': 'monitor',//could be 'monitor' or 'control' + 'root': '/ecv', + 'monitor': '/myapplication/route1', + 'validator': function(err, response, body){ + //to validate what we got from the monitor url + return true;//or false + } + }, + 'heartbeatInterval': 5000 //heartbeat rate +}); +``` + +## debug + +Ever imagined debugging to be simpler? Here's the good news, we've carefully designed the debugging process from the ground up of the new cluster. +With integration with ECV, worker lifecycle management, node-inspector, and bootstrap + websocket debug app (middleware to be exact). You're now +able to debug any running worker a few clicks away, same applies for a newly forked one. + +`http://localhost:9091/debug` (change host, port to your configured values) `debug` route is what we added as a middleware to the monitor app given. It presents an insight of the running workers, their health; in addition, the cluster cache status. You could hover on a worker pid to request a node-inspector based debug, the control flow is described at `__dirname/lib/public/images/live-debugging.png`. + +The experience is designed to be the same across different environments, whether dev, qa, or even production, the same debugging flow and mechanism would make diagnostics much more effective. + +## deps + +This is a preserved feature of cluster2, it simply list the npm ls result and give it under `http://localhost:9091/deps` route, which looks like the following. + +```javascript +{ + "name": "cluster2", + "version": "0.5.0", + "dependencies": { + "underscore": { + "version": "1.4.4", + "from": "underscore@~1.4.4" + }, + "usage": { + "version": "0.3.8", + "from": "usage@~0.3.8", + "dependencies": { + "bindings": { + "version": "1.1.1", + "from": "bindings@1.x.x" + } + } + }, + "when": { + "version": "2.3.0", + "from": "when@~2.3.0" + }, + "graceful-fs": { + "version": "2.0.1", + "from": "graceful-fs@~2.0.0" + }, + "gc-stats": { + "version": "0.0.1", + "from": "gc-stats@~0.0.1", + "resolved": "https://registry.npmjs.org/gc-stats/-/gc-stats-0.0.1.tgz" + }, + "bignumber.js": { + "version": "1.1.1", + "from": "bignumber.js@~1.1.1" + } + //... more dependencies not shown + } +} +``` + +## robustness + +This is sth we learned given the real experience of a node.js application, workers do get slower, whether that's memory leak, or GC becomes worse, it's easier to prepare +than to avoid. So as a step forward from the previous 'death watch', we're now proactively collecting performance statistics and to decide it a worker could be ended +before it gets slow. You could see the simple heurstic we put at `__dirname/lib/utils.js` # `assertOld` function. You can always overwrite this based on your application's characteristics, but this gives a good starting point based on heartbeat collected stats. + +```javascript + +exports.assertOld = function assertOld(maxAge){ + + maxAge = maxAge || 3600 * 24 * 3;//3 days + + return function(heartbeat){ + + return heartbeat.uptime >= maxAge; + }; +}; + +exports.assertBadGC = function assertBadGC(){ + + var peaks = {}; + + return function(heartbeat){ + + var pid = heartbeat.pid, + uptime = heartbeat.uptime, + currTPS = heartbeat.tps || (heartbeat.transactions * 1000 / heartbeat.cycle); + + if(currTPS <= 2){//intelligent heuristic, TPS too low, no good for sampling as the 1st phase. + return false; + } + + var peak = peaks[pid] = peaks[pid] || { + 'tps': currTPS, + 'cpu': heartbeat.cpu, + 'memory': heartbeat.memory, + 'gc': { + 'incremental': heartbeat.gc.incremental, + 'full': heartbeat.gc.full + } + };//remember the peak of each puppet + + if(currTPS >= peak.tps){ + peak.tps = Math.max(heartbeat.tps, peak.tps); + peak.cpu = Math.max(heartbeat.cpu, peak.cpu); + peak.memory = Math.max(heartbeat.memory, peak.memory); + peak.gc.incremental = Math.max(heartbeat.gc.incremental, peak.gc.incremental); + peak.gc.full = Math.max(heartbeat.gc.full, peak.gc.full); + } + else if(currTPS < peak.tps * 0.9 //10% tps drop + && heartbeat.cpu > peak.cpu + && heartbeat.memory > peak.memory + && heartbeat.gc.incremental > peak.gc.incremental + && heartbeat.gc.full >= peak.gc.full){//sorry, current gc.full is usually zero + + return true; + } + + return false; + } +}; + +{ + 'shouldKill': options.shouldKill || (function(){ //default assertions for killing a worker + + var assertions = [assertOld(_this.maxAge), assertBadGC()]; + + return function(heartbeat){ + + return _.some(assertions, function(a){ + + return a(heartbeat); + }); + }; + + })() +} -![Travis status](https://secure.travis-ci.org/ql-io/cluster2.png) +``` -NOTE: For node (<=0.6.x), use cluster2 version 0.3.1 +Apart from the above mentioned proactive collection, we noticed another subtle issue in practice. When a worker is dead, its load will be distributed to the rest of alives certainly, that adds some stress to the alives, but when more than one worker died at the same time, the stress could become problem. +Therefore, to prevent such from happening when worker is marked to be replaced, we made it a FIFO, further explained in `__dirname/lib/utils` # `deathQueue` function. Its purpose is to guarantee that no more than one worker could commit suicide and be replaced at the same time. -cluster2 is a node.js (>= 0.8.x) compatible multi-process management module. This module grew out of -our needs in operationalizing node.js for [ql.io](https://github.com/ql-io/ql.io) at eBay. Built on -node's `cluster`, cluster2 adds several safeguards and utility functions to help support real-world -production scenarios: +```javascript -* Scriptable start, shutdown and stop flows -* Worker monitoring for process deaths -* Worker recycling -* Graceful shutdown -* Idle timeouts -* Validation hooks (for other tools to monitor cluster2 apps) -* Events for logging cluster activities -* Exit with error code when the port is busy to fail start scripts -* Disable monitor -* and more coming soon +exports.deathQueue = (function(){ -## Usage + var tillPrevDeath = null, + queued = []; -### Getting cluster2 + return function deathQueue(pid, emitter, success, options){ - npm install cluster2 - + options = options || {}; -### Start a TCP Server + assert.ok(pid); + assert.ok(emitter); + assert.ok(success); - var Cluster = require('cluster2'), - net = require('net'); - var server = net.createServer(function (c) { - c.on('end', function () { - console.log('server disconnected'); - }); - c.write('hello\r\n'); - c.pipe(c); - }); + var wait = options.timeout || 60000, + death = util.format('worker-%d-died', pid), + logger = options.logger || { + 'debug' : function(){ + console.log.apply(console, arguments); + } + }; - var c = new Cluster({ - port: 3000, - cluster: true - }); - c.listen(function(cb) { - cb(server); - }); + if(!_.contains(queued, pid)){ -### Start a HTTP Server + queued.push(pid); - var Cluster = require('cluster2'), - http = require('http'); - var server = http.createServer(function (req, res) { - res.writeHead(200); - res.end('hello'); - }); - var c = new Cluster({ - port: 3000 - }); - c.listen(function(cb) { - cb(server); - }); + var tillDeath = when.defer(), + afterDeath = null, + die = function(){ -### Start an Express Server + var successor = success(); - var Cluster = require('cluster2'), - express = require('express'); - var app = express.createServer(); - app.get('/', function(req, res) { - res.send('hello'); - }); + //when successor is in place, the old worker could be discontinued finally + emitter.once(util.format('worker-%d-warmup', successor.process.pid), function(){ - var c = new Cluster({ - port: 3000, - }); - c.listen(function(cb) { - cb(app); - }); + logger.debug('[deathQueue] successor:%d of %d warmup', successor.process.pid, pid); -### Stop a Server - - var Cluster = require('cluster2'); - var c = new Cluster(); - c.stop(); - -### Gracefully Shutdown a Server - - var Cluster = require('cluster2'); - var c = new Cluster(); - c.shutdown(); - - -## Options - -Cluster2 takes the following options. - -* `cluster`: When `true` starts a number of workers. Use `false` to start the server as a single - process. Defaults to `true`. -* `pids`: A directory to write PID files for master and workers. -* `port`: Port number for the app, defaults to `3000`. -* `host`: Hostname or IP for the app listening, defaults to `0.0.0.0`. -* `monHost`: Hostname or IP for the monitor listening, defaults to `0.0.0.0`. -* `monPort`: Port number for the monitor URL, defaults to `3001`. Go to `http://:3001` to - view application logs (whatever is written to a `/logs` dir), and npm dependencies. -* `ecv`: ECV stands for "extended content verification". This is an object with the following - additional properties: - * `path`: A path to serve a heart beat. See below. - * `monitor`: A URI to check before emitting a valid heart beat signal - * `control`: When true, allows clients to enable or disable the signal. See below. - validator to validate the runtime health of the app. If found unhealthy, emits a disable -* `noWorkers`: Defaults to `os.cpus().length`. -* `timeout`: Idle socket timeout. Automatically ends incoming sockets if found idle for this - duration. Defaults to `30` seconds. -* `connThreshold`: When the number of connections processed exceeds this numbers, recycle the worker - process. This can help recover from slow leaks in your code or dependent modules. - -## Graceful Shutdown - -The purpose of `shutdown()` is to let the server reject taking new connections, handle all pending -requests and end the connecton so that no request dropped. In order to handling `shutdown()`, the -server must handle `close` events as follows. - - var serving = true; - var server = http.createServer(function (req, res) { - if(!serving) { - // Be nice and send a connection: close as otherwise the client may pump more requests - // on the same connection - res.writeHead(200, { - 'connection': 'close' - }); - } - res.writeHead(200); - res.end('hello'); - }); - server.on('close', function() { - serving = false; - }) - var c = new Cluster({ - port: 3000, - cluster: true - }); + emitter.to(['master', pid]).emit('disconnect', pid); -Completion of `shutdown()` does not necessarily mean that all worker processes are dead immediately. -The workers may take a while to complete processing of current requests and exit. The `shutdown()` -flow only guarantees that the server takes no new connections. + emitter.once(death, function(){ -## Cluster2 Events + logger.debug('[deathQueue] %d died', pid); -Cluster2 is an `EventEmitter` and emits the following events. + tillDeath.resolve(pid); -* `died`: Emitted when a worker dies. This event is also emitted during normal `shutdown()` or - `stop()`. -* `forked`: Emitted when a new worker is forked. -* ``: Emitted when a worker receives a signal (such as `SIGKILL`, `SIGTERM` or `SIGINT`). + if(tillPrevDeath === afterDeath){//last of dyingQueue resolved, clean up the dyingQueue -Here is an example that logs these events to the disk. + logger.debug('[deathQueue] death queue cleaned up'); - var Cluster = require('cluster2'), - http = require('http'); + tillPrevDeath = null; + } + }); - var server = http.createServer(function (req, res) { - res.writeHead(200); - res.end('hello'); - }); - var c = new Cluster({ - cluster: true, - port: 3000, - host: 'localhost' - }); - c.on('died', function(pid) { - console.log('Worker ' + pid + ' died'); - }); - c.on('forked', function(pid) { - console.log('Worker ' + pid + ' forked'); - }); - c.on('SIGKILL', function() { - console.log('Got SIGKILL'); - }); - c.on('SIGTERM', function(event) { - console.log('Got SIGTERM - shutting down'); - }); - c.on('SIGINT', function() { - console.log('Got SIGINT'); - }); - c.listen(function(cb) { - cb(server); - }); + setTimeout(function(){ -## Routing Traffic + if(!exports.safeKill(pid, 'SIGTERM', logger)){//worker still there, should emit 'exit' eventually -It is fairly common for proxies or load balancers deployed in front of node clusters, and those -proxies to use monitor URLs to detect the health of the cluster. Cluster2 includes a monitor -at `http://:/ecv`. You can change this by setting the `path` property when initializing -the cluster. + logger.debug('[deathQueue] worker:%d did not report death by:%d, kill by SIGTERM', pid, wait); + } + else{//suicide or accident already happended, process has run away + //we emit this from master on behalf of the run away process. -In case you want to take the node cluster out of rotation from the proxy/load balancer, you can do -so by setting `control` to `true` when initializing the cluster. At runtime, you can send a `POST` -request to `http://:/ecv/disable`. Once this is done, further requests to -`http://:/ecv` will get a network error. You can bring the cluster back to rotation by -sending a `POST` request to `http://:/ecv/enable`. + logger.debug('[deathQueue] worker:%d probably ran away, emit:%s on behalf', death); -Since it will be potentially disastrous to let artibrary clients enable/disable traffic, you should -configure your proxy/load balancer to prevent external traffic to `/ecv*`. + emitter.to(['master']).emit(death); + } -To test this, bring up an example + }, wait); + }); + }; - node examples/express/express-server.js + if(!tillPrevDeath){//1st in the dying queue, + afterDeath = tillPrevDeath = tillDeath.promise;//1 min + die(); + } + else{ + afterDeath = tillPrevDeath = tillPrevDeath.ensure(die); + } + } + }; + +})(); -and send a `GET` request to `http://localhost:3000/ecv` and notice the response. +``` - HTTP/1.1 200 OK - X-Powered-By: Cluster2 - content-type: text/plain - since: Fri May 18 2012 09:49:32 GMT-0700 (PDT) - cache-control: no-cache - Connection: keep-alive - Transfer-Encoding: chunked +Oh, one more thing, much as we hope that all workers will behave well, let us know when it's going to give up, in reality, they might not. +For an additional level of protection, we added a simple `nanny` monitor to our master, which simply collects each workers' last `heartbeat` event and check if any possible **runaway** happened. +Once detected, it will be treated the same as a suicide event, using the above `deathQueue`. This will ensure you won't have a cluster running fewer and fewer workers. - status=AVAILABLE&ServeTraffic=true&ip=127.0.0.1&hostname=somehost&port=3000&time=Fri May 18 2012 09:49:49 GMT-0700 (PDT) +```javascript -To flip the monitor into a disabled state, send a `POST` request to `http://localhost:3000/disable`. +exports.nanny = function nanny(puppets, emitter, success, options){ - HTTP/1.1 204 No Content - X-Powered-By: Cluster2 - since: Fri May 18 2012 09:54:25 GMT-0700 (PDT) - cache-control: no-cache - Connection: close + assert.ok(puppets); + assert.ok(emitter); + assert.ok(success); -Subsequent `GET` requests to `http://localhost:3000/ecv` will return a response similar to the one -below. + options = options || {}; - HTTP/1.1 400 Bad Request - X-Powered-By: Cluster2 - content-type: text/plain - since: Fri May 18 2012 09:54:25 GMT-0700 (PDT) - cache-control: no-cache - Connection: close - Transfer-Encoding: chunked + var tolerance = options.tolerance, + now = Date.now(); - status=DISABLED&ServeTraffic=false&ip=127.0.0.1&hostname=somehost&port=3000&time=Fri May 18 2012 09:55:17 GMT-0700 (PDT) + _.each(puppets, function(p){ -To flip the monitor back into an enabled state, send a `POST` request to `http://localhost:3000/enable`. + if(now - p.lastHeartbeat > tolerance){ + exports.deathQueue(p.pid, emitter, success, options); + + } + }); +}; +``` + +## caching + +This is as exciting as debugging, it allows workers to share computation results, watch over changes, in a fast and reliable manner. +We tried work delegation to master once, and found it error-prone and difficult to code against, caching makes things so much simpler, using domain socket, so much faster. +The atomic getOrLoad syntax makes sharing efficient, running cache manager as another worker and persistence support make it disaster recoverable. +It's like having a memcached process, only this is node, and you can debug it too. + +* **`cache`** -NOTE for 0.4.0 version -The major change is to support a general work delegation pattern between workers & master. In a few scenarios, we've seen duplicate work -done by each worker, that could be delegated to master to address and avoid the duplication of effort. And to make it general enough, we -defined the following delegation pattern: -worker -> master : message -message.type is "delegate" -message.delegate defines the actual message type -message.expect is optional, if not given, the delegate work is silently handled by master (e.g. logging remotely); if given, worker will expect a response message whose -type must equal message.expect; if given expect, the following will be enabled: message.matches defines the matching criteria of the response message, message.timeout defines -the max timeout of the delegate work. message.notification allows delegated work to publish changes detected later. -message.origin keeps the orginal message. -In cluster2, after master receives the message from worker, it would turn it into an event message, and find the proper listener to handle such. -The event handler could be config reader, remote logger, resource externalizer e.g. and they might/might not respond to master based on the expect. \ No newline at end of file +```javascript +var cache = require('cluster2/cache').use('cache-name', { + 'persist': true,//default false + 'expire': 60000 //in ms, default 0, meaning no expiration +}); + +``` +* **`keys`** + +```javascript +var cache;//assume the cache is in use as above + +cache.keys({ + 'wait': 100//this is a timeout option +}) +.then(function(keys){ +//the keys resolved is an array of all cached keys:string[] from the cache-manager's view +}); + +//to use the cache, we assume u've started the cluster2 with caching enabled, and you can select how cache manager should be run +listen({ + + 'noWorkers': 1, //default number of cpu cores + 'createServer': require('http').createServer, + 'app': app, + 'port': 9090, + 'monPort': 9091, + 'debug': { //node-inspector integration + 'webPort': 9092, + 'saveLiveEdit': true + }, + 'ecv': { + 'mode': 'control', + 'root': '/ecv' + }, + 'cache': { + 'enable': true,//true by default + 'mode': 'standalone'//as a standalone worker process by default, otherwise will crush with the master process + }, + 'heartbeatInterval': 5000 //heartbeat rate +}) +``` + +Note that, we allow you to use caching w/o cluster2, if you want to enable caching from none cluster2 runtime, the feature could be enabled via: + +```javascript + +//you can use this in unit test too as we did +require('cluster2/cache').enable({ + 'enable': true +}); + +``` + +* **`get`** +* with the loader, if concurrent `get` happens across the workers in a cluster, only one will be allowed to **load** while the rest will be in fact `watch` till that one finishes loading. +* this will reduce the stress upon the backend services which loads exact same data nicely + +```javascript +var cache; + +cache.get('cache-key-1', //key must be string + function(){ + return 'cache-value-loaded-1'; //value could be value or promise + }, + { + 'wait': 100//this is a timeout option + }) + .then(function(value){ + //the value resolved is anything already cached or the value newly loaded + //note, the loader will be called once and once only, if it failed, the promise of get will be rejected. + }) + .otherwise(function(error){ + + }); +``` +* **`set`** + +```javascript +var cache; + +cache.set('cache-key-1', //key must be string + 'cache-value-loaded-1', //value could be any json object + { + 'leaveIfNotNull': false,//default false, which allows set to overwrite existing values + 'wait': 100 + }) + .then(function(happens){ + //the happens resolved is a true/false value indicating if the value has been accepted by the cache manager + }) + .otherwise(function(error){ + + }); +``` +* **`del`** + +```javascript +var cache; + +cache.del('cache-key-1', //key must be string + { + 'wait': 100//this is a timeout option + }) + .then(function(value){ + //the old value deleted + }); +``` +* **`watch`** + +```javascript +var cache; + +cache.watch('cache-key-1', //key must be string or null (indicating watch everything) + function watching(value, key){ + //this is a callback which will be called anytime the associatd key has an updated value + }); +``` +* **`unwatch`** + +```javascript +var cache; + +cache.unwatch('cache-key-1', watching);//stop watching +``` + +## status + +This is a helpful piece evolved from the current cluster2, which is to allow applications to easily register status of any interest. +It allows each worker to register its own state, master would automatically aggregate all states from active workers. +It works nicely with our monitor capability (via debug middleware) + +* **`register`** + +```javascript +require('cluster2/status') + .register('status-name', + function(){ + return 'view';//view function + }, + function(value){ + //update function + }); +``` + +* **`statuses`** + +```javascript +require('cluster2/status') + .statuses(); //return names of registered statuses +``` + +* **`getStatus`** + +```javascript +require('cluster2/status') + .getStatus('status-name') + .then(function(status){ + //got status + }) + .otherwise(function(error){ + //err + }); +``` + +* **`setStatus`** + +```javascript +require('cluster2/status') + .setStatus('status-name', + 'value') + .then(function(set){ + //set or not + }) + .otherwise(function(error){ + //err + }); +``` diff --git a/TODO.md b/TODO.md deleted file mode 100644 index e05820a..0000000 --- a/TODO.md +++ /dev/null @@ -1,22 +0,0 @@ -* Basic cluster -* St art from js -* Start from command line args -* Add `since` header to report uptime -* `app.close()` during SIGTERM -* ECV -* Skip logging when disk is low -* exit story in worker unclear -* ws connection not working in chrome -* Mon test -* Update paas templates -* Graceful shutdown - stop connection listening -* Drain incoming connections on timeout -* Drain incoming connections on shutdown -* Process restart -* Process recycle -* ws connection causing workers to live -* Check for open port and and exit when busy with an error exit code -* Write start/shutdown/stop to log -* Send counters in bulk -* Traffic in and out - continue connection listening but update ecv -* Raise heartbeats thru logEmitter diff --git a/cache.js b/cache.js new file mode 100644 index 0000000..5143a7a --- /dev/null +++ b/cache.js @@ -0,0 +1,6 @@ +'use strict'; + +var cluster2 = process.cluster2 = process.cluster2 || {}; +cluster2.cache = cluster2.cache || require('./lib/cache'); + +module.exports = cluster2.cache; \ No newline at end of file diff --git a/emitter.js b/emitter.js new file mode 100644 index 0000000..541ad3d --- /dev/null +++ b/emitter.js @@ -0,0 +1,6 @@ +'use strict'; + +var cluster2 = process.cluster2 = process.cluster2 || {}; +cluster2.emitter = cluster2.emitter || require('./lib/emitter'); + +module.exports = cluster2.emitter; \ No newline at end of file diff --git a/examples/cluster-demo.js b/examples/cluster-demo.js new file mode 100644 index 0000000..95d78bf --- /dev/null +++ b/examples/cluster-demo.js @@ -0,0 +1,60 @@ +'use strict'; + +var listen = require('../lib/index').listen, + util = require('util'), + express = require('express'), + app = express(); + +listen({ + 'createServer': require('http').createServer, + 'app': app, + 'configureApp': function(){ + + //by the time, user application gets here, master already started, cache service started, caching is ready to be used. + + app.get('/', function(req, res){ + + var cache = require('../lib/cache').use('demo-cache'); + + var key = req.query.key, + val = req.query.value; + + if(!key){ + cache.get('key', function(){ + return 'value'; + }) + .then(function(value){ + res.send(util.format('hello from:%d whose cached value is:%j', process.pid, value), 200); + }); + } + else{ + console.log('[cache] set:%s=%j', key, val); + cache.set(key, val) + .then(function(set){ + res.send(util.format('[cache] set:%s to value:%j result:%s', key, val, set), 200); + }); + } + + }); + + return app;//in the end, the configured application (with middlewares, routes registered) should be returned + }, + 'debug': { + 'webPort': 9092, + 'saveLiveEdit': true + }, + 'gc': { + 'monitor': true + } +}) +.then(function(resolve){ + + require('../lib/status').register('worker', function(){ + + return process.pid; + }); +}) +.otherwise(function(error){ + + console.trace(error); +}); diff --git a/examples/cluster-longrun.js b/examples/cluster-longrun.js new file mode 100644 index 0000000..34ecf8f --- /dev/null +++ b/examples/cluster-longrun.js @@ -0,0 +1,124 @@ +'use strict'; + +var listen = require('../lib/index').listen, + util = require('util'), + path = require('path'), + express = require('express'), + store = new express.session.MemoryStore, + app = express(), + dust = require('dustjs-linkedin'), + cons = require('consolidate'), + routes = require('./routes'); + +listen({ + 'noWorkers': 3, + 'createServer': require('http').createServer, + 'app': app, + 'port': 8080, + 'monPort': 8081, + 'configureApp': function(app){ + + app.engine('dust', cons.dust); + + app.configure(function(){ + + app.set('template_engine', 'dust'); + app.set('domain', 'localhost'); + app.set('views', __dirname + '/views'); + app.set('view engine', 'dust'); + app.use(express.favicon()); + app.use(express.logger('dev')); + app.use(express.bodyParser()); + app.use(express.methodOverride()); + app.use(express.cookieParser('wigglybits')); + app.use(express.session({ + 'secret': 'whatever', + 'store': store + })); + app.use(express.session()); + app.use(app.router); + app.use(express.static(path.join(__dirname, 'public'))); + + //middleware + app.use(function(req, res, next){ + + if(req.session.user){ + req.session.logged_in = true; + } + + res.locals.message = req.flash(); + res.locals.session = req.session; + res.locals.q = req.body; + res.locals.err = false; + + next(); + }); + }); + + app.configure('development', function(){ + app.use(express.errorHandler()); + }); + + app.locals.inspect = util.inspect; + app.get('/', routes.index); + + return app; + }, + 'warmup': function(app, address){ + //warmup is done at both the initialization, and when the worker is to be replaced. + //the long run test verifies that with warmup, the 1st user request won't be slowed down even it's a new worker + var tillWarmUp = require('when').defer(); + + request.get('http://localhost:' + address.port, function(err, response, body){ + + tillWarmUp.resolve(app); + }); + + return tillWarmUp.promise; + }, + 'debug': { + 'webPort': 8082, + 'saveLiveEdit': true + }, + 'cache': { + 'enable': true, + 'mode': 'master' + }, + 'gc': { + 'monitor': true + }, + 'maxAge': 60,//1 minute, just to see how the workers get killed! + 'heartbeatInterval': 10000 +}) +.then(function(resolve){ + + require('../lib/status').register('worker', function(){ + return process.pid; + }); + + if(resolve.master){ //this means all workers have been warmed up! + + var request = require('request'), + _ = require('underscore'); + + (function round(){ + + _.each(_.range(0, 20), function(ith){ + + request.get('http://localhost:8080', function(err, response, body){ + + if(err || response.statusCode !== 200){ + console.log('[err:%j] response:%j and body:%s', err, response, body); + } + }); + }); + + setTimeout(round, 1000); + + })(); + } +}) +.otherwise(function(error){ + + console.trace(error); +}); diff --git a/examples/cluster-run.js b/examples/cluster-run.js new file mode 100644 index 0000000..b9a673c --- /dev/null +++ b/examples/cluster-run.js @@ -0,0 +1,40 @@ +'use strict'; + +var should = require('should'), + run = require('../lib/index').run; + +run({ + 'runnable': function(){ + + console.log('process:%d alive and runnable executed', process.pid); + //for user to put logic for testings here. + }, + 'noWorkers': 2, + 'debug': { + 'webPort': 9092, + 'saveLiveEdit': true + }, + 'cache': { + 'enable': true + }, + 'gc': { + 'monitor': true + } +}) +.then(function(resolve){ + + require('../lib/status').register('worker', function(){ + + return process.pid; + }); + + setTimeout(function(){ + //worker die in 1s, master die in 1s after promise resolved. + process.exit(0); + + }, 1000); +}) +.otherwise(function(error){ + + console.trace(error); +}); \ No newline at end of file diff --git a/examples/express/express-server.js b/examples/express/express-server.js deleted file mode 100644 index a62c739..0000000 --- a/examples/express/express-server.js +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var Cluster = require('../../lib/index.js'), - express = require('express'); - -// -// An express server cluster - -var serving = true; -var app = express.createServer(); -var monApp = express.createServer(); - -app.get('/', function(req, res) { - res.send('hello'); - if(!serving) { - req.connection.end(); - } -}); - -monApp.get('/monapp', function(req, res) { - res.send('Hello from Monitor app'); - if(!serving) { - req.connection.end(); - } -}); - -app.on('close', function() { - serving = false; -}) - -var c = new Cluster({ - port: 3000, - cluster: true, - timeout: 500, - noWorkers: 1, - connThreshold: 4, - ecv: { - path: '/ecv', // Send GET to this for a heartbeat - control: true, // send POST to /ecv/disable to disable the heartbeat, and to /ecv/enable to enable again - monitor: '/', - validator: function() { - return true; - } - } -}); - -c.on('died', function(pid) { - console.log('Worker ' + pid + ' died'); -}); -c.on('forked', function(pid) { - console.log('Worker ' + pid + ' forked'); -}); - -c.listen(function(cb) { - // You need to pass the app. monApp is optional. - // If monApp is not passed, cluster2 creates one for you. - cb(app, monApp); -}); diff --git a/examples/express/shutdown.js b/examples/express/shutdown.js deleted file mode 100644 index a2a080b..0000000 --- a/examples/express/shutdown.js +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var Cluster = require('../../lib/index.js'); -var c = new Cluster(); - -c.shutdown(); \ No newline at end of file diff --git a/examples/express/stop.js b/examples/express/stop.js deleted file mode 100644 index dc8709c..0000000 --- a/examples/express/stop.js +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -var Cluster = require('../../lib/index.js'); -var c = new Cluster(); -c.stop(); \ No newline at end of file diff --git a/examples/public/stylesheets/style.css b/examples/public/stylesheets/style.css new file mode 100644 index 0000000..f611ef8 --- /dev/null +++ b/examples/public/stylesheets/style.css @@ -0,0 +1,7 @@ +body { + padding: 50px; + font: 14px "Lucida Grande", Helvetica, Arial, sans-serif; +} +a { + color: #00B7FF; +} \ No newline at end of file diff --git a/examples/routes/index.js b/examples/routes/index.js new file mode 100644 index 0000000..5560be8 --- /dev/null +++ b/examples/routes/index.js @@ -0,0 +1,20 @@ +/* + * GET home page. + */ + +exports.index = function(req, res){ + + var cache = require('../../lib/cache').use('template_engine'); + + res.locals.session = req.session; + + cache.get('engine', function(){ + return req.app.settings.template_engine; + }) + .then(function(engine){ + + res.render('index', { + 'title': 'Express with ' + engine + }); + }); +}; \ No newline at end of file diff --git a/examples/views/index.dust b/examples/views/index.dust new file mode 100644 index 0000000..a258ac3 --- /dev/null +++ b/examples/views/index.dust @@ -0,0 +1,4 @@ +{>layout/} +{ + + + {title} + + + +

{title}

+ {+content} + This is the base content. + {/content} + + + \ No newline at end of file diff --git a/index.js b/index.js new file mode 100644 index 0000000..bd2c587 --- /dev/null +++ b/index.js @@ -0,0 +1,79 @@ +/* + * Copyright 2012 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +process.getLogger = process.getLogger || function defaultGetLogger(){ + + return { + + 'info': function(){ + console.log.apply(console, arguments); + }, + + 'debug': function(){ + console.log.apply(console, arguments); + } + } +}; + +var cluster2 = process.cluster2 = process.cluster2 || {}; + +cluster2.main = cluster2.main || require('underscore').extend(require('./lib/main'), { + + /** + * @return boolean whether the active process is master or not + */ + get isMaster(){ + + return require('cluster').isMaster; + }, + + /** + * @return boolean whether the active process is worker or not + */ + get isWorker(){ + + return require('cluster').isWorker; + }, + + /** + * @return the cluster emitter submodule + */ + get emitter(){ + + return require('./emitter'); + }, + + /** + * @return the cluster status submodule + */ + get status(){ + + return require('./status'); + }, + + /** + * @return the cluster cache submodule + */ + get cacheManager(){ + + return require('./cache'); + } +}); + +module.exports = cluster2.main; + diff --git a/lib/cache-common.js b/lib/cache-common.js new file mode 100644 index 0000000..903cddb --- /dev/null +++ b/lib/cache-common.js @@ -0,0 +1,92 @@ +'use strict'; + +var util = require('util'); + +var increment = 0, + writer = function writer(conn){ + + var writerOfConn = conn.writerOfConn || (function createWriter(){ + + var buffer = [], + directWriter = { + + 'write': function write(message){ + + if(!conn.write(message + '\r\n')){//kernel buffer cannot hold further + + conn.writerOfConn = bufferWriter; + + conn.once('drain', function(){//wait till 'drain' event and write access open + + conn.writerOfConn = directWriter; + + while(buffer.length){ + + directWriter.write(buffer.shift()); + } + }); + } + } + }, + + bufferWriter = { + + 'write': function write(message){ + + buffer.push(message); + } + }; + + return directWriter; + })(); + + return writerOfConn; + }; + +module.exports = { + + 'domainPath': process.env.CACHE_DOMAIN_PATH || './cluster-cache-domain', + + 'persistPath': process.env.CACHE_PERSIST_PATH || './cluster-cache-persist', + + 'status': { + 'success' : '1', + 'failure' : '-1' + }, + + 'types': { + 'NS' : 'ns', + 'GET' : 'get', + 'SET' : 'set', + 'LOCK' : 'lock', + 'DEL' : 'del', + 'ALL' : 'all', + 'INSPECT' : 'ins', + 'PING' : 'ping', + 'PONG' : 'pong' + }, + + 'serialize': function serialize(object){ + + return encodeURIComponent(JSON.stringify(object)); + }, + + 'deserialize': function deserialize(string){ + + return JSON.parse(decodeURIComponent(string)); + }, + + 'write': function(conn, message){ + + return writer(conn).write(message); + }, + + 'nextToken': function(){ + + return [process.pid, increment += 1].join('-'); + }, + + 'changeToken': 'chn' +}; + + diff --git a/lib/cache-mgr-worker.js b/lib/cache-mgr-worker.js new file mode 100644 index 0000000..83a60cb --- /dev/null +++ b/lib/cache-mgr-worker.js @@ -0,0 +1,72 @@ +'use strict'; + +var util = require('util'); +var when = require('when'); +var timeout = require('when/timeout'); +var assert = require('assert'); +var _ = require('underscore'); +var Worker = require('./worker').Worker; + +var CacheMgrWorker = exports.CacheMgrWorker = function (proc, options) { + Worker.call(this, proc, options); +} + +util.inherits(CacheMgrWorker, Worker); + +CacheMgrWorker.prototype.listen = function listen () { + var _this = this, + app = _this.app, + ports = _this.port, + createServer = _this.createServer, + warmUp = _this.warmUp, + debug = _this.debug, + wait = _this.timeout; + + assert.ok(createServer); + assert.ok(app); + assert.equal(_.isArray(ports), true, 'The cache manager worker should have a ports array'); + assert.ok(warmUp); + + var tillListen = when.defer(); + var run = function () { + _this.logger.info('[cache] manager %d listening on %j', _this.pid, ports); + var server = createServer(app).listen(ports, function(error) { + if (error) { + _this.logger.info('[cache] manager %d error: %j', _this.pid, error); + } + _this.logger.info('[cache] manager %d started listening on %j', _this.pid, ports); + when(warmUp()).ensure(function () { + _this.logger.info('[cache] manager %d warmed up', _this.pid); + _this.emitter.emit(util.format('worker-%d-warmup', _this.pid)); //tell everyone warmup is done + // cache manager does not need to switch port + _this.emitter.emit(util.format('worker-%d-listening', _this.pid), {port: ports}); //tell master, worker ready + + tillListen.resolve({ + 'server': server, + 'app': app, + 'port': port, + 'master': null, + 'worker': _this + }); + }); + }); + }; + + if (!debug) { + run(); + }else { + _this.emitter.once('run', run); + } + + return (wait > 0 ? timeout(_this.timeout, tillListen.promise) : tillListen.promise); +}; + +CacheMgrWorker.prototype.pause = function () { + throw new Error('pause is not supported in CacheMgrWorker'); +}; + +CacheMgrWorker.prototype.resume = function () { + throw new Error('resume is not supported in CacheMgrWorker'); +}; + + diff --git a/lib/cache-mgr.js b/lib/cache-mgr.js new file mode 100644 index 0000000..8143872 --- /dev/null +++ b/lib/cache-mgr.js @@ -0,0 +1,362 @@ +'use strict'; + +var common = require('./cache-common'), + _ = require('underscore'), + fs = require('graceful-fs'), + os = require('os'), + net = require('net'), + util = require('util'), + path = require('path'), + when = require('when'), + timeout = require('when/timeout'), + ensureDir = require('./utils').ensureDir; + +var success = common.status.success, + failure = common.status.failure, + NS = common.types.NS, + GET = common.types.GET, + SET = common.types.SET, + DEL = common.types.DEL, + ALL = common.types.ALL, + LOCK = common.types.LOCK, + INSPECT = common.types.INSPECT, + PING = common.types.PING, + PONG = common.types.PONG, + CHN = common.changeToken, + domain = '/tmp/cache.socket.' + process.pid, + domainPath = common.domainPath, + persistPath = common.persistPath, + ports = process.env.CACHE_PORTS ? JSON.parse(process.env.CACHE_PORTS) : [9190, 9191], + namespaces = { + '': { //meta namespace + '': { //meta of meta namespace itself + 'value': { //meta value + 'persist': true, //persist true + 'expire': 0, //never expire + 'lastModified': 0, + 'lastPersisted': -1 + } + } + } + }, + metaOfNs = function metaOfNs(namespace){ + + var entry = namespaces[''][namespace]; + + return entry ? entry.value : { + 'persist': false, + 'expire': 0 + };//default meta + }, + conns = [], + logger = process.getLogger(__filename); + +var manager = { + + 'domain': domain, + + 'ns': function(reply, token){ + + reply({ + 'type': NS, + 'token': token, + 'namespaces': _.keys(namespaces), + 'status': success + }); + }, + + 'all': function(reply, token, namespace){ + + var cache = namespaces[namespace] || {}; + + reply({ + 'type': ALL, + 'token': token, + 'keys': _.keys(cache), + 'status': success + }); + }, + + 'lock': function(reply, token, namespace, key, value){ + + var cache = namespaces[namespace] = namespaces[namespace] || {}, + write = cache[key] === undefined; + + if(write){ + cache[key] = { + 'lock': value + }; + } + + reply({ + 'type': LOCK, + 'token': token, + 'key': key, + 'status': write ? success: failure + }); + }, + + 'set': function(reply, token, namespace, key, value, overwrite){ + + var cache = namespaces[namespace] = namespaces[namespace] || {}, + meta = metaOfNs(namespace), + expire = meta.expire, + old = cache[key], + write = (old === undefined || overwrite); + + cache[key] = write ? { + 'lock': old ? old.lock : null, + 'value': value, + 'expire': expire > 0 ? Date.now + expire : 0 //expire at + } + : cache[key]; + + reply({ + 'type': SET, + 'token': token, + 'key': key, + 'status': write ? success: failure + }); + + if(write){ + + meta.lastModified = Date.now(); + + manager.notify(namespace, key, cache[key]['value']); + } + }, + + 'notify': function(namespace, key, value){ + + var notify = { + 'token': CHN, + 'ns': namespace, + 'key': key, + 'value': value + }; + + this.cacheSocket.send(notify); + + if(namespace === '' + && value === undefined + && key !== ''){ + //dump the cache + delete namespaces[key]; + } + }, + + 'get': function(reply, token, namespace, key, value){ + + var cache = namespaces[namespace] || {}, + entry = cache[key] || {}, + result = { + 'type': GET, + 'token': token, + 'key': key, + 'value': entry.value || value,//default + 'status': success + }; + + reply(result); + }, + + 'ins': function(reply, token, namespace, key){ + + var cache = namespaces[namespace] || {}, + meta = metaOfNs(namespace), + entry = cache[key], + result = { + 'type': INSPECT, + 'token': token, + 'key': key, + 'value': entry ? entry.value : null, + 'persist': meta.persist, + 'expire': meta.expire && entry ? Date.now() - entry.expire : meta.expire, + 'status': success + }; + + reply(result); + }, + + 'del': function(reply, token, namespace, key){ + + var cache = namespaces[namespace] || {}, + entry = cache[key] || {}, + value = entry.value; + + delete cache[key];//locker released too + + metaOfNs(namespace).lastModified = Date.now(); + + reply({ + 'type': DEL, + 'token': token, + 'key': key, + 'value': value, + 'status': success + }); + + manager.notify(namespace, key, undefined); + }, + + 'ping': function(token){ + + this.cacheSocket.send({ + 'type': PING, + 'token': PONG + }); + }, + + 'pong': function(reply, token){ + //nothing + } +}; + +module.exports = { + + 'createServer': function createServer(app) { + return app; + }, + + 'app': (function () { //'connection' listener + manager.cacheSocket = require('./cache-socket/cache-socket-factory').getCacheSocket('manager', 'json'); + + manager.cacheSocket.on('message', function (command, reply) { + manager[command.type].apply(manager, [ + reply, command.token, command.ns, command.key, command.value, !command.leaveIfNonNull + ]); + }); + + var keepAlive = setInterval(function(){ + //this is to keep the connection open, just to send PING, and receive PONG, and will be extended to validate the health of the connection later + manager.ping(PONG); + }, 3000);//ping/pong every 3secs + + manager.cacheSocket.once('close', function(){ + + loggerinfo('cacheSock closed'); + + clearInterval(keepAlive); + }); + + return manager.cacheSocket; + })(), + + 'port': ports, + + 'afterServerStarted': function() { //'listening' listener + + logger.info('[cache] manager started, persistence:%j', persistPath); + + ensureDir(persistPath); + + var persistMeta = path.join(persistPath, '.cache'); + + if(fs.existsSync(persistMeta)){ + + namespaces[''] = common.deserialize(fs.readFileSync(persistMeta, {'encoding': 'utf-8'})); + } + + logger.info('[cache] manager loading from persistences: %j', namespaces['']); + + _.each(namespaces, function(cache, namespace){ + + var pathOfNs = path.join(persistPath, namespace + '.cache'); + + if(!metaOfNs(namespace).persist){ + + if(fs.existsSync(pathOfNs)){//dump the persisted cache + fs.unlinkSync(pathOfNs); + } + + delete namespaces[''][namespace];//dump from meta + } + else{ + + if(fs.existsSync(pathOfNs)){ + //load from persisted cache + namespaces[namespace] = common.deserialize(fs.readFileSync(pathOfNs, {'encoding': 'utf-8'})); + } + else{ + namespaces[namespace] = {};//empty cache + } + } + }); + + logger.info('[cache] manager loaded from all persistences'); + + //register domain to where all other users and the master process could watch + fs.writeFileSync(domainPath, JSON.stringify(ports)); + logger.debug('[cache] manager wrote %j to %s', ports, domainPath); + + var nextUpdate = null, + updateTask = function updateTask(){ + + //expiration monitoring every 10 seconds + var now = Date.now(), + cachesToPersist = {}; + + _.each(namespaces, function(cache, namespace){ + + var meta = metaOfNs(namespace), + persist = meta.persist, + expire = meta.expire; + + logger.debug('[cache][maintain] at:%d upon namespace:%s meta:%j', now, namespace, meta); + + if(expire > 0){ + _.each(cache, function(entry, key){ + + if(entry.expire <= now){ + + var old = cache[key]; + + delete cache[key]; + + manager.notify(namespace, key, old); + } + }); + } + + if(persist && (meta.lastPersisted < meta.lastModified)){ + cachesToPersist[namespace] = cache; + } + }); + + logger.debug('[cache][maintain] expiration finished, and persist the following:%j', _.keys(cachesToPersist)); + + when.all(_.map(cachesToPersist, function(cache, namespace){ + + var tillPersist = when.defer(), + persistence = path.join(persistPath, namespace + '.cache'); + + logger.debug('[cache][maintain] to persist namespace:%s to:%s', namespace, persistence); + + fs.writeFile(persistence, common.serialize(cache), function(err){ + + logger.debug('[cache][maintain] persisted:%s %s', persistence, err ? 'with error' : 'successfully'); + + metaOfNs(namespace).lastPersisted = Date.now(); + + tillPersist.resolve(!err); + }); + + return timeout(tillPersist, 10000);//too big? + })) + .ensure(function(){ + + nextUpdate = setTimeout(updateTask, 10000); + }) + }; + + updateTask(); + + process.once('SIGINT', function(){ + + clearTimeout(nextUpdate); + + }); + + return manager; + } +}; diff --git a/lib/cache-socket/cache-mgr-socket.js b/lib/cache-socket/cache-mgr-socket.js new file mode 100644 index 0000000..4ae0e5a --- /dev/null +++ b/lib/cache-socket/cache-mgr-socket.js @@ -0,0 +1,28 @@ +'use strict'; + +var util = require('util'); +var axon = require('axon'); +var CacheSocket = require('./cache-socket.js'); + +function CacheMgrSocket(format) { + var _this = this; + + _this.pubSock = axon.socket('pub'); + _this.repSock = axon.socket('rep'); + + _this.repSock.on('message', function (msg, reply) { + _this.emit('message', msg, reply); + }); + + return CacheSocket.call(_this, [_this.repSock, _this.pubSock], format); +} + +util.inherits(CacheMgrSocket, CacheSocket); + +CacheMgrSocket.prototype.send = function send(msg) { + var _this = this; + + _this.pubSock.send(msg); +}; + +module.exports = CacheMgrSocket; diff --git a/lib/cache-socket/cache-socket-factory.js b/lib/cache-socket/cache-socket-factory.js new file mode 100644 index 0000000..60622c4 --- /dev/null +++ b/lib/cache-socket/cache-socket-factory.js @@ -0,0 +1,15 @@ +'use strict'; + +var CacheUsrSocket = require('./cache-usr-socket.js'); +var CacheMgrSocket = require('./cache-mgr-socket.js'); + +exports.getCacheSocket = function (type, format) { + if (type === 'manager') { + return new CacheMgrSocket(format); + }else if (type === 'user') { + return new CacheUsrSocket(format); + }else { + throw new Error('Unknown cache socket type'); + } + return null; +}; diff --git a/lib/cache-socket/cache-socket.js b/lib/cache-socket/cache-socket.js new file mode 100644 index 0000000..8f8efa9 --- /dev/null +++ b/lib/cache-socket/cache-socket.js @@ -0,0 +1,144 @@ +'use strict'; + +var when = require('when'); +var _ = require('underscore'); +var EventEmitter = require('events').EventEmitter; +var util = require('util'); + +function CacheSocket(sockets, format) { + EventEmitter.call(this); + return this.initialize(sockets, format); +} + +util.inherits(CacheSocket, EventEmitter); + +CacheSocket.prototype.initialize = function initialize(sockets, format) { + format = format || 'json'; + sockets = sockets || []; + sockets = _.isArray(sockets) ? sockets : [sockets]; + + _.each(sockets, function (socket) { + socket.format(format); + socket.on('error', function (error) { + _this.emit('error', error); + }); + }); + + this.sockets = sockets; +}; + +CacheSocket.prototype.listen = function listen(ports, hosts, cb) { + var _this = this; + + if (_.isFunction(hosts)) { + cb = hosts; + hosts = undefined; + } + + ports = ports || []; + if (ports.length < _this.sockets.length) { + var error = new Error('The number of ports does not match the number of sockets'); + if (cb) { + cb(error); + } + _this.emit('error', error); + return _this; + } + + hosts = hosts || []; + + when.map(_.range(_this.sockets.length), function (ith) { + var tillBind = when.defer(); + _this.sockets[ith].bind(ports[ith], hosts[ith], function (error) { + if (error) { + tillBind.reject(error); + }else { + tillBind.resolve(_this.sockets[ith]); + } + }); + return tillBind.promise; + }).then(function (resolved) { + if (cb) { + cb(null); + } + _this.emit('listen'); + }).otherwise(function (error) { + if (cb) { + cb(error); + } + _this.emit('error', error); + }); + + return _this; +}; + +CacheSocket.prototype.close = function close(cb) { + var _this = this; + + when.map(_this.sockets, function (socket) { + var tillClose = when.defer(); + socket.close(function (error) { + if (error) { + tillClose.reject(error); + }else { + tillClose.resolve(socket); + } + }); + return tillClose.promise; + }).then(function (resolved) { + if (cb) { + cb(null); + } + _this.emit('close'); + }).otherwise(function (error) { + if (cb) { + cb(error); + } + _this.emit('error', error); + }); +}; + +CacheSocket.prototype.connect = function connect(ports, hosts, cb) { + var _this = this; + + if (_.isFunction(hosts)) { + cb = hosts; + hosts = undefined; + } + + ports = ports || []; + if (ports.length < _this.sockets.length) { + var error = new Error('The number of ports does not match the number of sockets'); + if (cb) { + cb(error); + } + _this.emit('error', error); + return _this; + } + + hosts = hosts || []; + + when.map(_.range(_this.sockets.length), function (ith) { + var tillConnect = when.defer(); + _this.sockets[ith].connect(ports[ith], hosts[ith], function (error) { + if (error) { + tillConnect.reject(error); + }else { + tillConnect.resolve(_this.sockets[ith]); + } + }); + return tillConnect.promise; + }).then(function (resolved) { + if (cb) { + cb(null); + } + _this.emit('connect'); + }).otherwise(function (error) { + if (cb) { + cb(error); + } + _this.emit('error', error); + }); +}; + +module.exports = CacheSocket; diff --git a/lib/cache-socket/cache-usr-socket.js b/lib/cache-socket/cache-usr-socket.js new file mode 100644 index 0000000..a15fd53 --- /dev/null +++ b/lib/cache-socket/cache-usr-socket.js @@ -0,0 +1,27 @@ +'use strict'; + +var util = require('util'); +var axon = require('axon'); +var CacheSocket = require('./cache-socket.js'); + +function CacheUsrSocket(format) { + var _this = this; + _this.subSock = axon.socket('sub'); + _this.reqSock = axon.socket('req'); + + _this.subSock.on('message', function (msg) { + _this.emit('message', msg); + }); + + return CacheSocket.call(_this, [_this.reqSock, _this.subSock], format); +} + +util.inherits(CacheUsrSocket, CacheSocket); + +CacheUsrSocket.prototype.send = function send(msg, reply) { + var _this = this; + + _this.reqSock.send(msg, reply); +}; + +module.exports = CacheUsrSocket; diff --git a/lib/cache-usr.js b/lib/cache-usr.js new file mode 100644 index 0000000..c29030c --- /dev/null +++ b/lib/cache-usr.js @@ -0,0 +1,499 @@ +'use strict'; + +var _ = require('underscore'), + net = require('net'), + util = require('util'), + when = require('when'), + timeout = require('when/timeout'), + fs = require('graceful-fs'), + common = require('./cache-common'), + cacheSocket = require('./cache-socket/cache-socket-factory.js').getCacheSocket('user', 'json'); + +var success = common.status.success, + NS = common.types.NS, + ALL = common.types.ALL, + GET = common.types.GET, + SET = common.types.SET, + DEL = common.types.DEL, + LOCK = common.types.LOCK, + INSPECT = common.types.INSPECT, + PONG = common.types.PONG, + CHN = common.changeToken, + nextToken = common.nextToken, + logger = process.getLogger(__filename), + domain = null, + userDeferred = when.defer(), + handlers = { + + }, + changes = { + + }, + anyChanges = { + + }, + stats = { + + }, + conn = null, + ports = null, + //TODO, support connection pooling to speed up the cache operations if needed. + reconnect = function reconnect(error){ + + cacheSocket.on('error', function (error) { + logger.error(error); + }); + + cacheSocket.connect(ports, function (error) { + logger.info('[cache] user connected to cache server on %j', ports); + if (error) { + logger.error(error); + userDeferred.reject(error); + }else if (!userDeferred.hasBeenResolved) { + userDeferred.resolve(process.user = user); + userDeferred.hasBeenResolved = true; + } + }); + + cacheSocket.on('message', function(response){ + var token = response.token, + namespace = response.ns, + key = response.key, + value = response.value; + + if(token === CHN){ + + changes[namespace] = changes[namespace] || {}; + _.each(changes[namespace][key] || [], function(whenChange){ + whenChange(value, key); + }); + + _.invoke(anyChanges[namespace] || [], 'call', null, value, key); + }else if(token === PONG){//just to keep the connection open + + cacheSocket.send({ + 'type': PONG, + 'token': PONG + }, function () { + // do not need to reply + }); + } + }); + + cacheSocket.once('close', function(error){ + + reconnect(error); + }); + }, + reply = function (response) { + logger.debug('[cache] user get response %j', response); + + var token = response.token, + namespace = response.ns, + key = response.key, + value = response.value; + + handlers[token].apply(user, [ + response.status, + key || response.keys || response.namespaces, + value, + response.persist, + response.expire + ]); + }; + +var user = process.user || { + + 'ns': function(options){ + + options = options || {}; + + var token = nextToken(), + wait = options.wait, + tillNs = when.defer(), + handler = function(status, namespaces){ + + if(success === status){ + + tillNs.resolve(namespaces || []); + } + else{ + + tillNs.reject(new Error('failed to get namespaces')); + } + }; + + handlers[token] = handler; + + cacheSocket.send({ + 'type': NS, + 'token': token + }, reply); + + return (wait > 0 ? timeout(wait, tillNs.promise) : tillNs.promise).ensure(function(){ + delete handlers[token]; + }); + }, + + 'keys': function(namespace, options){ + + options = options || {}; + + var token = nextToken(), + wait = options.wait, + tillKeys = when.defer(), + handler = function(status, keys){ + + if(success === status){ + + tillKeys.resolve(keys || []); + } + else{ + + tillKeys.reject(new Error('failed to get keys')); + } + }; + + handlers[token] = handler; + + cacheSocket.send({ + 'type': ALL, + 'token': token, + 'ns': namespace + }, reply); + + return (wait > 0 ? timeout(wait, tillKeys.promise) : tillKeys.promise).ensure(function(){ + delete handlers[token]; + }); + }, + + 'get': function(namespace, key, loader, options){ + + options = options || {}; + + var token = nextToken(), + wait = options.wait, + tillGet = when.defer(), + handler = function handler(status, key, value){ + + if(success === status && value !== undefined){//got the value + + user.stat(namespace, 'hit'); + tillGet.resolve(value); + } + else if(loader){//must atomically load the value + + var watchOthers = function watchOthers(changed){ + //unregister itself immediately + user.unwatch(namespace, key, watchOthers); + + if(changed !== undefined){ + user.stat(namespace, 'hit'); + tillGet.resolve(changed); + } + else{ + user.stat(namespace, 'error'); + tillGet.reject(new Error('loader failed')); + } + }; + + user.watch(namespace, key, watchOthers); + user.lock(namespace, key, { + 'wait': wait + }) + .then(function(locked){ + //only one of the concurrent writers will be given the set===true + if(locked){ + + var handleError = function handleError(error){ + throw error; + }; + + user.stat(namespace, 'miss'); + user.unwatch(namespace, key, watchOthers);//unregister immediately as i'm about to write the value + try{ + //promise or value + when(loader(), function(value){ + + user.stat(namespace, 'load'); + //success value loaded + user.set(namespace, key, value, { + 'wait': wait + }) + .then( + _.bind(tillGet.resolve, tillGet, value), + handleError + ); + + }, + handleError); + } + catch(e){ + //in case loaded fail + user.stat(namespace, 'error'); + user.del(namespace, key).ensure(function(){ + tillGet.reject(e); + }); + } + } + }); + } + else{ + //got nothing + user.stat(namespace, 'miss'); + tillGet.resolve(null);//no value, but resolved + } + }; + + + //local copy miss, fetch from master cache + handlers[token] = handler; + + cacheSocket.send({ + 'type': GET, + 'token': token, + 'ns': namespace, + 'key': key + }, reply); + + return (wait > 0 ? timeout(wait, tillGet.promise) : tillGet.promise).ensure(function(){ + delete handlers[token]; + }); + }, + + 'inspect': function(namespace, key, options){ + + options = options || {}; + + var token = nextToken(), + wait = options.wait, + tillInspect = when.defer(), + handler = function(status, key, value, persist, expire){ + + if(success === status && value !== undefined){ + + tillInspect.resolve([value, persist, expire]); + } + else{ + tillInspect.reject(new Error('no value found for key')); + } + }; + + //local copy miss, fetch from master cache + handlers[token] = handler; + + cacheSocket.send({ + 'type': INSPECT, + 'token': token, + 'ns': namespace, + 'key': key + }, reply); + + return (wait > 0 ? timeout(wait, tillInspect.promise) : tillInspect.promise).ensure(function(){ + delete handlers[token]; + }); + }, + + /** + * @param key string + * @param value Object + * @param options { + persist boolean (whether should survice cache-mgr failure) + expire number (time to live, default null, won't expire) + wait number (timeout after wait expired) + leaveIfNonNull boolean (true means the set will backout if the value exists, default as false, which will overwrite the value) + * } + */ + 'set': function(namespace, key, value, options){//must guarantee that value has no '\r\n' in it (if it's a string or any complex type) + + options = options || {}; + + var token = nextToken(), + wait = options.wait, + tillSet = when.defer(), + handler = function(status, key){ + + tillSet.resolve(success === status); + }; + + handlers[token] = handler; + + cacheSocket.send({ + 'type': SET, + 'token': token, + 'ns': namespace, + 'key': key, + 'value': value, + 'leaveIfNonNull': options.leaveIfNonNull + }, reply); + + return (wait > 0 ? timeout(wait, tillSet.promise) : tillSet.promise).ensure(function(){ + delete handlers[token]; + }); + }, + + 'lock': function(namespace, key, options){//must guarantee that value has no '\r\n' in it (if it's a string or any complex type) + + options = options || {}; + + var token = nextToken(), + wait = options.wait, + tillLock = when.defer(), + handler = function(status, key){ + + tillLock.resolve(success === status); + }; + + handlers[token] = handler; + + cacheSocket.send({ + 'type': LOCK, + 'token': token, + 'ns': namespace, + 'key': key, + 'value': process.pid + }, reply); + + return (wait > 0 ? timeout(wait, tillLock.promise) : tillLock.promise).ensure(function(){ + delete handlers[token]; + }); + }, + + /** + * @param key string + * @param wait number (timeout after wait expired) + */ + 'del': function(namespace, key, options){ + + options = options || {}; + + var token = nextToken(), + wait = options.wait, + tillDel = when.defer(), + handler = function(status, key, value){ + + tillDel.resolve(success === status ? value : null); + }; + + handlers[token] = handler; + + cacheSocket.send({ + 'type': DEL, + 'token': token, + 'ns': namespace, + 'key': key + }, reply); + + return (wait > 0 ? timeout(wait, tillDel.promise) : tillDel.promise).ensure(function(){ + delete handlers[token]; + }); + }, + + 'watch': function(namespace, key, callback){ + + if(!key){ + anyChanges[namespace] = anyChanges[namespace] || []; + anyChanges[namespace].push(callback); + } + else{ + changes[namespace] = changes[namespace] || {}; + changes[namespace][key] = changes[namespace][key] || []; + changes[namespace][key].push(callback); + } + }, + + 'unwatch': function(namespace, key, callback){ + + if(!key){ + anyChanges[namespace] = _.without(anyChanges[namespace] || [], callback); + } + else{ + changes[namespace] = changes[namespace] || {}; + changes[namespace][key] = _.without(changes[namespace][key] || [], callback); + } + }, + + 'stat': function(namespace, action){ + + var stat = stats[namespace] = stats[namespace] || { + + 'hit': 0, + 'miss': 0, + 'load': 0, + 'error': 0 + }; + + if(action){ + stat[action] += 1; + } + + return stat; + }, + + 'switchPorts': function (p) { + if (p && JSON.stringify(ports) !== p) { + ports = JSON.parse(p); + reconnect(); + //cacheSocket.close(); + } + }, + + 'switchDomain': function(d){//only in case of cache-mgr down and needs to switch to a new one + + if(d && domain !== d){ + + domain = d; + if(conn){ + conn.writable = false; + conn.end(); + } + reconnect(); + } + }, + + 'pong': function(){ + + } +}; + +//when the cache-mgr is created by a replacement process, the new domain will be written to the same file being watched below +var tillExists = function(path){ + fs.exists(path, function(exists){ + if(exists){ + fs.watchFile(path, + function(){ + fs.readFile(path, { + 'encoding': 'utf-8' + }, + function(err, p){ + logger.info('[cache] switching ports:%s', p); + user.switchPorts(p); + }); + }); + } + else{ + process.nextTick(function(){ + tillExists(path); + }); + } + }); +}; + +exports.user = function(p){ + + if(!process.userPromise){//each process needs at most one cache user + + logger.info('[cache] user created'); + + tillExists(common.domainPath); + + p = p || fs.readFileSync(common.domainPath, {'encoding':'utf-8'}); + + user.switchPorts(p); + + process.userPromise = userDeferred.promise; + } + + return process.userPromise; +}; + diff --git a/lib/cache.js b/lib/cache.js new file mode 100644 index 0000000..8629fb2 --- /dev/null +++ b/lib/cache.js @@ -0,0 +1,202 @@ +'use strict'; + +var _ = require('underscore'), + when = require('when'), + timeout = require('when/timeout'), + pipeline = require('when/pipeline'), + utils = require('./utils.js'); + +var Cache = exports.Cache = function(namespace, usrAndMetaPromise){ + + _.extend(this, { + + 'namespace': namespace, + + 'getUsrAndMeta': function(){ + + return usrAndMetaPromise; + }, + + 'pipeKeys': function(usrAndMeta){ + + return usrAndMeta.usr.keys(namespace); + }, + + 'getPipeGet': function(key, loader, options){ + + return function(usrAndMeta){ + + return usrAndMeta.usr.get(namespace, key, loader, options); + }; + }, + + 'getPipeSet': function(key, value, options){ + + return function(usrAndMeta){ + + return usrAndMeta.usr.set(namespace, key, value, options); + }; + }, + + 'getPipeDel': function(key, options){ + + return function(usrAndMeta){ + + return usrAndMeta.usr.del(namespace, key, value, usrAndMeta.meta); + }; + }, + + 'getPipeWatch': function(key, onChange){ + + return function(usrAndMeta){ + + return usrAndMeta.usr.watch(namespace, key, onChange); + }; + }, + + 'getPipeUnwatch': function(key, onChange){ + + return function(usrAndMeta){ + + return usrAndMeta.usr.unwatch(namespace, key, onChange) + }; + }, + + 'pipeStat': function(usrAndMeta){ + + return usrAndMeta.usr.stat(namespace); + }, + + 'pipeDestroy': function(usrAndMeta){ + + return usrAndMeta.usr.del('', namespace); + } + }); +}; + +Cache.prototype.meta = function(){ + + return this.getUsrAndMeta().then(function(usrAndMeta){ + + return usrAndMeta.meta; + }); +} + +Cache.prototype.keys = function keys(){ + + return pipeline([this.getUsrAndMeta, this.pipeKeys]); +}; + +Cache.prototype.get = function get(key, loader, options){ + + return pipeline([this.getUsrAndMeta, this.getPipeGet(key, loader, options)]); +}; + +Cache.prototype.set = function set(key, value, options){ + + return pipeline([this.getUsrAndMeta, this.getPipeSet(key, value, options)]); +}; + +Cache.prototype.del = function del(key, options){ + + return pipeline([this.getUsrAndMeta, this.getPipeDel(key, options)]); +}; + +Cache.prototype.watch = function watch(key, onChange){ + + return pipeline([this.getUsrAndMeta, this.getPipeWatch(key, onChange)]); +}; + +Cache.prototype.unwatch = function unwatch(key, onChange){ + + return pipeline([this.getUsrAndMeta, this.getPipeUnwatch(key, onChange)]); +}; + +Cache.prototype.stat = function stat(){ + + return pipeline([this.getUsrAndMeta, this.pipeStat]); +}; + +Cache.prototype.destroy = function destroy(){ + + return pipeline([this.getUsrAndMeta, this.pipeDestroy]); +}; + +module.exports = { + + 'enable': _.once(function(options, master){ + + if(!options.enable || !require('cluster').isMaster || (process.cluster2 && process.cluster2.caching)){ + //only master is allowed to enable cache, caching could be enabled only once + return; + } + + process.cluster2 = process.cluster2 || {}; + process.cluster2.caching = options.enable; + + if(options.mode === 'standalone' && master){ + + return master.fork(master.options, { + 'CACHE_MANAGER': true, + 'CACHE_DOMAIN_PATH': options.domainPath, + 'CACHE_PERSIST_PATH': options.persistPath + }); + } + else{ + var basePort = master ? master.port + 10 : 9190; + + return utils.pickAvailablePorts(basePort, basePort + 20, 2).then(function (ports) { + process.env.CACHE_PORTS = JSON.stringify(ports); + var mgr = require('./cache-mgr'); + var svr = mgr.createServer(mgr.app); + process.cacheServer = svr; + svr.listen(mgr.port, mgr.afterServerStarted); + }).otherwise(function (error) { + logger.error(error); + }); + } + }), + + 'use': function(namespace, options){ + + var logger = process.getLogger(__filename), + actualOptions = options || {}; + + _.defaults(actualOptions, { + 'persist': false, + 'expire': 0, + 'timeout': 3000, + 'lastPersisted': -1 + }); + + return new Cache(namespace, pipeline([ + + function(domain){ + + logger.debug('[cache] using domain:%s', domain); + return require('./cache-usr').user(domain); + }, + + function(usr){ + + logger.debug('[cache] usr ready, and using namespace:%s & options:%j', namespace, actualOptions); + return when.join(usr, usr.get('', namespace, function(){ + + return actualOptions; + })); + }, + + function(resolve){ + + var usr = resolve[0], + meta = resolve[1]; + + logger.debug('[cache] namespace:%s reserved with meta:%j', namespace, meta); + return { + 'usr': usr, + 'meta': meta + }; + } + ], actualOptions.domain));//optional + } +}; diff --git a/lib/component-status.js b/lib/component-status.js deleted file mode 100644 index 556e33a..0000000 --- a/lib/component-status.js +++ /dev/null @@ -1,342 +0,0 @@ -var _ = require('underscore'), - cluster = require('cluster'), - EventEmitter = require('events').EventEmitter, - util = require('util'); - -function DEFAULT_REDUCER(memoize, element) { - - return { - value : memoize ? memoize.value + element : element//whether it's string or number, most of the scenarios could be handled - }; -} - -function AVERAGE_REDUCER(memoize, element) { - - return { - total : memoize ? memoize.total + element : element, - count : memoize ? memoize.count + 1 : 1, - get value(){ - return this.total / this.count; - } - } -} - -function ARRAY_REDUCER(memoize, element) { - - return { - value : memoize ? memoize.value.concat([element]) : [element] - }; -} - -function FIRST_REDUCER(memoize, element) { - - return { - value : memoize ? memoize.value : element - }; -} - -//componentStatus module allows application to #register their component view handlers -//each handler should take a simple JSON object {'component':', 'view':'[JSON|HTML]'} and produce a result -//which has same keys, except for that view should contain the actual view content, as string to be displayed -var ComponentStatus = exports.ComponentStatus = function(emitter){ - - var components = this.components = {}, - workers = this.workers = {}, - reducers = this.reducers = {}; - - this.emitter = emitter; - emitter.on('new-component-status', function(component){ - if (component.worker !== process.pid && cluster.isMaster) { - - if (!components[component.name] && component.loader) { - require(component.loader.path).load.apply(null, component.loader.args); - } - } - - //console.log('[cluster2] master component-status:' + JSON.stringify(component)); - components[component.name] = { - 'reducer': component.reducer, - 'count': components[component.name] ? components[component.name].count + 1 : 1 - }; - - workers[component.worker] = workers[component.worker] || []; - workers[component.worker].push(component.name); - }); - - emitter.on('worker-died', function(worker){ - - _.each(workers[worker] || [], function(component){ - components[component].count = components[component].count - 1; - }); - }); - - this.reducer('default', DEFAULT_REDUCER) - .reducer('sum', DEFAULT_REDUCER) - .reducer('concat', DEFAULT_REDUCER) - .reducer('avg', AVERAGE_REDUCER) - .reducer('array', ARRAY_REDUCER) - .reducer('first', FIRST_REDUCER); -}; - -//worker -ComponentStatus.prototype.register = function(name, handler, reducer, updater, loader){ - - var emitter = this.emitter; - emitter.emit('new-component-status', { - 'name' : name, - 'reducer': reducer, - 'worker': process.pid, - 'loader': loader - }); - - emitter.on('get-component-status', function(component, now, options){ - if(_.isEqual(name, component)){ - emitter.emit( - util.format('get-component-status-%s-%d-%s', component, now, options.worker ? process.pid : ''), - handler(options.params)); - } - }); - - if(updater){ - emitter.on('update-component-status', function(component, now, options, value){ - if(_.isEqual(name, component)){ - emitter.emit( - util.format('update-component-status-%s-%d-%s', component, now, options.worker ? process.pid : ''), - updater(options.params, value)); - } - }); - } - - if(cluster.isWorker){ - var components = this.components = this.components || {}; - components[name] = { - 'reducer': reducer, - 'count': components[name] ? components[name].count + 1 : 1 - }; - } - - return this; -}; - -//master -ComponentStatus.prototype.reducer = function(name, handler){ - - this.reducers[name] = handler; - return this; -} - -//master -ComponentStatus.prototype.getComponents = function(){ - - return _.keys(this.components); -}; - -//master -ComponentStatus.prototype.getStatus = function(component, options){ - - options = options || {}; - - var emitter = this.emitter, - params = options.params || [], - worker = options.worker, - done = options.done, - expects = worker ? 1 : this.components[component].count, - reducer = this.reducers[this.components[component].reducer] || DEFAULT_REDUCER, - now = Date.now(), - event = util.format('get-component-status-%s-%d-%s', component, now, worker || ''), - all = [], - collect = function(status){ - all.push(status); - if((expects -= 1) === 0){ - clearTimeout(timeOut); - emitter.removeListener(event, collect); - done(_.reduce(all, reducer, null).value); - } - }, - timeOut = setTimeout(function(){ - emitter.removeListener(event, collect); - var partial = _.reduce(all, reducer, null); - done(partial ? partial.value : null); - }, 3000); - - //console.log('[cluster2] getStatus:' + component + ';event:' + event + ';expects:' + expects); - emitter.on(event, collect); - //console.log('[cluster2] expects:' + event); - emitter.emit('get-component-status', component, now, params); - - return this; -}; - -ComponentStatus.prototype.setStatus = function(component, options, value){ - - options = options || {}; - - var emitter = this.emitter, - params = options.params || [], - worker = options.worker, - done = options.done, - expects = worker ? 1 : this.components[component].count, - reducer = this.reducers[this.components[component].reducer] || DEFAULT_REDUCER, - now = Date.now(), - event = util.format('update-component-status-%s-%d-%s', component, now, worker || ''), - all = [], - collect = function(status){ - all.push(status); - if((expects -= 1) === 0){ - clearTimeout(timeOut); - emitter.removeListener(event, collect); - done(_.reduce(all, reducer, null).value); - } - }, - timeOut = setTimeout(function(){ - emitter.removeListener(event, collect); - var partial = _.reduce(all, reducer, null); - done(partial ? partial.value : null); - }, 3000); - - //console.log('[cluster2] getStatus:' + component + ';event:' + event + ';expects:' + expects); - emitter.on(event, collect); - //console.log('[cluster2] expects:' + event); - emitter.emit('update-component-status', component, now, params, value); - - return this; -}; - -//default single process emitter is the process itself -var emitter = { - - 'handlers': {}, - - 'emit': function(){ - process.emit.apply(process, arguments); - }, - - 'on': function(event, handler){ - if(!emitter.handlers[event]){ - emitter.handlers[event] = []; - process.on(event, function(){ - var params = arguments; - _.each(emitter.handlers[event], function(handler){ - handler.apply(null, params); - }); - }); - } - emitter.handlers[event].push(handler); - }, - - 'removeListener': function(event, handler){ - emitter.handlers[event] = _.without(emitter.handlers[event] || [], handler); - } -}; - -//overwrite the emitter in case the process is the master of a cluster -if(process.cluster && process.cluster.clustered){ - var workers = process.cluster.workers; - emitter = { - 'handlers': { - - }, - - 'emit': function(){ - var args = _.toArray(arguments), - event = args.shift(); - - //console.log('[cluster2] master emits:' + event + ':\n' + JSON.stringify(args) + ':\nto workers:' + _.keys(workers).length); - _.each(_.values(workers), function(worker){ - try{ - worker.send({ - type: event, - params: args - }); - } - catch(error){ - console.log('[cluster2] master sending to worker failed: ' + error); - } - }); - - process.emit.apply(process, arguments); - }, - - 'on': function(event, handler){ - - emitter.handlers[event] = emitter.handlers[event] || []; - emitter.handlers[event].push(handler); - - _.each(_.values(workers), function(worker){ - if(!worker.clusterEventHandler){ - worker.clusterEventHandler = function(message){ - _.invoke(emitter.handlers[message.type] || [], 'apply', null, message.params); - }; - - worker.on('message', worker.clusterEventHandler); - } - }); - - if(emitter.handlers[event].length === 1){ - process.on(event, function(){ - _.invoke(emitter.handlers[event] || [], 'apply', null, arguments); - }); - } - }, - - 'removeListener': function(event, handler){ - - emitter.handlers[event] = _.without(emitter.handlers[event] || [], handler); - - if(emitter.handlers[event].length === 0){ - process.removeAllListeners(event); - } - } - }; - - process.cluster.emitter.on('listening', function(pid){ - //console.log('[cluster2] master found new worker, and will hook up with activated listener:' + pid); - var worker = workers[pid]; - if(!worker.clusterEventHandler){ - worker.clusterEventHandler = function(message){ - _.invoke(emitter.handlers[message.type] || [], 'apply', null, message.params); - }; - - worker.on('message', worker.clusterEventHandler); - } - }); - - process.cluster.emitter.on('died', function(pid){ - //bugfix of slow vi, as some worker died, and we didn't reduce the expectations. - process.emit.apply(process, ['worker-died', pid]); - }); -} - -if(cluster.isWorker){ - emitter = { - 'handlers': {}, - - 'emit': function(){ - - var args = _.toArray(arguments), - event = args.shift(); - - process.send({ - type: event, - params: args - }); - }, - - 'on': function(event, handler){ - this.handlers[event] = this.handlers[event] || []; - this.handlers[event].push(handler); - }, - - 'removeListener': function(event, handler){ - this.handlers[event] = _.without(this.handlers[event] || [], handler); - } - }; - - process.on('message', function(message){ - _.each(emitter.handlers[message.type] || [], function(h){ - h.apply(null, message.params); - }); - }); -} - -exports.componentStatus = new ComponentStatus(emitter); diff --git a/lib/ecv.js b/lib/ecv.js index 556e3a2..809063b 100644 --- a/lib/ecv.js +++ b/lib/ecv.js @@ -1,170 +1,88 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -"use strict"; - -var http = require('http'), - _ = require('underscore'), - os = require('os'); - -/** - * The ECV check sends a "/tables" request to the running server. Anything other than a valid JSON response is - * treated as an error. - */ -var hostname = os.hostname(); - -exports.enable = function(apps, options, emitter, validator) { - var path = options.ecv.path || '/ecv'; - var monitor = options.ecv.monitor || undefined; - var control = options.ecv.control || false; - var root = path || '/ecv'; - var disabled; - - _.each(apps, function (app) { - if (!_.isFunction(app.app.get)) { // this looks to be tcp server ... ecv is app's responsibility! - return; - } - app.app.get(root, function (req, res) { - var tosend = { - date:new Date, - port:_.isArray(options.port) ? options.port[0] : options.port - }; - if (app.disabled) { - // Drop the ball - away(req, res, tosend); - return; - } - var coptions = { - host:'localhost', - port:_.isArray(options.port) ? options.port[0] : options.port, - path:monitor || '/', - method:'GET', - headers:{ - host:'localhost', - connection:'close', - accept:'application/json' - } - }; - var creq = http.request(coptions, function (cres) { - cres.setEncoding('utf8'); - var data = ''; - cres.on('data', function (chunk) { - data = data + chunk; - }); - - cres.on('end', function () { - if (cres.statusCode >= 300) { - // Not happy - unhappy(req, res, tosend); - } - else { - try { - if (validator) { - validator.apply(this, [res.status, res.headers, data]); - } - happy(req, res, tosend); - } - catch (e) { - // Not happy - unhappy(req, res, tosend); - } - } - }); - }); - creq.on('error', function (err) { - unhappy(req, res, tosend.date); - }); - creq.end(); - }); - - if (control === true) { - app.app.post(root + '/disable', function (req, res) { - app.disabled = true; - emitter.emit('warning', { - message:'Disable request received' - }); - if (process.send) { - process.send({ - command:'disable' - }); - } - res.writeHead(204, { - 'since':new Date(Date.now() - process.uptime() * 1000), - 'cache-control':'no-cache', - 'X-Powered-By':'Cluster2', - 'Connection':'close' - }); - res.end() - }); - - app.app.post(root + '/enable', function (req, res) { - app.disabled = false; - emitter.emit('warning', { - message:'Enable request received' - }); - if (process.send) { - process.send({ - command:'enable' - }); - } - res.writeHead(204); - res.end() - }); - - process.on('message', function (message) { - if (message && message.command) { - app.disabled = message.command === 'disable'; - } - }); - } - }); -}; - -function happy(req, res, tosend) { - res.writeHead(200, { - 'content-type': 'text/plain', - 'since': new Date(Date.now() - process.uptime()*1000), - 'cache-control': 'no-cache', - 'X-Powered-By': 'Cluster2', - 'Connection': 'close' - }); - res.write('status=AVAILABLE&ServeTraffic=true&ip='+ req.connection.address()['address'] +'&hostname='+ hostname +'&port=' + tosend.port+ '&time=' + tosend.date.toString()); - res.end(); -} +'use strict'; -function unhappy(req, res, tosend) { - res.writeHead(500, { - 'content-type': 'text/plain', - 'since': new Date(Date.now() - process.uptime()*1000), - 'cache-control': 'no-cache', - 'X-Powered-By': 'Cluster2', - 'Connection': 'close' - }); - res.write('status=WARNING&ServeTraffic=false&ip='+ req.connection.address()['address'] +'&hostname='+ hostname +'&port=' + tosend.port + '&time=' + tosend.date.toString()); - res.end(); -} +//ecv is redesigned to be a simpler middleware of the monitoring app +//ecv should show a consistent view of the entire cluster +//ecv should have a root path, which is what the ops requests will be +//ecv should have 2 modes, monitor vs. controlled +//monitor mode requires a monitor url, a validator which allows ecv to issue request and let validator checks and decide if the ecv should be positive or negative +//controled mode is simply an on/off switch based on 2 more url routes (enable/disable) +var request = require('request'), + assert = require('assert'); + +exports.enable = function enable(app, options){ + + var logger = process.getLogger(__filename), + mode = options.mode || 'control', //default is control + disabled === false, therefore, ecv is mark up + root = options.root, + positive = options.positive || function(req, res){ + res.send(200); + }, + negative = options.negative || function(req, res){ + res.send(500); + }; + + assert.ok(root); + + logger.info('[ecv] enabled in the mode:%s', mode); + + if('monitor' === mode){ + + var monitor = options.monitor, //monitor must be given, and it's expected in full url format + validator = options.validator || function(error, response, body){ + + return !error && response.statusCode < 400; + }; + + assert.ok(monitor); + + app.use(function(req, res, next){ + + if(req.url !== root){ + + next(); + } + else{ + request.get(monitor, function(error, response, body){ + + (validator(error, response, body) ? positive : negative)(req, res); + }); + } + }); + } + else if('control' === mode){ + + var emitter = options.emitter, + markUp = options.markUp, + markDown = options.markDown; + + app.ecv = {'disabled': options.disabled}; + + app.use(function(req, res, next){ + + if(req.url === markDown){ + + app.ecv.disabled = true; + emitter.to(['master']).emit('markDown'); + logger.info('[ecv] traffic disabled'); + + negative(req, res); + } + else if(req.url === markUp){ + + app.ecv.disabled = false; + emitter.to(['master']).emit('markUp'); + logger.info('[ecv] traffic enabled'); + + positive(req, res); + } + else if(req.url !== root){ -function away(req, res, tosend) { - res.writeHead(400, { - 'content-type': 'text/plain', - 'since': new Date(Date.now() - process.uptime()*1000), - 'cache-control': 'no-cache', - 'X-Powered-By': 'Cluster2', - 'Connection': 'close' - }); - res.write('status=DISABLED&ServeTraffic=false&ip='+ req.connection.address()['address'] +'&hostname='+ hostname +'&port=' + tosend.port + '&time=' + tosend.date.toString()); - res.end(); + next(); + } + else{ + //just to tell if ecv is on/off + (app.ecv.disabled ? negative : positive)(req, res); + } + }); + } } diff --git a/lib/emitter.js b/lib/emitter.js new file mode 100644 index 0000000..7b111a8 --- /dev/null +++ b/lib/emitter.js @@ -0,0 +1,246 @@ +//cluster-emitter is a different EventEmitter which allows all messages send between master & slaves, with an additional parameters as 'target' +//the rest of the api should be the same as EventEmitter, so 'on', 'once', 'removeListener', 'removeAllListeners', 'emit' +'use strict'; + +var cluster = require('cluster'), + _ = require('underscore'); + +var masterEmitter = {//masterEmitter is the emitter variation in master process + + get logger(){ + return process.getLogger(__filename); + }, + + get workers(){//return active workers + + return cluster.workers; + }, + + 'handlers': { + //a map of event handlers + }, + + 'on': function(event, handler, duplicateAllowed){ + + var _this = this; + _this.handlers[event] = _this.handlers[event] || []; + + if(!_.contains(_this.handlers, handler) || duplicateAllowed){//avoid duplicates + _this.handlers[event].push(handler); + } + + _.each(_this.workers, function(worker){ + + if(!worker.clusterEventHandler){ + //this is the 1st time a handler is registered, must register 'message' handler on all workers + worker.clusterEventHandler = function(message){ + _.invoke(_this.handlers[message.type] || [], 'apply', null, message.params); + }; + + worker.on('message', worker.clusterEventHandler); + } + }); + + if(_this.handlers[event].length === 1){ + //this is the 1st time a handler of this event is registered in master process + process.on(event, function(){ + _.invoke(_this.handlers[event] || [], 'apply', null, arguments); + }); + } + }, + + 'once': function(event, handler){ + + var _this = this, + handleOnce = function handleOnce(){ + //wrap handler, and remove itself immediately after + handler.apply(null, arguments); + _this.removeListener(event, handleOnce); + }; + + _this.on(event, handleOnce); + }, + + 'emit': function(){ + + var _this = this, + pids = _.map(_this.workers, function(worker){//map all workers' pids + return worker.process.pid; + }); + + return _this.emitTo(pids.concat(['self'])/*plus 'self'*/, arguments); + }, + + 'to': function(targets){ + + var _this = this; + + return { + 'emit': function(){ + //set audience to @param targets + return _this.emitTo(targets, arguments); + } + }; + }, + + 'emitTo': function(targets, args){ + + args = _.toArray(args || []); + + var _this = this, + event = args.shift(), + audience = _.filter(_this.workers, function(worker){ + //filter workers, include only those in the @param targets + return _.contains(targets, worker.process.pid); + }); + + _.each(audience, function(worker){ + try{ + worker.send({ + 'type': event, + 'params': args + }); + } + catch(error){ + _this.logger.warn('[cluster2] master sending to worker failed:%j', error); + } + }); + + if(_.contains(targets, 'master') || _.contains(targets, 'self') || _.contains(targets, process.pid)){ + //audience including 'master' itself + args.unshift(event); + process.emit.apply(process, args); + } + }, + + 'removeListener': function(event, handler){ + + var _this = this; + _this.handlers[event] = _.without(_this.handlers[event] || [], handler); + + if(_this.handlers[event].length === 0){ + process.removeAllListeners(event); + } + }, + + 'removeAllListeners': function(event){ + + this.handlers[event] = []; + process.removeAllListeners(event); + } + }, + //slaveEmitter is the emitter variation in all worker processes + slaveEmitter = { + + 'handlers': {}, + + 'on': function(event, handler){ + + var _this = this; + _this.handlers[event] = _this.handlers[event] || []; + _this.handlers[event].push(handler); + + if(_this.handlers[event].length === 1){ + //1st time this event handler is registered in worker process + process.on(event, function(){ + _.invoke(_this.handlers[event] || [], 'apply', null, arguments); + }); + } + }, + + 'once': function(event, handler){ + var _this = this, + handleOnce = function handleOnce(){ + handler.apply(null, arguments); + _this.removeListener(event, handleOnce); + }; + + _this.on(event, handleOnce); + }, + + 'emit': function(){ + //default audience being both 'master' & 'worker' itself + return this.emitTo(['master', 'self'], arguments); + }, + + 'to': function(targets){ + + var _this = this; + + return { + 'emit': function(){ + //set audience to @param targets + return _this.emitTo(targets, arguments); + } + }; + }, + + 'emitTo': function(targets, args){ + + args = _.toArray(args || []); + + var event = args.shift(); + + if(_.contains(targets, 'master')){ + //audience includes 'master' + process.send({ + 'type': event, + 'params': args + }); + } + + if(_.contains(targets, 'self') || _.contains(targets, process.pid)){ + //audience includes 'worker' itself + args.unshift(event); + process.emit.apply(process, args); + } + }, + + 'removeListener': function(event, handler){ + + var _this = this; + _this.handlers[event] = _.without(_this.handlers[event] || [], handler); + + if(_this.handlers[event].length === 0){ + process.removeAllListeners(event); + } + }, + + 'removeAllListeners': function(event){ + + this.handlers[event] = []; + process.removeAllListeners(event); + } + }; + +//export the proper emitter based on whose runtime this is, master vs. worker. +if(cluster.isMaster){ + + var emitter = module.exports = masterEmitter; + + //any newly forked worker must get a replication of the known event handlers + cluster.on('fork', function(worker){ + + if(!worker.clusterEventHandler){ + + worker.clusterEventHandler = function(message){ + _.invoke(emitter.handlers[message.type] || [], 'apply', null, message.params); + }; + + worker.on('message', worker.clusterEventHandler); + } + }); +} +else{ + + var emitter = module.exports = slaveEmitter; + + //enable listening to master's messages + process.on('message', function(message){ + + _.each(emitter.handlers[message.type] || [], function(h){ + h.apply(null, message.params); + }); + }); +} + diff --git a/lib/index.js b/lib/index.js deleted file mode 100644 index d308738..0000000 --- a/lib/index.js +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -var Process = require('./process.js'), - ecv = require('./ecv.js'), - _ = require('underscore'), - assert = require('assert'), - os = require('os'), - when = require('when'), - util = require('util'), - net = require('net'), - events = require('events'); - -// Trap all uncaught exception here. -process.on('uncaughtException', function (error) { - // TODO: This has to the log file - console.error(error.stack || error); -}); - -exports.version = require('../package.json').version; -exports.defaultOptions = { - cluster: true, - port: 3000, - monPort: 3001, - ecv: { - path: '/ecv' - }, - monPath: '/', - noWorkers: os.cpus().length -}; - -var Cluster = module.exports = function Cluster(options) { - // Extend from EventEmitter - events.EventEmitter.call(this); - - this.options = {}; - _.extend(this.options, exports.defaultOptions); - _.extend(this.options, options); - - assert.notEqual(this.options.port, this.options.monPort, "monitor port & application port cannot use the same!"); -} - -util.inherits(Cluster, events.EventEmitter); - -/** - * Start the cluster - */ -Cluster.prototype.listen = function(createApp, cb) { - - var self = this, - options = self.options; - - assert.ok(_.isFunction(createApp), 'createApp must be a function'); - - if(options.cluster) { - var master = new Process({ - pids: process.cwd() + '/pids', - logs: process.cwd() + '/logs', - port: options.port, - host: options.host || '0.0.0.0', - monPort: options.monPort, - monHost: options.monHost || '0.0.0.0', - monPath: options.monPath, - ecv: options.ecv, - noWorkers: options.noWorkers, - timeout: options.timeout || 30 * 1000, // idle socket timeout - connThreshold: options.connThreshold || 10000, // recycle workers after this many connections - uptimeThreshold: options.uptimeThreshold || 3600 * 24, // 24 hours (uptimeThreshold is in seconds) - heartbeatInterval: options.heartbeatInterval, - maxHeartbeatDelay: options.maxHeartbeatDelay, - emitter: self - }); - - if(options.stop) { - master.stop() - } - else if(options.shutdown) { - master.shutdown(); - } - else { - initApp(function (app, monApp) { - master.listen(app, monApp, function () { - if(options.ecv) { - ecv.enable(app, options, self, function (data) { - return true; - }); - } - if(cb) { - cb(app, monApp); - } - }); - }); - } - } - else { - // Temp Fix to unblock tech talk demo - var ports = _.isArray(options.port) ? options.port : [options.port]; - if (ports.length > 1) { - console.log('Provide a single port for non-cluster mode. Exiting.'); - process.exit(-1); - } - - createApp.call(null, function (app, monApp) { - //adding monApp to none-cluster mode - var Monitor = require('./monitor.js'), - monitor = new Monitor({ - monitor: options.monitor || monApp, - stats: self.stats, - host: options.monHost, - port: options.monPort, - path: options.monPath - }); - - monitor.listen(options.monPort, options.host); - - app.listen(ports[0], options.host, function () { - if (options.ecv) { - //bugfix by huzhou@ebay.com, in cluster=false mode, ecv failed because of wrong params, should use array of 'app':app object - ecv.enable([{'app':app}], options, self, function (data) { - return true; - }); - } - if (cb) { - cb(app, monitor); - } - }); - - //register the master worker itself, as it doesn't go through master process creation - var componentStatus = self.componentStatusResolved = require('./component-status.js').componentStatus; - componentStatus.register('worker', function(){ - return 'm' + process.pid; - }, 'array'); - - self.emit('component-status-initialized', componentStatus); - }); - } - - function initApp(cb) { - createApp.call(null, function (app, monApp) { - // If the port is already occupied, this will exit to prevent node workers from multiple - // masters hanging around together - var ports = _.isArray(app) - ? _.reduce(app, - function(ports, anApp){ - return ports.concat(anApp.port && anApp.app - ? _.isArray(anApp.port) ? anApp.port : [anApp.port] - : []); - }, - []) - : _.isArray(options.port) ? options.port : [options.port]; - - exitIfBusyPort(options.host, ports, ports.length - 1, function(){ - cb(_.filter(_.isArray(app) ? app : [{app: app, port: options.port}], - function(app){ - return app.app && app.port; - }), monApp); - }); - }); - } - - function exitIfBusyPort(host, port, index, cb) { - if(index < 0) { - return cb(); - } - var server = net.createServer(); - server.on('error', function (e) { - if(e.code === 'EADDRINUSE') { - console.error('Port is use ..' + port[index]); - process.exit(-1); - } - }); - if (require('cluster').isMaster) { - server.listen(port[index], host, function() { //'listening' listener - exitIfBusyPort(host, port, index-1, function(){ - server.close(); - cb(); - }) - }); - } - else { - process.nextTick(cb); - } - } - - return self; -}; - -Cluster.prototype.componentStatus = function(){ - - if(!this.componentStatusPromise){ - - var componentStatusDeferred = when.defer(); - this.componentStatusPromise = componentStatusDeferred.promise; - - if(!this.componentStatusResolved){ - this.once('component-status-initialized', function(componentStatus){ - componentStatusDeferred.resolve(componentStatus); - }); - } - else{ - componentStatusDeferred.resolve(this.componentStatusResolved); - } - } - - return this.componentStatusPromise; -}; - -Cluster.prototype.stop = function () { - var master = new Process({ - pids: process.cwd() + '/pids' - }); - master.stop(); -}; - -Cluster.prototype.shutdown = function () { - var master = new Process({ - pids: process.cwd() + '/pids' - }); - master.shutdown(); -}; diff --git a/lib/main.js b/lib/main.js new file mode 100644 index 0000000..ddab29b --- /dev/null +++ b/lib/main.js @@ -0,0 +1,265 @@ +/* + * Copyright 2012 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var readMasterPid = require('./utils').readMasterPid, + getLogger = require('./utils').getLogger, + argv = require('optimist').argv, + when = require('when'), + path = require('path'), + os = require('os'), + _ = require('underscore'), + assert = require('assert'); + +/** + * @return promise of Server starts + * + * listen will create the Master process, and delegate to its#listen + * it gives back a promise for the master's server to be started (monitoring application) + * + * the resolved promise will always give: + * { + * 'server': server, + * 'app': app, + * 'port': port, + * 'master': master, + * 'worker': worker + * } + * + * if cluster.isMaster === true, the promise will be resolved as monCreateServer, monApp, monPort, master, null + * otherwise, the promise instead will be resolved as createServer, app, port, null, worker + */ +module.exports = { + + /** + * + * @param options + * @return promise + * + * options is a map contains almost every configurable aspect of cluster. + *
    + *
  • emitter: the event driver of the entire cluster, usually being ./emitter, could be overwritten for test or other purposes
    • + *
  • noWorkers: the number of workers the cluster will spawn, default being from the value of '--noWorkers' in argv, otherwise the number of cpu cores on the machine
    • + *
  • monCreateServer: a function which creates the server for monitor application, default being http#createServer
    • + *
  • monApp: an express application to allow monitoring purpose of the entire cluster, default being ./monitor, which supports 'debug', 'ecv' features etc.
    • + *
  • monPort: the port to run the monitor application on, default being from the value of '--monPort' in argv, otherwise 8081
    • + *
  • monConfigureApp: a function to adapt the monitor application before monitor application starts, default doing nothing
    • + *
  • createServer: a function which creates the server for user application (running in workers), default beging http#createServer
    • + *
  • app: the user application to run across all workers, must be provided
    • + *
  • port: the port to run the user application on, default being from the value of '--port' in argv, otherwise 8080
    • + *
  • configureApp: a function to adapt the user application, usually to register all routes, middlewares, default doing nothing
    • + *
  • warmUp: a function to run after the user application started listening, usually to hit some routes to load caches, templates, resources for performance sake.
    • + *
  • shouldKill: a function to determine if a worker is going wild, and must be terminated (then replaced) based on collected heartbeat stats, default being a combination of ./utils#assertOld & ./utils#assertBadGC
    • + *
  • stopTimeout: an integer of MS to wait before further action upon a worker to be disconnected, default being from the value of '--stop-timeout' in argv, otherwise 1 minute
    • + *
  • ecv.root: a string for the url route ecv will be using in monitor application, default being from the value of '--ecv.root' in argv, otherwise '/ecv'
    • + *
  • ecv.mode: one of ['control', 'monitor'] options, default being from the value of '--ecv.mode' in argv, otherwise 'control'
    • + *
  • ecv.disable: a boolean value used in 'control' mode, telling whether ecv should start as 'disabled' or 'enabled', default being from the value of '--ecv.disable' in argv, otherwise true
    • + *
  • ecv.markUp: a string for the url route ecv will accept to 'enable' the traffic in 'control' mode, default beging from the value of '--ecv.mark-up' in argv, otherwise '/ecv/enable'
    • + *
  • ecv.markDown: a string for the url route ecv will accept to 'disable' the traffic in 'control' mode, default being from the value of '--ecv.mark-down' in argv, otherwise '/ecv/disable'
    • + *
  • ecv.monitor: a string for the url to be tested in 'monitor' mode, default being the value of '--ecv.monitor' in argv, otherwise '/'
    • + *
  • ecv.validator: a function to validate the http response from hitting the monitor url given, default being checking status code equal 200
    • + *
  • debug.debugPort: an integer of the port for node-inspector to connect to the user application process, default being from the value of '--debug.debug-port' in argv, otherwise 5858
    • + *
  • debug.webPort: an integer of port to run the node-inspector server, default being from the value of '--debug.web-port' in argv, otherwise 8082
    • + *
  • debug.saveLiveEdit: a boolean value whether or not allowing node-inspector's change should be saved to the file, default being from the value of '--debug.save-live-edit' in argv, otherwise false
    • + *
  • cache.enable: a boolean value whether or not caching service should be started for the cluster, default being from the value of '--cache.enable', otherwise true
    • + *
  • cache.mode: one of ['standalone', 'master'] options, which tells the caching service to be run as a process or within master, default being from the value of '--cache.mode' in argv, otherwise 'standalone'
    • + *
  • gc.monitor: a boolean value whether or not to monitor the GC stats of each worker, default being the value of '--trace-gc' in argv, otherwise true
    • + *
  • gc.idleNotification: a boolean value whether or not node should send 'idle' notification to v8, default being the opposite to the value of '--nouse-idle-notification', otherwise false
    • + *
  • nanny.enable: a boolean value whether or not nanny monitoring is needed, default being the value of '--nanny.enable' in argv, otherwise true
    • + *
  • nanny.tolerance: an integer of MS to wait from action upon suspicious run away workers, default being the value of '--nanny.tolerance' in argv, otherwise 3 mins
    • + *
  • maxAge: an integer of seconds to determine that a worker is to be terminated, used by ./utils#assertOld, default being the value of '--max-age' in argv, otherwise 3 days
    • + *
  • heartbeatInterval: an integer of MS for each worker to report its health, default being the value of '--heartbeat-interval' in argv, otherwise 1 min
    • + *
+ */ + 'listen': function listen(options) { + + exports.getLogger = process.getLogger = options.getLogger || getLogger; + + assert.ok(process.getLogger); + + // Trap all uncaught exception here. + process.on('uncaughtException', function (error) { + + process.getLogger(__filename).info(error.stack || error); + }); + + var emitter = require('./emitter'), + actualOptions = { + 'pids': argv.pids || path.join(process.cwd(), '/pids'), + 'emitter': emitter, + 'noWorkers': argv.noWorkers || os.cpus().length, + 'monCreateServer': require('./monitor').monCreateServer, + 'monApp': require('./monitor').monApp, + 'monPort': argv.monPort || 8081, + 'warmUpPort': argv.warmUpPort || 8083, + 'monConfigureApp': function(monApp){ + + return monApp; + }, + 'createServer': require('http').createServer, + 'configureApp': function(app){ //happens before server listen + + return app; + }, + 'app': null,//must be provided by the options + 'port': argv.port || 8080, + 'warmUp': function(app, address){ //happens after server listen + + return app; + }, + 'shouldKill': null, //should be a function determine if worker is old or not + 'stopTimeout': argv['stop-timeout'] || 60000,//1 min + 'debug': { + 'webPort': argv['debug.web-port'] || 8082, + 'debugPort': argv['debug.debug-port'] || 5858, + 'saveLiveEdit': argv['debug.save-live-edit'] || false, + 'hidden':[] + }, + 'ecv': { + 'root': argv['ecv.root'] || '/ecv', + 'mode': argv['ecv.mode'] || 'control', + 'disable': argv['ecv.disable'] || true, + 'markUp': argv['ecv.mark-up'] || '/ecv/enable', + 'markDown': argv['ecv.mark-down'] || '/ecv/disable', + 'monitor': argv['ecv.monitor'] || '/', + 'validator': function(err, response, body){ + return !err && response && response.statusCode === 200; + }, + 'emitter': emitter + }, + 'cache': { + 'enable': argv['cache.enable'] || true, + 'mode': argv['cache.mode'] || 'standalone', //a process will be allocated dedicated to it, otherwise crush that into master + 'domainPath': argv['cache.domain-path'] || './cluster-cache-domain', + 'persistPath': argv['cache.persist-path'] || './cluster-cache-persist' + }, + 'gc': { + 'monitor': argv['trace-gc'] || true, + 'idleNotification': !argv['nouse-idle-notification'] //idle notification by default, unless explicitly set in the argv + }, + 'nanny': { + 'enable': argv['nanny.enable'] || true, + 'tolerance': argv['nanny.tolerance'] || 60000 * 3 + }, + 'maxAge': argv['max-age'] || 60 * 60 * 24 * 3, //how long could a worker keep running, default will be 3 days + 'heartbeatInterval': argv['heartbeat-interval'] || 5000 + }; + + //apply highest precendence of configurations from @param options + _.extend(actualOptions.ecv, options.ecv || {}); + _.extend(actualOptions.debug, options.debug || {}); + _.extend(actualOptions.cache, options.cache || {}); + _.extend(actualOptions.gc, options.gc || {}); + _.extend(actualOptions, _.omit(options, 'ecv', 'debug', 'cache', 'gc')); + + //validate all required parameters were given + assert.ok(actualOptions.monCreateServer); + assert.ok(actualOptions.monApp); + assert.ok(actualOptions.monPort); + assert.ok(actualOptions.createServer); + assert.ok(actualOptions.app); + assert.ok(actualOptions.port); + assert.notEqual(actualOptions.port, actualOptions.monPort, "monitor port & application port cannot use the same!"); + assert.notEqual(actualOptions.port, actualOptions.debug.debugPort, "debug port & application port cannot use the same!"); + assert.notEqual(actualOptions.monPort, actualOptions.debug.debugPort, "monitor port & debug port cannot use the same!"); + + var Master = require('./master').Master; + + return new Master(process, actualOptions).listen(); + }, + + /** + * experimental support to run anything, no need to be a server + */ + 'run': function run(options){ + + exports.getLogger = process.getLogger = options.getLogger || getLogger; + + assert.ok(process.getLogger); + + // Trap all uncaught exception here. + process.on('uncaughtException', function (error) { + + process.getLogger(__filename).error(error.stack || error); + }); + + var emitter = require('./emitter'), + actualOptions = { + 'pids': argv.pids || path.join(process.cwd(), '/pids'), + 'emitter': emitter, + 'noWorkers': argv.noWorkers || os.cpus().length, + 'monCreateServer': require('./monitor').monCreateServer, + 'monConfigureApp': function(monApp){ + + return monApp; + }, + 'monApp': require('./monitor').monApp, + 'monPort': argv.monPort || 8081, + 'runnable': options.runnable, //runnable is a function to be called without scope (closure) + 'warmUp': function(runnable){ + return runnable; + }, + 'shouldKill': null, //should be a function determine if worker is old or not + 'stopTimeout': 60000,//1 min + 'debug': { + 'webPort': argv['debug.webPort'] || 8082, + 'debugPort': argv['debug.debugPort'] || 5858, + 'saveLiveEdit': argv.saveLiveEdit || false, + 'hidden':[] + }, + 'cache': { + 'enable': argv['cache.enable'] || true, + 'mode': argv['cache.mode'] || 'standalone' //a process will be allocated dedicated to it, otherwise crush that into master + }, + 'gc': { + 'monitor': argv['trace-gc'] || true, + 'idle-notification': !argv['nouse-idle-notification'] || false + }, + 'nanny': { + 'enable': argv['nanny.enable'] || true, + 'tolerance': argv['nanny.tolerance'] || 60000 * 3 + }, + 'maxAge': argv['max-age'] || 0,//how long could a worker keep running, default will be 3 days + 'heartbeatInterval': argv['heartbeat.interval'] || 5000 + }; + + _.extend(actualOptions.debug, options.debug || {}); + _.extend(actualOptions.cache, options.cache || {}); + _.extend(actualOptions.gc, options.gc || {}); + _.extend(actualOptions, _.omit(options, 'debug', 'cache', 'gc')); + + assert.ok(actualOptions.monCreateServer); + assert.ok(actualOptions.monApp); + assert.ok(actualOptions.monPort); + assert.ok(actualOptions.runnable); + assert.notEqual(actualOptions.monPort, actualOptions.debug.debugPort, "monitor port & debug port cannot use the same!"); + + var Master = require('./master').Master; + + return new Master(process, actualOptions).run(); + }, + + get monitor(){ + + return require('./monitor'); + }, + + get ecv(){ + + return require('./ecv'); + } +}; diff --git a/lib/master.js b/lib/master.js new file mode 100644 index 0000000..e8f56ec --- /dev/null +++ b/lib/master.js @@ -0,0 +1,559 @@ +'use strict'; + +var _ = require('underscore'), + util = require('util'), + when = require('when'), + pipeline = require('when/pipeline'), + timeout = require('when/timeout'), + utils = require('./utils'), + assert = require('assert'), + cluster = require('cluster'), + request = require('request'), + Puppet = require('./puppet').Puppet; + +var tillMaster = when.defer(); + +exports.master = tillMaster.promise; + +var Master = exports.Master = function(proc, options){ + + return this.initialize(proc, options); +}; + +Master.prototype.initialize = function initialize(proc, options){ + + var _this = this, + pids = options.pids; + + utils.writePid(process.pid, pids); + + if(cluster.isMaster) { + + _.extend(_this, { + 'pid': proc.pid, + 'process': proc, + 'emitter': utils.decorateEmitter(options.emitter), + 'logger': process.getLogger(__filename), + 'runnable': options.runnable, + 'createServer': options.monCreateServer, + 'app': options.monApp, + 'port': options.monPort, + 'warmUpPort': options.warmUpPort, + 'ecv': options.ecv, + 'cache': options.cache, + 'gc': options.gc, + 'configureApp': function(monApp){ + + assert.ok(_this.createServer); + assert.ok(_this.app); + assert.ok(_this.port); + + //cache is 1st enabled, ahead of any worker process, ahead of master's monitor app configuration + //to allow cache service started earlier than where it's required. + require('./cache').enable(options.cache, _this); + + require('./ecv').enable(monApp, options.ecv); + + require('./status').register('deathQueue', function(){ + return _this.deathQueue || []; + }); + + //monitor app could also be configured now, should return a value or promise. + return when.all([ + utils.rejectIfPortBusy('localhost', options.port), + utils.rejectIfPortBusy('localhost', options.monPort), + utils.rejectIfPortBusy('localhost', options.warmUpPort), + utils.rejectIfPortBusy('localhost', _this['debug.debugPort']), + utils.rejectIfPortBusy('localhost', _this['debug.webPort']) + ]) + .then( + function (ports) { + + console.log('selected ports approved'); + return options.monConfigureApp(monApp); + }, + function (error) { + + console.log('ports rejected'); + + _this.logger.error('[master] one of the ports:%j we need has been occupied, please shutdown your program on that port; error:%j', [ + options.port, + options.monPort, + _this['debug.debugPort'], + _this['debug.webPort'], + error + ]); + + process.exit(-1); + }); + }, + 'warmUp': function(){ + + if(!_this.gc.idleNotification && !_.contains(process.argv, '--nouse-idle-notification')){ + //the ugly way to force --nouse-idle-notification is actually to modify process.argv directly in master + var argv = _.toArray(process.argv), + node = argv.shift(); + argv.unshift('--nouse-idle-notification'); + argv.unshift(node); + + process.argv = argv; + //cluster#setupMaster or 'settings' could only handle execArgv, and won't help with '--nouse-idle-notification' + } + + //wait for all puppets to be ready, and then enable ECV as the last step + return pipeline([ + utils.pickAvailablePorts, + function(warmUpPorts){ + + return _.map(_.range(0, _this.noWorkers), function(ith){ + + return _this.fork(_this.options, { + 'warmUpPort': warmUpPorts.shift() + }); + }); + }, + function(){ + + var pids = _.map(_this.puppets, function(p){ + return p.pid; + }); + + return utils.markUpAfterAllListening(_this.emitter, pids) + .then(function(){ + _this.markUp(); + }) + .otherwise(function(error){ + _this.logger.info('[master] got warmup error from some worker:%j, please check and mark up', error); + }); + } + ], options.warmUpPort, options.warmUpPort + _this.noWorkers * 3, _this.noWorkers); + }, + 'noWorkers': options.noWorkers, + 'shouldKill': options.shouldKill || (function(){ + + var assertions = [utils.assertOld(options.maxAge), utils.assertBadGC()]; + + return function(heartbeat){ + + return _.some(assertions, function(a){ + + return a(heartbeat); + }); + }; + + })(), + 'stopTimeout': options.stopTimeout, + 'debug.debugPort': options.debug.debugPort, + 'debug.webPort': options.debug.webPort, + 'debug.saveLiveEdit': options.debug.saveLiveEdit, + 'debug.hidden': options.debug.hidden || [], + 'options': options, + 'puppets': { + + }, + 'deathQueue': [], + 'status': require('./status'), + 'isMaster': true, + 'isWorker': false + }); + + if(options.nanny && options.nanny.enable){ + _this.nannyScheduler = setInterval( + _.bind(utils.nanny, _this.puppets, _this.deathQueue, _this.emitter, function(){ + _this.fork(options, proc.env); + }, + { + 'logger': _this.logger, + 'tolerance': options.nanny.tolerance + }), + options.heartbeatInterval); + } + + cluster.on('fork', function(worker){//interestingly, on fork or #fork could complete in disorder... + + _this.puppets[worker.process.pid] = _this.puppets[worker.process.pid] + || new Puppet(_this, worker, options, worker.process.env || {}); + + _this.puppets[worker.process.pid].worker = worker; + }); + + cluster.on('online', function(worker){ + _this.puppets[worker.process.pid].whenOnline(); + }); + + cluster.on('listening', function(worker, address){ + _this.puppets[worker.process.pid].whenListening(address); + }); + + cluster.on('exit', function(worker){ + + var deadPid = worker.process.pid; + //exit puppet + _this.puppets[deadPid].whenExit(); + //mark dead worker's pid in /pids path + utils.markDeadPid(deadPid, pids); + }); + + _this.emitter.on('heartbeat', function(heartbeat){ + _this.puppets[heartbeat.pid].whenHeartbeat(heartbeat); + }); + + _this.emitter.on('dismiss', function(pid){ + _this.puppets[pid].dismiss(); + }); + + process.once('SIGINT', _.bind(_this.whenStop, _this)); + process.once('SIGTERM', _.bind(_this.whenExit, _this)); + + tillMaster.resolve(_this); + + return _this; + } + else if(proc.env && proc.env.CACHE_MANAGER){ + + var CacheMgrWorker = require('./cache-mgr-worker').CacheMgrWorker, + manager = require('./cache-mgr'); + + _.extend(options, { + 'createServer': manager.createServer, + 'app': manager.app, + 'port': manager.port, + 'warmUp': manager.afterServerStarted, + 'debug': false, + 'CACHE_DOMAIN_PATH': options.domainPath, + 'CACHE_PERSIST_PATH': options.persistPath, + 'maxAge': null + }); + + tillMaster.reject(new Error('this is cache manager worker')); + + return new CacheMgrWorker(proc, options); + } + else{ + + var Worker = require('./worker').Worker; + + _.extend(options, { + 'debug': process.env.debug, + 'CACHE_DOMAIN_PATH': options.domainPath, + 'CACHE_PERSIST_PATH': options.persistPath + }); + + tillMaster.reject(new Error('this is worker')); + + return new Worker(proc, options); + } +}; + +Master.prototype.listen = function listen(){ + + var _this = this, + createServer = _this.createServer, + monApp = _this.app, + monPort = _this.port, + configureApp = _this.configureApp, + warmUp = _this.warmUp, + wait = _this.timeout; + + assert.ok(createServer); + assert.ok(monApp); + assert.ok(monPort), + assert.ok(configureApp); + assert.ok(warmUp); + + var tillListen = when.defer(); + + when(configureApp(monApp)).ensure(function(configured){ + + var server = createServer(monApp).listen(monPort, function(){ + + _this.logger.info('[master] warmUp'); + + when(warmUp(monApp, server.address())).ensure(function(){ + + _this.logger.info('[master] warmUp complete'); + + tillListen.resolve({ + 'server': server, + 'app': monApp, + 'port': monPort, + 'master': _this, + 'worker': null + }); + }); + }); + }); + + return (wait > 0 ? timeout(_this.timeout, tillListen.promise) : tillListen.promise); +}; + +Master.prototype.pause = function pause(pid){ + + if(!pid){ + return when.map(this.puppets, function(p){ + return p.pause(); + }); + } + else{ + return this.puppets[pid].pause(); + } +}; + +Master.prototype.resume = function resume(pid){ + + if(!pid){ + return when.map(this.puppets, function(p){ + return p.resume(); + }); + } + else{ + return this.puppets[pid].resume(); + } +}; + +Master.prototype.run = function run(){ + + var _this = this, + warmUp = _this.warmUp, + wait = _this.timeout; + + assert.ok(warmUp); + + require('./cache').enable(_this.cache, _this); + + var tillRun = when.defer(); + + when(warmUp()).ensure(function(warmedUp){ + + _this.logger.info('[master] warmUp complete: %j', warmedUp); + + tillRun.resolve({ + 'master': _this, + 'worker': null + }); + }); + + return (wait > 0 ? timeout(_this.timeout, tillRun.promise) : tillRun.promise); +}; + +Master.prototype.debug = function debug(pid){ + + var _this = this, + noWorkers = _this.noWorkers; + + pid = pid && _.isString(pid) ? parseInt(pid, 10) : null; + + _this.logger.info('[debug] master debugging %d, currently busy: %s', pid, _this.debugging); + + if(!_this.debugging){ + _this.debugging = true; + } + else{ + return;//cannot debug more than one worker + } + + _this.markDown(); + + //debug fresh + _.each(_this.puppets, function(puppet){ + + if(!puppet.cacheManager && pid !== puppet.pid){ + _this.logger.info('[master][debug] fresh, disconnecting:%j vs pid:%j', puppet.pid, pid); + puppet.dismiss(); + } + }); + + //send signal to the puppet + var pending = when.defer(), + ready = pending.promise; + + if(!pid){ + + _this.fork(_this.options, { + 'debug': true + }); + + pending.resolve(true); + } + else{ + + var debugging = _this.puppets[pid]; + + _this.logger.info('[master][debug] going to debug live:%d status:%s', pid, debugging !== null); + + ready = debugging.debug(); + + _this.emitter.once('debug-finished', function(){ + + debugging.dismiss(); + }); + } + + ready.then(function(){//must wait for the signal to be sent, and child process ready to accept connection on 5858 + + _this.logger.info('[master][debug] starting inspector'); + + var inspector = utils.startInspector(_this['debug.webPort'], + _this['debug.debugPort'], + _this['debug.saveLiveEdit'], + _this['debug.hidden'], + _this.logger); + + inspector.on('message', function inspectorListener(msg){ + + if(msg.event === 'SERVER.LISTENING'){ + + var inspectorUrl = msg.address.url; + _this.logger.debug('[master][debug] got inspector url:%s', inspectorUrl); + _this.emitter.emit('debug-inspector', inspectorUrl); + + inspector.removeListener('message', inspectorListener); + } + }); + + _this.emitter.once('debug-finished', function(){//u'll hear back from the debug app about this + + if(_this.debugging === true){ + _this.debugging = false; + + inspector.kill('SIGTERM'); + //restore the workers + _.each(_.range(0, noWorkers), function(ith){ + _this.fork(_this.options, {}); + }); + + //mark up ecv now + _this.markUp(); + } + }); + }) + .otherwise(function(){ + + _this.logger.error('[debug] master did not get worker to debug mode:%d, debug request aborted', pid); + }); +}; + +Master.prototype.markDown = function markDown(){ + + var _this = this, + tillMarkDowm = when.defer(); + //mark down ecv + if(_this.ecv.mode === 'control'){ + _this.logger.info('[master][ecv] marking down'); + request.get(util.format('http://127.0.0.1:%d/%s', _this.port, _this.ecv.markDown), function(err, response, body){ + if(!err){ + tillMarkDowm.resolve({ + 'response': response, + 'body': body + }); + } + else{ + tillMarkDowm.reject(err); + } + }); + } + else{ + _this.logger.info('[master][ecv] markDown ignored, not in control mode'); + tillMarkDowm.reject(new Error('not in control mode')); + } + + return tillMarkDowm.promise; +}; + +Master.prototype.markUp = function markUp(){ + + var _this = this, + tillMarkUp = when.defer(); + //mark up ecv + if(_this.ecv.mode === 'control'){ + _this.logger.info('[master][ecv] marking up'); + request.get(util.format('http://127.0.0.1:%d/%s', _this.port, _this.ecv.markUp), function(err, response, body){ + if(!err){ + tillMarkUp.resolve({ + 'response': response, + 'body': body + }); + } + else{ + tillMarkUp.reject(err); + } + }); + } + else{ + _this.logger.info('[master][ecv] markUp ignored, not in control mode'); + tillMarkUp.reject(new Error('not in control mode')); + } + + return tillMarkUp.promise; +}; + +Master.prototype.fork = function fork(options, env){ + + var _this = this, + tillFork = when.defer(); + + if (env.CACHE_MANAGER) { + utils.pickAvailablePorts(_this.port + 10, _this.port + 30, 2).then(function (ports) { + _this.logger.info('[master] pick up ports %j for cache manager', ports); + env.CACHE_PORTS = JSON.stringify(ports); + tillFork.resolve(cluster.fork(env)); + }).otherwise(function (error) { + tillFork.reject(error); + }); + }else { + tillFork.resolve(cluster.fork(env)); + } + + return tillFork.promise.then(function (forked) { + //create and register the puppet managing this forked worker immediately + _this.puppets[forked.process.pid] = new Puppet(_this, forked, options, env); + }); +}; + +Master.prototype.beforeStop = function beforeStop(){ + + var _this = this; + + if(_this.nannyScheduler){ + //stop nanny monitoring, otherwise it will conflict with the worker decrease + clearInterval(_this.nannyScheduler); + } + + _this.emitter.emit('debug-finished');//kill inspector please + + _.each(_this.puppets, function(puppet){ + //disconnect each worker + puppet.dismiss(); + }); +}; + +Master.prototype.whenStop = function whenStop(){ + + var _this = this, + cycle = 1000, + deadline = Date.now() + _this.stopTimeout, + gracefully = function(){ + + if(_.keys(_this.puppets).length === 0){//wait till each live puppets to be disconnected, then exit + process.exit(0); + } + + if(Date.now() >= deadline){ + process.exit(-1); + } + else{ + setTimeout(gracefully, cycle); + } + };//check every second + + _this.beforeStop(); + + gracefully();//shutdown gracefully +}; + +Master.prototype.whenExit = function whenExit(){ + + this.beforeStop(); + + process.exit(-1);//shutdown bruteforcely +}; diff --git a/lib/misc.js b/lib/misc.js deleted file mode 100644 index f9ad691..0000000 --- a/lib/misc.js +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var fs = require('fs'), - assert = require('assert'), - when = require('when'), - timeout = require('when/timeout'); - -// Utility to ensure that certain directories exist -exports.ensureDir = function(dir, clean) { - try { - fs.readdirSync(dir); - if(clean) { - var paths = fs.readdirSync(dir); - paths.forEach(function(filename) { - try { - fs.unlink(dir + '/' + filename); - } - catch(e) {} - }); - } - } - catch(e) { - fs.mkdirSync(dir, 0755); - } -} - -exports.forMemoryNum = function(memory) { - var strMemory; - if(memory < 1024) { - strMemory = memory + ' Bytes'; - } - if(memory < 1024 * 1024) { - strMemory = (memory / 1024).toFixed(2) + ' KB'; - } - else { - strMemory = (memory / (1024 * 1024)).toFixed(2) + ' MB'; - } - return strMemory; -}; - - -var tillPrevDeath = null; - -exports.deathQueue = function deathQueue(worker, emitter, success){ - - assert.ok(worker); - assert.ok(emitter); - assert.ok(success); - - var tillDeath = when.defer(), - afterDeath = null, - die = function(){ - - var successor = success(), - workerPid = worker.pid, - expectPid = successor.pid; - - //when successor is in place, the old worker could be discontinued finally - emitter.on('listening', function onListen(onboard){ - - if(expectPid === onboard){ - - emitter.removeListener('listening', onListen); - - worker.kill('SIGINT'); - - emitter.on('died', function onDeath(death){ - - if(death === workerPid){ - - emitter.removeListener('died', onDeath); - - tillDeath.resolve(workerPid); - - if(tillPrevDeath === afterDeath){//last of dyingQueue resolved, clean up the dyingQueue - tillPrevDeath = null; - } - } - }); - } - - }); - }; - - if(!tillPrevDeath){ - //1st in the dying queue, - afterDeath = tillPrevDeath = timeout(60000, tillDeath.promise);//1 min timeout - die(); - } - else{ - //some one in the queue already, wait till prev death and then start `die` - afterDeath = tillPrevDeath = timeout(60000, tillPrevDeath.ensure(die)); - } -}; diff --git a/lib/monitor.js b/lib/monitor.js index 80ec76a..e339480 100644 --- a/lib/monitor.js +++ b/lib/monitor.js @@ -1,157 +1,334 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var misc = require('./misc.js'), - ejs = require('ejs'), - fs = require('fs'), - util = require('util'), - os = require('os'), - _ = require('underscore'); - -// Monitor -var Monitor = module.exports = function Monitor(options) { - this.options = options || {port: 8081, stats: {}, path: '/'}; - this.stats = this.options.stats; - - var app = options.monitor; - - if(!app){ - var express = require('express'); - app = express.createServer(); - - var self = this; - app.set('views', __dirname + '/../public/views'); - app.use(express.static(__dirname + '/../public')); - app.set('view engine', 'html'); - - app.get(this.options.path, function (req, res) { - var accept = (req.headers || {}).accept || ''; - if(accept.search('json') > 0) { - res.contentType('application/json'); - res.send(JSON.stringify(getStats(self.stats, req.connection))); - } - else { - res.render('index.ejs', getStats(self.stats, req.connection)); - } - }); - - app.get(/^\/logs?(?:\/(\d+)(?:\.\.(\d+))?)?/, function (req, res) { - var root, paths, logs, stats; - var file = process.cwd() + req.url; - if(req.url === '/logs') { - root = process.cwd() + '/logs'; - paths = fs.readdirSync(root); - logs = []; - paths.forEach(function (filename) { - stats = fs.statSync(root + '/' + filename); - logs.push({ - filename: filename, - stats: stats - }) - }); - var data = getStats(self.stats, req.connection); - data.logs = logs; - res.render('logs.ejs', data); - } - else { - var stat = fs.statSync(file); - res.writeHead(200, { - 'Content-Type': 'text/plain', - 'Content-Length': stat.size - }); - var readStream = fs.createReadStream(file); - util.pump(readStream, res, function (e) { - if(e) { - console.error(e.stack || e); - } - res.end(); - }); - } - }); - - app.get('/deps', function(req, res) { - var npm = require('npm'); - npm.load({}, function() { - npm.commands.ls([], true, function(e, data) { - res.writeHead(200, { - 'Content-Type': 'application/json' - }); - - var seen = []; - var out = JSON.stringify(data, function (k, o) { - if(typeof o === "object") { - if(-1 !== seen.indexOf(o)) return '[Circular]'; - seen.push(o); - } - return o; - }, 2); - res.end(out); - }); - }); - }); - - app.get('/ComponentStatus(/[^/]+)?', function(req, res){ - - //when component = empty, this will list all the component ids and links associated with them. - - //otherwise, we'll collect the component status from all workers - var component = req.query.component, //should come from param - view = req.query.view || 'html',//should come from param - worker = req.query.worker, - componentStatus = require("./component-status.js").componentStatus; - - if(component){ - componentStatus.getStatus(component, { - 'params': req.query, - 'worker': worker, - 'done': function(result){ - res.send(JSON.stringify(result), 200); - } - }); - } - else{ - res.send(JSON.stringify(componentStatus.getComponents()), 200); - } - }); - } - - return app; +'use strict'; + +var express = require('express'), + when = require('when'), + path = require('path'), + ejs = require('ejs'), + fs = require('graceful-fs'), + _ = require('underscore'); + +var logger = process.getLogger(__filename), + monApp = express(), + masterPromise = require('./master').master, + status = require('./status'); + +//the monitor app should support the following: + +//debug +var debugMiddleware = exports.debugMiddleware = function(req, res, next){ + + if(req.url === '/help'){ + + return res.sendfile(path.join(__dirname, './public/images/live-debugging.png')); + } + + if(req.url === '/deps'){ + + require('./utils').npmls + .then(function(deps){ + res.type('json').send(deps); + }) + .otherwise(function(error){ + res.send(500); + }); + + return; + } + + if(req.url !== '/debug'){ + + return next(); + } + + logger.debug('[monitor][index] renders'); + + res.render('index', { + + }); }; -function getStats(master, socket) { - master.hostname = os.hostname(); - master.os = os.type() + ' ' + os.release(); - master.averageLoad = os.loadavg().map( - function (n) { - return n.toFixed(2); - }).join(' '); - master.coresUsed = master.noWorkers + ' of ' + os.cpus().length; - master.memoryUsageAtBoot = misc.forMemoryNum(master.freemem) + ' of ' + - misc.forMemoryNum(master.totalmem); - master.totalMem = os.totalmem().toFixed(3); - master.currentMemoryUsage = (os.totalmem() - os.freemem()) / 1024000; - master.hostCpu = (_.reduce(os.cpus(), function (memo, cpu) { - return memo + (cpu.times.user / - (cpu.times.user + cpu.times.nice + - cpu.times.sys + cpu.times.idle + cpu.times.irq)); - }, 0) * 100 / os.cpus().length).toFixed(2); - - if(socket) { - master.address = socket.address(); - } - - return {master: master}; -} +monApp.use('/scripts', express.static(path.join(__dirname, './public/scripts'))); +monApp.use('/stylesheets', express.static(path.join(__dirname, './public/stylesheets'))); +monApp.engine('.ejs', require('ejs').__express); +monApp.set('views', path.join(__dirname, './views')); +monApp.set('view engine', 'ejs'); +monApp.use(debugMiddleware); + +exports.monApp = monApp; + +exports.monCreateServer = function createServer(app){ + + var server = require('http').createServer(app), + io = require('socket.io').listen(server, {'log': false}), + debugging = undefined, + inspector = null, + state = null, + statuses = {}, + sockets = [], + pauses = {};//all of these vars are now scoped in monCreateServer to make unique + + logger.info('[monitor] server started'); + + var workers = _.once(function tick(){ + + masterPromise.then(function(master){ + + var pids = _.keys(master.puppets); + + logger.debug('[monitor][workers] pids:%j', pids); + + when.map(master.status.statuses(), function(status){ + + return master.status.getStatus(status); + }) + .then(function(statusesOfWorkers){ + + logger.debug('[monitor][workers] statusesOfWorkers:%j', statusesOfWorkers); + + _.each(statusesOfWorkers, function(statusOfWorkers){ + + _.each(statusOfWorkers, function(statusOfWorker){ + + statuses[statusOfWorker.name] = statuses[statusOfWorker.name] || {}; + statuses[statusOfWorker.name][statusOfWorker.pid] = statusOfWorker.status; + }); + }); + + var arrOfStatues = _.map(statuses, function(v, k){ + v.name = k; + return v; + }); + + _.each(sockets, function(socket){ + + if(_.isEmpty(_.difference(socket.knownPids, pids)) && _.isEmpty(_.difference(pids, socket.knownPids))){ + + logger.debug('[monitor][workers] emits status changes:%j', arrOfStatues); + socket.emit('status-change', pids, arrOfStatues); + } + else{ + + logger.debug('[monitor][workers] detects workers changed:%j from:%j', pids, socket.knownPids); + socket.knownPids = pids;//update pids + + fs.readFile(path.join(__dirname, '/views/workers.ejs'), { + 'encoding': 'utf-8' + }, + function(err, read){ + + var html = ejs.render(read, { + 'pids': pids, + 'debugging': debugging, + 'inspector': inspector, + 'state': state, + 'statuses': arrOfStatues, + 'pauses': pauses + }); + + logger.debug('[monitor][workers] renders workers view:%s', html); + socket.emit('workers', { + 'view': 'html', + 'html': html + }); + }); + } + }); + }); + }); + setTimeout(tick, 1000);//everyone 1s + }); + + var watchings = [], + watchExists = function watchExists(usr, namespace){ + + if(_.contains(watchings, namespace)){ + return; + } + + watchings.push(namespace); + usr.watch(namespace, null, function(value, key){ + + logger.debug('[monitor][cache][%s] detects change of:%s', namespace, key); + usr.inspect(namespace, key).then(function(status){ + + logger.debug('[monitor][cache][%s] emits cache-changed event over websocket', namespace); + _.invoke(sockets, 'emit', 'cache-changed', namespace, key, status[0], status[1], status[2]); + }); + }); + }, + watchFresh = _.once(function(usr){ + + usr.watch('', null, function(value, namespace){ + + logger.info('[monitor][cache] detects new namespace:%s', namespace); + fs.readFile(path.join(__dirname, '/views/caches.ejs'), { + 'encoding': 'utf-8' + }, + function(err, read){ + + var html = ejs.render(read, { + 'namespace': namespace, + 'caches': [] + }); + + logger.debug('[monitor][cache][%s] renders empty caches view', namespace); + _.invoke(sockets, 'emit', 'caches', { + 'view': 'html', + 'namespace': namespace, + 'html': html + }); + }); + + watchExists(usr, namespace); + }); + }); + + function caches(){ + + masterPromise.then(function(){ + + require('./cache-usr').user().then(function(usr){ + + usr.ns() + .then(function(namespaces){ + + logger.debug('[monitor][cache] existing namespaces:%s', namespaces); + + _.each(namespaces || [], function(namespace){ + + watchExists(usr, namespace); + + usr.keys(namespace).then(function(keys){ + + logger.debug('[monitor][cache][%s] existing keys:%s', namespace, keys); + + when.map(keys, function(k){ + + return usr.inspect(namespace, k); + }) + .then(function(values){ + + fs.readFile(path.join(__dirname, '/views/caches.ejs'), { + 'encoding': 'utf-8' + }, + function(err, read){ + + var html = ejs.render(read, { + 'namespace': namespace, + 'caches': _.map(keys, function(k, i){ + return { + 'key': k, + 'value': values[i][0], + 'persist': values[i][1], + 'expire': values[i][2] + }; + }) + }); + + logger.debug('[monitor][cache][%s] inspected:\n%s\nand renders caches view:\n%s', JSON.stringify(values), namespace, html); + _.each(sockets, function(socket){ + if(!_.contains(socket.namespaces, namespace)){ + + socket.namespaces.push(namespace); + socket.emit('caches', { + 'view': 'html', + 'namespace': namespace, + 'html': html + }); + } + }) + }); + }); + }); + }); + }); + + watchFresh(usr); + }); + }); + } + + //the debug flow is as the following: + //app view accepts debug request, and we call master#debug to start + //app view shows the debug as preparing till we hear 'debug-inspector' event from the master + //app view shows the inspector's url for user to go to + //if it's debugging live, the worker is still running and debug is started as well + //otherwise the debugging fresh, the worker won't start till user comes back to the app view and 'resume' the debug (after he/she puts all the breakpoints needed) + //user could continue the debug till the app view accepts 'debug-finshed' request and propagate this event to the master + + io.sockets.on('connection', function (socket) { + + logger.info('[monitor] accepted new websocket and sending workers & caches view'); + + socket.knownPids = []; + socket.namespaces = []; + sockets.push(socket); + + workers(); + + caches(); + + socket.once('debug', function debug(pid){ + + debugging = pid; + inspector = null; + state = null; + + logger.info('[debug] %s requested', pid); + + masterPromise.then(function(master){ + + socket.once('debug-started', function(){ + + state = 'debug-started'; + master.emitter.to(['master']).emit('debug-started'); + logger.info('debug:%s start requested', pid); + }); + + socket.once('debug-finished', function(){ + + debugging = null; + inspector = null; + state = 'debug-finshed'; + + master.emitter.to(['master']).emit('debug-finished'); + logger.info('[debug] %s finished', pid); + socket.once('debug', debug); + }); + + master.emitter.once('debug-inspector', function(inspectorUrl){ + + inspector = inspectorUrl; + logger.info('[debug] %s inspector ready:%s', pid, inspector); + socket.knownPids = [];//force an update + }); + + master.debug(pid); + }); + }); + + socket.on('pause', function pause(pid){ + + masterPromise.then(function(master){ + if(!pauses[pid]){ + pauses[pid] = master.pause(pid); + } + }); + }); + + socket.on('resume', function resume(pid){ + + masterPromise.then(function(master){ + master.resume(pid); + delete pauses[pid]; + }); + }); + + socket.once('close', function close(){ + + sockets = _.without(sockets, socket); + }); + }); + + return server; +}; diff --git a/lib/process.js b/lib/process.js deleted file mode 100644 index ed5cf83..0000000 --- a/lib/process.js +++ /dev/null @@ -1,697 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the 'License'); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var misc = require('./misc.js'), - ecv = require('./ecv.js'), - Monitor = require('./monitor.js'), - _ = require('underscore'), - assert = require('assert'), - cluster = require('cluster'), - EventEmitter = require('events').EventEmitter, - os = require('os'), - fs = require('fs'), - when = require('when'), - timeout = require('when/timeout'), - util = require('util'), - usage = require('usage'), - memwatch = require('memwatch'); - -var debug = process.env['cluster2']; -function log() { - if(debug) { - console.log(JSON.stringify(arguments)); - } -} - -// Master process -var Process = module.exports = function Process(options) { - this.options = options || {}; - this.emitter = this.options.emitter || new EventEmitter(); - var self = this; - - // Stats - this.stats = { - workers: {}, - noWorkers: 0, - workersKilled: 0 - }; - - this.options.maxHeartbeatDelay = options.maxHeartbeatDelay || 3*60000; //default 3 mins - - this._heartbeats = []; - - this.killall = function(signal) { - log('killall called with signal ', signal); - var that = this, fullname; - fs.readdir(that.options.pids, function(err, paths) { - var count = paths.length; - if(count === 0) { - return; - } - var mf = _.find(paths, function(path) { - return /master\./.test(path); - }); - paths.forEach(function(filename) { - fullname = that.options.pids + '/' + filename; - if(/worker\./.test(filename)) { - that.kill(fullname, signal, function() { - count = count - 1; - if(count === 1 && mf) { - log('Sending ', signal, ' to the master'); - that.kill(that.options.pids + '/' + mf, signal); - } - }); - } - }); - }) - }; - - this.kill = function(fullname, signal, f) { - log('sending ', signal, ' to ', fullname); - fs.readFile(fullname, 'ascii', function(err, data) { - var pid = parseInt(data); - if(pid === process.pid) { - log('Unlinking ', fullname); - fs.unlinkSync(fullname); - process.exit(0); - } - else { - try { - process.kill(pid, signal); - } - catch(e) { - log(e.stack || e); - } - } - fs.unlink(fullname, function(err) { - log('Unlinking ', fullname); - if(err) { - console.error('Unable to delete ' + fullname); - } - if(f) { - assert.ok('function' === typeof f); - f(); - } - }); - }); - }; - - this.emitter.on('SIGINT', function() { - if(cluster.isMaster) { - self.killall(('SIGINT')); - clearInterval(self._heartbeatScheduler); - } - }); - this.emitter.on('SIGTERM', function() { - if(cluster.isMaster) { - self.killall('SIGTERM'); - clearInterval(self._heartbeatScheduler); - } - }); - this.emitter.on('SIGKILL', function() { - if(cluster.isMaster) { - self.killall('SIGKILL'); - clearInterval(self._heartbeatScheduler); - } - }); - - this.createWorker = function () { - var worker = cluster.fork().process; - var self = this; - self.lastTime = self.lastTime || Date.now(); - - fs.writeFileSync(util.format('%s/worker.%d.pid', this.options.pids, worker.pid), worker.pid); - - self.emitter.emit('forked', worker.pid); - - // Collect counters from workers - worker.on('message', function (message) { - if(message.type === 'counter') { - var name = message.name; - if(!self.stats.workers[message.pid]) { - self.stats.workers[message.pid] = {}; - } - var pidStats = self.stats.workers[message.pid]; - if(!pidStats[name]) { - pidStats[name] = 0 - } - pidStats[name]++; - self.emitter.emit('listening', message.pid); - } - if(message.type === 'heartbeat'){ - if(message.pid != process.pid){ - self._heartbeats.push(message);//must append to the tail - // update the last heartbeat time for the worker - var workerStats = self.stats.workers[message.pid]; - //console.log('heartbeat ' + process.pid); - workerStats.lastHeartbeatAt= Date.now(); - } - - self._heartbeatScheduler = self._heartbeatScheduler || setInterval(function () { - - var count = self._heartbeats.length, - threads = {}, - aggr = {}; - - _.each(_.range(0, count), function(aggregated){ - - var heartbeat = self._heartbeats.shift(); - - _.each(heartbeat, function(val, key){ - aggr[key] = (aggr[key] || 0) + val; - }); - - threads[heartbeat.pid] = heartbeat.pid; - }); - - //emit the aggregated heartbeat message - self.emitter.emit('heartbeat', { - 'pid': process.pid, - 'usertime': aggr.usertime / count,//avg - 'systime': aggr.systime / count,//avg - 'uptime': aggr.uptime / count,//avg - 'totalmem': aggr.totalmem / count,//avg - 'freemem': aggr.freemem / count,//avg - 'totalConnections': aggr.totalConnections,//total - 'pendingConnections': aggr.pendingConnections,//total - 'timedoutConnections': aggr.timedoutConnections,//total - 'fullGCs': aggr.fullGCs,//total - 'incrementalGCs': aggr.incrementalGCs,//total - 'heapCompactions': aggr.heapCompactions,//total - 'totalTransactions': aggr.totalTransactions,//total - 'totalDuration': aggr.totalDuration,//total - 'errors': aggr.errors,//total - 'threads': _.keys(threads).length, - 'interval': Date.now() - self.lastTime - }); - - self.lastTime = Date.now(); - - // Check the last heartbeat time of all the workers - _.each(self.stats.workers, function (workerStats, pid) { - var now = Date.now(); - if (now - workerStats.lastHeartbeatAt> self.options.maxHeartbeatDelay) { - // this worker hasn't been sending heartbeat for maxHeartbeatDelay - log(util.format('[Cluster2] Detected worker%d is not responsive for %d', pid, now - workerStats.lastHeartbeatAt)); - var deathQueue = require('./misc').deathQueue; - deathQueue(self.workers[pid], self.emitter, function () { - // create a successor - var successor = self.createWorker(); - self.workers[successor.pid] = successor; - log(util.format('[Cluster2] Created a new worker with pid %d', successor.pid)); - return successor; - }); - } - }); - - }, self.options.heartbeatInterval || 60000); - } - else if(message.type === 'suicide'){ //TODO, deathQueue - - var deathQueue = require('./misc').deathQueue; - - deathQueue(worker, self.emitter, function(){ - - var successor = self.createWorker(); - self.workers[successor.pid + ''] = successor; - return successor; - }); - } - else if(message.type === 'delegate'){//delegate is a proposed work pattern between master & workers, there're jobs workers would like to delegate to master - //and master would use messaging to interact with the actual handler module, and send the result back to all workers; besides, such handler might notify - //the master whenever there're some change of the results to publish to all workers in the future. - var delegate = message.delegate, - expect = message.expect; - - if(expect){//there're jobs which expects immediate responses, in case of those, onExpect handler is created, timeout might be applied - var deferred = when.defer(), - origin = message, - matches = message.matches, - targets = message.targets, - isExpected = function(origin, response){ - return _.reduce(matches, function(memoize, match){ - return memoize && _.isEqual(origin[match], response[match]); - }, - true); - }, - onExpect = function(message){ - if(isExpected(origin, message)){ - self.emitter.removeListener(expect, onExpect); - deferred.resolve(message); - } - }, - send = function(message){ - message.type = expect; - if(targets){ - var workers = _.reduce(self.workers, function(memoize, worker){ - memoize[worker.pid] = worker; - return memoize; - }, {}); - _.each(_.compact(targets), function(target){ - var targetWorker = self.workers[target]; - if(targetWorker){ - targetWorker.send(message); - } - }); - } - else{ - self.notifyWorkers(message); - } - }, - timeOut = setTimeout(function(){ - log('[cluster] reject timeout:' + JSON.stringify(message)); - deferred.reject(new Error('timeout')); - }, message.timeOut || 10000); - - self.emitter.on(expect, onExpect); - - deferred.promise - .then(function(message){ - clearTimeout(timeOut); - send(message); - }) - .otherwise(function(error){ - log('[cluster] fail error:' + error); - message.error = error; - send(message); - }) - .ensure(function(){ - if(message.notification){//this is for future update notifications, and registered afterwards. - if(!self._notifications){ - self._notifications = {}; - } - if(!self._notifications[expect]){//make sure notifications won't be repeatedly registered. - self._notifications[expect] = true; - self.emitter.on(expect, function(message){ - send(message); - }); - } - } - }); - } - - self.emitter.emit(delegate, message); - } - }); - - this.stats.noWorkers++; - - worker.on('message', function(message) { - if(message && message.command) { - self.notifyWorkers(message); - } - }); - - return worker; - } - - this.notifyWorkers = function(message) { - _.each(self.workers, function(worker, pid) { - try{ - worker.send(message); - } - catch(error){ - log('[cluster2] cannot send message to worker:' + pid); - } - }); - } -} - -Process.prototype.listen = function() { - - process.on('uncaughtException', function(err) { - // handle the error safely - log('[fatal] ' + err); - }); - - var exit = process.exit; - process.exit = function(){ - log('[cluster2] exit unexpectedly' + new Error().stack); - exit.apply(process, arguments); - }; - - var self = this, apps, monApp, cb; - - if(arguments.length === 3) { - apps = arguments[0]; - monApp = arguments[1]; - cb = arguments[2]; - } - else if (arguments.length === 2) { - apps = arguments[0]; - cb = arguments[1]; - } - if(cluster.isMaster) { - - this.stats.pid = process.pid; - this.stats.start = new Date(); - this.stats.totalmem = os.totalmem(); - this.stats.freemem = os.freemem(); - this.stats.workers = this.workers = []; - - //before monitor app starts - process.cluster = { - clustered: true, - emitter: self.emitter, - workers: self.workers - }; - //register the master worker itself - var componentStatus = self.componentStatusResolved = require('./component-status.js').componentStatus; - componentStatus.register('worker', function(){ - return 'm' + process.pid; - }, 'array'); - - self.emitter.emit('component-status-initialized', componentStatus); - - // Monitor to serve log files and other stats - typically on an internal port - var monitor = new Monitor({ - monitor: monApp, - stats: self.stats, - host: self.options.monHost, - port: self.options.monPort, - path: self.options.monPath - }); - - monitor.once('listening', function() { - misc.ensureDir(process.cwd() + '/pids', true); // Ensure pids dir - misc.ensureDir(process.cwd() + '/logs'); // Ensure logs dir - - fs.writeFileSync(util.format('%s/master.%d.pdf', self.options.pids, self.stats.pid), self.stats.pid); - log('Master ', process.pid, ' started'); - - // Fork workers - for(var i = 0; i < self.options.noWorkers; i++) { - var worker = self.createWorker(); - self.workers[worker.pid + ''] = worker; - } - - var deathWatcher = function (worker, code, signal) { - - worker = worker.process; - - log('[cluster2] death watch activated, worker:' + worker.pid + '\tcode:' + code + '\tsignal:' + signal + '\texit:' + worker.exitCode); - if(code === 0) { - self.emitter.emit('died', worker.pid); - self.stats.workersKilled++; - self.stats.noWorkers--; - delete self.workers[worker.pid + '']; - delete self.stats.workers[worker.pid]; - return; - } - - self.emitter.emit('died', worker.pid); - self.stats.workersKilled++; - self.stats.noWorkers--; - delete self.workers[worker.pid + '']; - delete self.stats.workers[worker.pid]; - //bugfix by huzhou@ebay.com, worker & replacement name collision - var replacement = self.createWorker(); - self.workers[replacement.pid + ''] = replacement; - - log('[cluster2] updated worker list:' + _.keys(self.workers)); - }; - cluster.on('exit', deathWatcher); - - process.on('SIGINT', function() { - cluster.removeListener('exit', deathWatcher); - self.emitter.emit('SIGINT'); - }); - - process.on('SIGTERM', function() { - log(process.pid, ' got SIGTERM'); - self.emitter.emit('SIGTERM', { - pid: process.pid, - type: 'master' - }); - var interval = setInterval(function() { - if(self.stats.noWorkers === 0) { - clearInterval(interval); - process.exit(0); - } - }, 100); - }); - - _.each(apps, function(app){ - app.app.on('connection', function(conn) { - log('master conn listener'); - }); - }); - - cb.call(null); - }); - - monitor.on('error', function (e) { - if(e.code === 'EADDRINUSE') { - console.error('Address in use ...'); - process.exit(-1); - } - }); - var monHost = this.options.monHost || '0.0.0.0'; - monitor.listen(this.options.monPort, monHost).once('listening', function(){ - //redundant for express2, absolutely needed for express3 and above - monitor.emit('listening'); - }); - } - else { - - var listening = false, conns = 0, totalConns = 0, timedoutConns = 0, noAppClosed = 0, graceful = _.once(function graceful(signal, code){ - - _.each(apps, function(app){ - if(app.listening){ - try { - app.app.close(); - } - catch(e) { - - } - } - }); - - // put the emitter in the process - process.emitter = self.emitter; - self.emitter.emit(signal, { - pid: process.pid, - type: 'worker' - }); - - // Once all pending connections are closed, exit. - var internal = setInterval(function() { - if(conns === 0) { - clearInterval(internal); - process.exit(code); - } - }, 100); - }); - - process.on('SIGINT', function() { - - graceful('SIGINT', 0); - }); - - process.on('SIGTERM', function() { - - log(process.pid + ' got SIGTERM'); - - graceful('SIGTERM', 0); - }); - - // Set time out on idle sockets - function monitorConnection(conn) { - //increase connections - conns++; - totalConns++; - //idle timeout - conn.setTimeout(self.options.timeout, function () { - - timedoutConns++; - self.emitter.emit('warning', { - message: 'Client socket timed out' - }); - conn.destroy(); - }); - //decrease connection - conn.on('close', function() { - conns--; - }) - } - - _.each(apps, function(app){ - var notifyInitialized = _.once(function(){ - - var componentStatus = self.componentStatusResolved = require('./component-status.js').componentStatus; - componentStatus.register('worker', function(){ - return process.pid; - }, 'array'); - - self.emitter.emit('component-status-initialized', componentStatus); - }); - - app.app.once('listening', function() { - app.listening = true; - process.send({ - type:'counter', - name:process.pid, - pid:process.pid - }); - - notifyInitialized(); - }); - - // Workers are net.Servers - var ports = _.isArray(app.port) ? app.port : [app.port]; - var host = self.options.host ? self.options.host : '0.0.0.0'; - var servers = _.map(ports, function(port) { - log('Worker ', process.pid, ' listening to ', port); - return app.app.listen(port, host); - }); - - _.each(servers, function(server){ - server.once('listening', function() { - if(self.options.ecv) { - ecv.enable(apps, self.options, self.emitter, function(data) { - return true; - }); - } - cb(); - //redundant for express2, absolutely needed for express3 and above - app.app.emit('listening'); - }); - server.on('connection', monitorConnection); - }); - }); - - // Recycle self when no of connections connection threshold - // we'd like to have the threshold randomized in between [1, 1.5) of the given threshold - // to avoid all workers die around the same time. This is in particular important for boxes of small number of cpu cores - var connThreshold = self.options.connThreshold, - uptimeThreshold = self.options.uptimeThreshold; - - var recycle = setInterval(function() { - - var uptime = process.uptime(); - if(totalConns > connThreshold || uptime >= uptimeThreshold) { - - log('[cluster2] exit because of connThreshold:' + connThreshold + ':' + totalConns + '; or uptime:' + uptime + ' has exceeded:' + uptimeThreshold); - clearInterval(recycle); - - //wait for master's order - process.send({ - 'type': 'suicide' - }); - } - }, - 1000); - - var memStats = { - 'num_full_gc': 0, - 'num_inc_gc': 0, - 'heap_compactions': 0 - }; - memwatch.on('stats', function(stats){ - _.extend(memStats, stats || {}); - }); - - var txStats = { - 'count': 0, - 'totalDuration': 0 - } - self.emitter.on('rootTransaction', function(tx){ - txStats.count += 1; - txStats.totalDuration += tx.duration; - }); - - var errors = 0; - self.emitter.on('errorTransaction', function(){ - - errors += 1; - }); - - // Heartbeat - make sure to clear this on 'close' - var heartbeat = setInterval(function () { - - usage.lookup(process.pid, function(err, result) { - - if(!err){ - var memTotal = Math.pow(2, 31) - 1,//should be 4g full space, but that exceeds MAX_INT, reduce it to MAX_INT - heartbeat = { - 'pid': process.pid, - 'usertime': result.cpu, - 'systime': result.cpu, - 'uptime': Math.round(process.uptime()), - 'totalmem': memTotal, - 'freemem': memTotal - result.memory, - 'totalConnections': totalConns, - 'pendingConnections': conns, - 'timedoutConnections': timedoutConns, - 'fullGCs': memStats['num_full_gc'], - 'incrementalGCs': memStats['num_inc_gc'], - 'heapCompactions': memStats['heap_compactions'], - 'totalTransactions': txStats.count, - 'totalDuration': txStats.totalDuration, - 'errors': errors - }; - - self.emitter.emit('heartbeat', heartbeat); - var toMaster = { - 'type': 'heartbeat' - }; - _.extend(toMaster, heartbeat); - - process.send(toMaster); - - //reset - memStats.num_full_gc = 0; - memStats.num_inc_gc = 0; - memStats.heap_compactions = 0; - txStats.count = 0; - txStats.totalDuration = 0; - errors = 0; - } - }); - - }, self.options.heartbeatInterval || 60000); - // put the heartbeat interval id in the process context - process.heartbeat = heartbeat; - - _.each(apps, function(app){ - - app.app.once('close', function() { - - noAppClosed++; - - if(noAppClosed >= noAppClosed.length){ - clearInterval(heartbeat); - clearInterval(recycle); - } - }); - }); - } - - process.on('exit', function () { - - log(process.pid, ' is about to exit.'); - }); -}; - -Process.prototype.stop = function() { - this.emitter.emit('SIGKILL'); -}; - -Process.prototype.shutdown = function() { - log('Shutdown request received - emitting SIGTERM'); - this.emitter.emit('SIGTERM'); -}; - diff --git a/lib/public/images/live-debugging.png b/lib/public/images/live-debugging.png new file mode 100644 index 0000000..626228b Binary files /dev/null and b/lib/public/images/live-debugging.png differ diff --git a/lib/puppet.js b/lib/puppet.js new file mode 100644 index 0000000..67a0313 --- /dev/null +++ b/lib/puppet.js @@ -0,0 +1,245 @@ +'use strict'; + +var when = require('when'), + util = require('util'), + _ = require('underscore'), + deathQueue = require('./utils').deathQueue; + +/** + * Puppet is a handle of the worker at the Master's runtime, to simplify the state machine & control over workers. + * NOTE, Puppet only runs in master's runtime, it has access to master instance, and the worker handle to communicate via messaging. + * NOTE, Puppet is a state machine, which are [FORKED, ACTIVE, PAUSED, OLD, DIED], the transitions are events based. + */ +var Puppet = exports.Puppet = function Puppet(master, worker, options, env){ + + var _this = this; + + env = env || {}; + + _.extend(_this, { + 'pid': worker.process.pid, + 'logger': master.logger, + 'emitter': master.emitter, + 'cacheManager': env.CACHE_MANAGER, + 'debugging': env.debug, + 'port': options.port, + 'warmUpPort': options.warmUpPort, + 'worker': worker, + 'lastHeartbeat': Date.now() + }); + + function defaultExit(){ + + if(!worker.suicide && _this.state !== _this.diedState){//accident, must be revived + master.fork(options, env); + } + + _this.state = _this.diedState; + + delete master.puppets[_this.pid]; + } + + _this.forkedState = { + + 'dismiss': function(){ + + worker.disconnect(); + }, + + 'whenOnline': function(){ + + if(_this.debugging){ + //we could enable debugging here + _this.emitter.once('debug-started', function(){ + _this.emitter.to([_this.pid]).emit('run'); + }); + + _this.emitter.once('debug-finished', function(){ + _this.dismiss(); + }); + + _this.debug(); + } + }, + + 'whenListening': function(address){ + + if(address.port === _this.port){ + _this.state = _this.activeState; + } + }, + + 'whenHeartbeat': function(){ + //no interest + }, + + 'whenExit': defaultExit + }; + + _this.activeState = { + + 'dismiss': function(){ + + worker.disconnect(); + }, + + 'whenHeartbeat': function(heartbeat){ + + if(master.shouldKill(heartbeat)){//when in active state, each heartbeat must be examined to determine if this worker is old + + //deathQueue function guarantees that only one worker could commit suicide at a time + deathQueue(master.deathQueue, _this.pid, master.emitter, function(){//success function + + _this.activeState = _this.oldState; + + return master.fork(options, env); + }); + } + }, + + 'whenExit': defaultExit + }; + + _this.pausedState = { + + 'dismiss': function(){ + + worker.disconnect(); + }, + + 'whenHeartbeat': function(heartbeat){ + + //no interest + }, + + 'whenListening': function(){ + + //ignored + }, + + 'whenExit': defaultExit + }; + + _this.oldState = { + + 'dismiss': function(){ + + worker.suicide = true; //MUST set suicide true, to avoid master spawning extra worker + + worker.disconnect(); + }, + + 'whenExit': function(){ + + _this.state = _this.diedState; + + //already exit, clean up + delete master.puppets[_this.pid]; + } + }; + + _this.diedState = { + + 'whenExit': function(){ + + //already exit, just clean up + delete master.puppets[_this.pid]; + } + }; + + _this.state = _this.forkedState; +}; + +Puppet.prototype.debug = function(){ + + var _this = this, + pid = _this.pid, + worker = _this.worker, + tillDebugPortListen = when.defer(); + + _this.emitter.once(util.format('debug-%d-listening', pid), function(){ + + _this.logger.info('[debug] puppet: %d received debug-port-listening', pid); + + tillDebugPortListen.resolve(pid); + }); + + worker.process.kill('SIGUSR1');//make it debug ready + + _this.logger.info('[debug] master sent signal SIGUSR1 to %d', pid); + + return tillDebugPortListen.promise; +}; + +Puppet.prototype.dismiss = function(){ + + return this.state.dismiss(); +}; + +Puppet.prototype.pause = function(){ + + var _this = this, + tillPaused = when.defer(); + + _this.emitter.once(util.format('worker-%d-paused', _this.pid), function(){ + if(_this.state === _this.activeState){ + _this.state = _this.pausedState; + tillPaused.resolve(_this); + } + else{ + tillPaused.reject(new Error('state changed before pause completed')); + } + }); + + _this.emitter.to([_this.pid]).emit('pause'); + + return tillPaused.promise; +}; + +Puppet.prototype.resume = function(){ + + var _this = this, + tillResumed = when.defer(); + + _this.emitter.once(util.format('worker-%d-resumed', _this.pid), function(){ + if(_this.state === _this.pausedState){ + _this.state = _this.activeState; + tillResumed.resolve(_this); + } + else{ + tillResumed.reject(new Error('state changed before resume completed')); + } + }); + + _this.emitter.to([_this.pid]).emit('resume'); + + return tillResumed.promise; +}; + +Puppet.prototype.whenOnline = function(){ + + return this.state.whenOnline(); +}; + +Puppet.prototype.whenListening = function(address){ + + this.logger.info('[master] detects worker:%d is listening now on:%j', this.pid, address); + + return this.state.whenListening(address); +}; + +Puppet.prototype.whenHeartbeat = function(heartbeat){ + + this.lastHeartbeat = Date.now(); //MUST update #lastHeartbeat to survive nanny check + + return this.state.whenHeartbeat(heartbeat); +}; + +Puppet.prototype.whenExit = function(){ + + this.logger.info('[master] detects exit of worker:%d', this.pid); + + this.emitter.to(['master']).emit(util.format('worker-%d-died', this.pid)); + + return this.state.whenExit(); +}; diff --git a/lib/status.js b/lib/status.js new file mode 100644 index 0000000..5d456b7 --- /dev/null +++ b/lib/status.js @@ -0,0 +1,151 @@ +'use strict'; + +//status is to allow workers to expose their important states to master, and possibly let monitor app show them clearly in its views + +var when = require('when'), + timeout = require('when/timeout'), + cluster = require('cluster'), + util = require('util'), + _ = require('underscore'); + +var emitter = require('./emitter'), + noop = function(){ + + }; + +module.exports = (function(emitter){ + + var logger = process.getLogger(__filename), + registry = {}; //map of registered status names to status entities + //(each entity is a struct {'pid':null, 'view':null, 'update':null} + + emitter.on('new-status', function(name, pid, view, update){ + + registry[name] = registry[name] || []; + + if(_.some(registry[name], function(r){ //avoid duplicate component registry + return r.pid === pid; + })){ + + logger.info('[status] register status ignored due to existing registry'); + } + else{ + + registry[name].push({ + 'pid': pid, + 'view': view, + 'update': update + }); + + logger.info('[status] register status: %s from %d, in process: %d', name, pid, process.pid); + } + }); + + emitter.on('del-status', function(name, pid){ + + registry[name] = _.filter(registry[name], function(r){ + return r.pid !== pid; + }); + + if(!registry[name].length){ + delete registry[name]; + } + + logger.info('[status] unregister status: %s in process: %d', name, pid); + }); + + cluster.on('disconnect', function(worker){ + + var pid = worker.process.pid; + + _.each(registry, function(arr, name){ + + registry[name] = _.filter(arr, function(r){ + return r.pid !== pid; + }); + + if(!registry[name].length){ + delete registry[name]; + } + }); + + logger.debug('[status] auto updated after worker:%j got disconnected', registry); + }); + + return { + + 'register': function register(name, view, update){ + + emitter.to(['master', 'self']).emit('new-status', name, process.pid, view, update); + + emitter.on(util.format('get-status-%s', name), function(echo){ + + emitter.to(['master', 'self']).emit(echo, view()); + }); + + emitter.on(util.format('set-status-%s', name), function(value, echo){ + + update = update || noop;//in case update wasn't given + + emitter.to(['master', 'self']).emit(echo, update(value)); + }); + }, + + 'unregister': function unregister(name){ + + emitter.to(['master', 'self']).emit('del-status', name, process.pid); + + emitter.removeAllListeners(util.format('get-status-%s', name)); + + emitter.removeAllListeners(util.format('set-status-%s', name)); + }, + + 'statuses': function(){ + + return _.keys(registry); + }, + + 'getStatus': function getStatus(name, wait){ + + return when.map(registry[name], function(r){ + + var tillGet = when.defer(), + echo = util.format('get-status-%s-%d-%d', name, r.pid, Date.now()); + + emitter.once(echo, function(status){ + + tillGet.resolve({ + 'pid': r.pid, + 'name': name, + 'status': status + }); + }); + + emitter.to([r.pid]).emit(util.format('get-status-%s', name), echo); + + return timeout(tillGet.promise, wait || 1000); + }); + }, + + 'setStatus': function setStatus(name, value, wait){ + + return when.map(registry[name], function(r){ + + var tillSet = when.defer(), + echo = util.format('set-status-%s-%d-%d', name, r.pid, Date.now()); + + emitter.once(echo, function(status){ + tillSet.resolve({ + 'pid': r.pid, + 'name': name, + 'status': status + }); + }); + + emitter.to([r.pid]).emit(util.format('set-status-%s', name), value, echo); + + return timeout(tillSet.promise, wait || 1000); + }); + } + }; +})(emitter); diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..70d8066 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,501 @@ +'use strict'; + +var http = require('http'), + path = require('path'), + util = require('util'), + when = require('when'), + timeout = require('when/timeout'), + cluster = require('cluster'), + winston = require('winston'), + request = require('request'), + fs = require('graceful-fs'), + _ = require('underscore'), + fork = require('child_process').fork, + execFile = require('child_process').execFile, + EventEmitter = require('events').EventEmitter, + assert = require('assert'), + ACCESS = parseInt('0755', 8); + +exports.decorateEmitter = function decorateEmitter(emitter){ + + if(!emitter.to){//if it's a normal EventEmitter, we simply add 'to' function to return the emitter itself which will allow a following #emit call + emitter.to = function(){ + return emitter; + }; + } + + return emitter; +}; + +exports.rejectIfPortBusy = function rejectIfPortBusy(host, port){ + + var deferred = when.defer(), + server = http.createServer(function(req, res){ + + res.writeHead(200, {'Content-Type': 'text/plain'}); + res.end(port.toString(10)); + }); + + server.once('error', function(e){ + deferred.reject(new Error('Port is in use:' + port)); + }); + + server.listen(port, host, function(){ //'listening' listener + + request.get(util.format('http://%s:%d/', host, port), function(err, response, body){ + + if(!err && response && response.statusCode === 200 && parseInt(body, 10) === port){ + server.close(function(){ + process.nextTick(function(){ + deferred.resolve(port); + }); + }); + } + else{ + deferred.reject(new Error('Port is in use:' + port)); + } + }); + }); + + return timeout(3000, deferred.promise); +}; + +global.portsAlreadyPicked = []; + +exports.pickAvailablePort = function pickAvailablePort(min, max){ + + function checkAvailability(deferred, port){ + + if(port > max){ + deferred.reject(new Error('no port available')); + } + else if(_.contains(global.portsAlreadyPicked, port)){ + checkAvailability(deferred, port + 1); + } + else{ + exports.rejectIfPortBusy('localhost', port) + .then(function(port){ + deferred.resolve(port); + global.portsAlreadyPicked.push(port); + }) + .otherwise(function(){ + checkAvailability(deferred, port + 1); + }); + } + } + + var available = when.defer(); + + checkAvailability(available, min); + + return available.promise; +}; + +exports.pickAvailablePorts = function pickAvailablePorts(min, max, count){ + + return when.map(_.range(0, count), function(ith){ + + return exports.pickAvailablePort(min, max); + }); +}; + +exports.ensureDir = function ensureDir(dir, clean) { + try { + var paths = fs.readdirSync(dir); + if(clean) { + paths.forEach(function(filename) { + try { + fs.unlinkSync(path.join(dir, filename)); + } + catch(e) { + + } + }); + } + } + catch(e) { + fs.mkdirSync(dir, ACCESS); + } +}; + +exports.writePid = function writePid(pid, dir) { + + pid = pid || process.pid; + dir = dir || path.join(process.cwd(), '/pids'); + + exports.ensureDir(dir); + + var persist = util.format('%s.%d.pid', cluster.isMaster ? 'master' : 'worker', pid); + + fs.writeFileSync(path.join(dir, persist), pid, { + 'encoding': 'utf-8' + }); +}; + +exports.markDeadPid = function markDeadPid(pid, dir) { + + pid = pid || process.pid; + dir = dir || path.join(process.cwd(), '/pids'); + + exports.ensureDir(dir); + + //this is to exclude it from readPids, such that at time of shutdown, those already dead won't become suspects + fs.renameSync(path.join(dir, util.format('worker.%d.pid', pid)), + path.join(dir, util.format('dead.%d.%d.pid', pid, Date.now()))); +}; + +exports.readPids = function readPids(dir) { + + dir = dir || path.join(process.cwd(), '/pids'); + exports.ensureDir(dir); + + return _.map(_.filter(fs.readdirSync(dir), function(filename){ + return /master\./.test(filename) || /worker\./.test(filename); + }), + function(filename){ + + return parseInt(fs.readFileSync(path.join(dir, filename), { + 'encoding': 'utf-8' + }), 10); + }); +}; + +exports.readMasterPid = function readMasterPid(dir) { + + dir = dir || path.join(process.cwd(), '/pids'); + exports.ensureDir(dir); + + return parseInt(fs.readFileSync(path.join(dir, _.filter(fs.readdirSync(dir), function(filename){ + return /master\./.test(filename); + })[0]), + { + 'encoding': 'utf-8' + }), + 10); +}; + +exports.safeKill = function safeKill(pid, signal, logger){ + + try{ + process.kill(pid, signal); + return false; + } + catch(e){ + //verify error is Error: ESRCH + logger.debug('[shutdown] safeKill received:%j', e); + return e.errno === 'ESRCH'; //no such process + } +}; + +exports.getNodeInspectorPath = function getNodeInspectorPath(){ + + return require.resolve('node-inspector/bin/inspector'); +}; + +exports.startInspector = function startInspector(webPort, debugPort, saveLiveEdit, hidden, logger){ + + logger = logger || {'info': _.bind(console.log, console)}; + hidden = hidden || []; + + logger.info('[utils] starting node-inspector webPort:%d, debugPort:%d, saveLiveEdit:%s, hidden:%j', webPort, debugPort, saveLiveEdit, hidden); + + var inspectorPath = exports.getNodeInspectorPath(), + inspectorArgs = [ + '--web-port=' + webPort, //where node-inspector process listens to users' request from v8 + '--debug-port=' + debugPort, //where node-inspector process subscribes to debugging app process + '--save-live-edit=' + saveLiveEdit, //whether or not user could modify the debugging source and save it + '--hidden=' + JSON.stringify(hidden)//files excluded from adding breakpoints + ]; + + logger.info('[utils] starting node-inspector at:%s with args:%j', inspectorPath, inspectorArgs); + + assert.ok(inspectorPath); + + //NOTE, this is not _this.fork, but child_process.fork + return fork(inspectorPath, inspectorArgs, { + 'silent': true + }); +}; + +exports.getLogger = (function(dir){ + + dir = dir || path.join(process.cwd(), './log'); + exports.ensureDir(dir); + + var fileLoggerTransport = new (winston.transports.File)({ + 'filename': path.join(dir, process.pid + '.log'), + 'maxsize': 4 * 1024 * 1024,//4mb + 'maxFiles': 4//4 files max, 16mb for each process + }), + loggers = { + + }; + + //to avoid the annoying error, winston registers too many 'error' handler on the same transport instance. + fileLoggerTransport.setMaxListeners(1024); + + return function getLogger(category){ + + if(!loggers[category]){ + loggers[category] = new (winston.Logger)({ + 'transports': [ + new (winston.transports.Console)({ + 'colorize': 'true', + 'label': category + }), + fileLoggerTransport + ] + }); + } + + return loggers[category]; + }; +})(); + +//here's the key feature for cluster3, based on the historic tps info, memory usage, gc rate, we could determine if a puppet should +//enter an old state from active + +exports.assertOld = function assertOld(maxAge){ + + maxAge = maxAge || 3600 * 24 * 3;//3 days + + return maxAge > 0 + ? function(heartbeat){ + return heartbeat.uptime >= maxAge; + } + : function(heartbeat){ //forever young. + return false; + }; +}; + +exports.assertBadGC = function assertBadGC(){ + + var peaks = {}; + + return function(heartbeat){ + + var pid = heartbeat.pid, + uptime = heartbeat.uptime, + currTPS = heartbeat.tps || (heartbeat.transactions * 1000 / heartbeat.cycle); + + if(currTPS <= 2){//TPS too low, no good for sampling. + return false; + } + + var peak = peaks[pid] = peaks[pid] || { + 'tps': currTPS, + 'cpu': heartbeat.cpu, + 'memory': heartbeat.memory, + 'gc': { + 'pauseMS': heartbeat.gc.pauseMS + } + };//remember the peak of each puppet + + if(currTPS >= peak.tps){ + peak.tps = Math.max(heartbeat.tps, peak.tps); + peak.cpu = Math.max(heartbeat.cpu, peak.cpu); + peak.memory = Math.max(heartbeat.memory, peak.memory); + peak.gc.pauseMS = Math.max(heartbeat.gc.pauseMS, peak.gc.pauseMS); + } + else if(currTPS < peak.tps * 0.9 //10% tps drop + && heartbeat.cpu > peak.cpu + && heartbeat.memory > peak.memory + && heartbeat.gc.pauseMS > peak.gc.pauseMS){ + //tps drops, while cpu/memory/gc pause all rise, it's highly likely that GC has gone wild + return true; + } + + return false; + } +}; + +exports.deathQueueGenerator = function(options){ + + var tillPrevDeath = null; + + return function deathQueue(queue, pid, emitter, success){ + + options = options || {}; + + assert.ok(queue); + assert.ok(pid); + assert.ok(emitter); + assert.ok(success); + + var wait = options.timeout || 60000, + retry = options.retry || 3, + death = util.format('worker-%d-died', pid), + logger = options.logger || { + 'debug' : function(){ + console.log.apply(console, arguments); + } + }; + + if(!_.contains(queue, pid)){ + + queue.push(pid); + + var tillDeath = when.defer(), + afterDeath = null, + die = function die(retry){ + + if(!retry){ + if(tillPrevDeath){ + tillPrevDeath.reject(new Error('[deathQueue] failed after retries')); + } + tillPrevDeath = null;//reset + } + + var successor = success(), + successorPid = successor.process.pid, + successorEvent = util.format('worker-%d-listening', successorPid), + successorGuard = setTimeout(function onSuccessorTimeout(){ + //handle error case of successor not 'listening' after started + exports.safeKill(pid, 'SIGTERM', logger); + + logger.debug('[deathQueue] successor:%d did not start listening, kill by SIGTERM', successorPid); + //cancel onListening event handler of the dead successor + emitter.removeListener(successorEvent, onSuccessorListening); + //retry of the 'die' process + die(retry - 1); + + }, wait); + + //when successor is in place, the old worker could be discontinued finally + emitter.once(successorEvent, function onSuccessorListening(){ + + clearTimeout(successorGuard); + logger.debug('[deathQueue] successor:%d of %d is ready, wait for %s and timeout in:%dms', successorPid, pid, death, wait); + + var deathGuard = setTimeout(function(){ + + if(!exports.safeKill(pid, 'SIGTERM', logger)){ + //worker still there, should emit 'exit' eventually + logger.debug('[deathQueue] worker:%d did not report death by:%d, kill by SIGTERM', pid, wait); + } + else{//suicide or accident already happended, process has run away + //we emit this from master on behalf of the run away process. + logger.debug('[deathQueue] worker:%d probably ran away, emit:%s on behalf', death); + //immediately report death to the master + emitter.to(['master']).emit(death); + } + + }, wait); + + emitter.to(['master', pid]).emit('dismiss', pid); + + emitter.once(death, function(){ + + logger.debug('[deathQueue] %d died', pid); + + clearTimeout(deathGuard);//release the deathGuard + + tillDeath.resolve(pid); + + if(tillPrevDeath === afterDeath){//last of dyingQueue resolved, clean up the dyingQueue + + logger.debug('[deathQueue] death queue cleaned up'); + + tillPrevDeath = null; + + queue.splice(0, queue.length); + } + }); + }); + }; + + if(!tillPrevDeath){//1st in the dying queue, + afterDeath = tillPrevDeath = tillDeath.promise;//1 min + die(retry); + } + else{ + afterDeath = tillPrevDeath = tillPrevDeath.ensure(_.bind(die, null, retry)); + } + } + }; + +}; + +exports.deathQueue = exports.deathQueueGenerator({ + 'timeout': 60000 +}); + +//this function is supposed to nanny all the puppets by checking their last heartbeat time +//if it has exceeded the max tolerance, we'll think of it as run away, and put into the deathQueue + +exports.nanny = function nanny(puppets, queue, emitter, success, options){ + + assert.ok(puppets); + assert.ok(queue); + assert.ok(emitter); + assert.ok(success); + + options = options || {}; + + var tolerance = options.tolerance || 60000 * 3, + now = Date.now(); + + _.each(puppets, function(p){ + + if(now - p.lastHeartbeat > tolerance){ + + exports.deathQueue(queue, p.pid, emitter, success, options); + + } + }); +}; + +exports.markUpAfterAllListening = function markUpAfterAllListening(emitter, expects){ + + return when.map(expects, function(pid){ + + var tillListening = when.defer(); + + emitter.once(util.format('worker-%d-warmup-failure', pid), function(error){ + + tillListening.reject(error); + }); + + emitter.once(util.format('worker-%d-listening', pid), function(address){ + + tillListening.resolve(address); + }); + + return timeout(60000, tillListening.promise); + }); +}; + +exports.gcstats = (function(){ + + var bin = require('gc-stats/build/Release/gcstats'), + emitter = new EventEmitter(); + + bin.afterGC(function(stats) { + emitter.emit('stats', stats); + }); + + return emitter; +})(); + +exports.npmls = (function npmls(){ + + var tillList = when.defer(), + execPath = process.execPath, + execArgv = [path.join(require.resolve('npm'), '../../bin/npm-cli.js'), 'ls', '--json', '--depth=10']; + + execFile(execPath, execArgv, { + 'cwd': process.cwd(), + 'encoding': 'utf-8' + }, + function(err, stdout){ + + if(err){ + tillList.reject(err); + } + else{ + tillList.resolve(stdout); + } + }); + + return tillList.promise; +})(); diff --git a/lib/views/caches.ejs b/lib/views/caches.ejs new file mode 100644 index 0000000..4cd4d66 --- /dev/null +++ b/lib/views/caches.ejs @@ -0,0 +1,54 @@ +
+
+

Cache: <%=namespace%>

+
+
+ + + + + + + + + + + + <%for(var k = 0, len = caches.length; k < len; k += 1){ + var entry = caches[k];%> + + + + + + + <%}%> + +
#valuepersistedexpire
<%=entry.key%><%=JSON.stringify(entry.value)%><%=entry.persist ? true : false%><%=entry.expire ? entry.expire + 'ms' : ''%>
+ +
+
+ + \ No newline at end of file diff --git a/lib/views/index.ejs b/lib/views/index.ejs new file mode 100644 index 0000000..6ec6268 --- /dev/null +++ b/lib/views/index.ejs @@ -0,0 +1,84 @@ + + + + cluster2 + + + + + + + + + + + + + +
+
+

Active Workers

+
+
+ +
+
+ +
+
+

Active Cache

+
+
+ +
+
+ + + + + + + + diff --git a/lib/views/workers.ejs b/lib/views/workers.ejs new file mode 100644 index 0000000..57af6cb --- /dev/null +++ b/lib/views/workers.ejs @@ -0,0 +1,150 @@ + + + + + <%for(var w = 0, len = pids.length; w < len; w +=1){ + var pid = pids[w];%> + + <%}%> + + + + + <%for(var s = 0, slen = statuses.length; s < slen; s += 1){ + var status = statuses[s];%> + + + <%for(var w = 0, wlen = pids.length; w < wlen; w += 1){ + var pid = pids[w], + type = typeof status[pid];%> + + <%}%> + + <%}%> + +
# +
+ worker: <%=pid%> + + <%if(debugging && debugging === pid){%> + + + <%}else if(debugging){%> + + <%}else{%> + + <%}%> +
+
<%=status.name%> + <%if(type === 'string' || type === 'number' || type === 'boolean' || type === 'null'){%> +
+ <%=status[pid]%> +
+ <%} + else{ + for(var st in status[pid] || {}){%> +
+ <%=st%> : <%=JSON.stringify(status[pid][st])%> +
+ <%} + }%> +
+ + diff --git a/lib/worker.js b/lib/worker.js new file mode 100644 index 0000000..2f96560 --- /dev/null +++ b/lib/worker.js @@ -0,0 +1,423 @@ +'use strict'; + +var _ = require('underscore'), + when = require('when'), + util = require('util'), + timeout = require('when/timeout'), + usage = require('usage'), + uvmon = require('nodefly-uvmon'), + BigNumber = require('bignumber.js'), + assert = require('assert'); + +var Worker = exports.Worker = function(proc, options){ + + var _this = this, + emitter = _this.emitter = require('./utils').decorateEmitter(options.emitter); + + _.extend(_this, { + 'pid': proc.pid, + 'process': proc, + 'logger': proc.getLogger(__filename), + 'options': options, + 'runnable': options.runnable, + 'createServer': options.createServer, + 'app': options.app, + 'port': options.port, + 'warmUpPort': process.env.warmUpPort || options.warmUpPort, + 'configureApp': function(app){ + + if(_.isFunction(app.use)){//make sure this middleware is ahead of others, to collect tps information + + app.use(function(req, res, next){ + + var begin = Date.now(); + res.once('finish', function(){ + _this.transactions += 1; + _this.durations += Date.now() - begin; + }); + + next(); + }); + } + + return options.configureApp(app); + }, + 'warmUp': options.warmUp, + 'debug': options.debug || false, + 'timeout': options.timeout || 5000, + 'aliveConnections': 0, + 'totalConnections': 0, + 'transactions': 0, + 'durations': 0, + 'totalTransactions': new BigNumber(0), + 'totalDurations': new BigNumber(0), + 'heartbeatInterval': options.heartbeatInterval || 60000,//1 min, the heartbeat shouldn't be too frequent, which could cause false assertion of utils#assertBadGC + 'status': require('./status'), + 'status.os': { + + }, + 'gc': { + 'monitor': options.gc.monitor, + 'explicit': options.gc.explicit, + 'incremental': 0, + 'full': 0, + 'pauseMS': 0 + }, + 'error': { + 'count': 0, + 'fatal': 0 + }, + 'isMaster': false, + 'isWorker': true + }); + + if(_this.gc.monitor){ + //in dev, we will not monitor gc, because, oddly enough, it conflicts with socket.io + //which is the key to our hot reload functionality + + var gcstats = require('./utils').gcstats; + + gcstats.on('stats', function(stats) { + //cannot tell if it's incremental or full, just to check if pauseMS is too long + _this.whenGC(stats.pauseMS, stats.pauseMS < 500 ? 'incremental' : 'full'); + }); + } + + process.once('disconnect', _.bind(_this.whenStop, _this, 'disconnect')); + process.once('SIGINT', _.bind(_this.whenStop, _this)); + process.once('SIGTERM', _.bind(_this.whenExit, _this)); + process.once('SIGUSR1', function(){ + + process.nextTick(function(){ + emitter.emit(util.format('debug-%d-listening', _this.pid)); + }); + }); + + emitter.on('error', function(err){ + + if(err && err.fatal){ + _this.error.fatal += 1; + } + else{ + _this.error.count += 1; + } + }); + + emitter.on('pause', function onPause(){ + _this.logger.info('worker:%d pause request', process.pid); + _this.pause().then(function(){ + emitter.emit(util.format('worker-%d-paused', _this.pid)); + }); + }); + + emitter.on('resume', function onResume(){ + _this.resume().then(function(){ + emitter.emit(util.format('worker-%d-resumed', _this.pid)); + }); + }); + + _this.status.register('status.os', + function(){ + return _this['status.os']; + }, + function(status){ + return _this['status.os'] = status; + }); + + _this.whenHeartbeat(); +}; + +Worker.prototype.listen = function listen(){ + + var _this = this, + app = _this.app, + port = _this.port, + warmUpPort = _this.warmUpPort, + createServer = _this.createServer, + configureApp = _this.configureApp, + warmUp = _this.warmUp, + debug = _this.debug, + wait = _this.timeout; + + assert.ok(createServer); + assert.ok(app); + assert.ok(port); + assert.ok(configureApp); + assert.ok(warmUp); + + var tillListen = when.defer(), + run = function(){ + + when(configureApp(app)).ensure(function(configured){ //configure app before warming up + + _this.logger.debug('[worker] %d app configured', _this.pid); + + var warmUpServer = createServer(app).listen(warmUpPort, function(){ //warming up by starting listening on `warmUpPort` + + _this.logger.info('[worker] %d started warming on:%d', _this.pid, warmUpPort); + _this.emitter.emit(util.format('worker-%d-warming', _this.pid)); + + function actualListenAfterWarmUp(){ + + warmUpServer.close(function(){ + + //switching listening port from `warmUpPort` to the actual `port` + var server = _this.server = createServer(app).listen(port, function(){ + + _this.logger.info('[worker] %d started listening on:%d', _this.pid, port); + //tell master, worker ready + _this.emitter.emit(util.format('worker-%d-listening', _this.pid), server.address()); + + //connection monitoring, including live/total connections and idle connections + server.on('connection', function(conn){ + _this.whenConnected(conn); + }); + + tillListen.resolve({ + 'server': server, + 'app': app, + 'port': port, + 'master': null, + 'worker': _this + }); + }); + }); + } + + //notify the `warmUp` callback, server is listening at `address` + when(warmUp(app, warmUpServer.address())) + .then(function(){ + + _this.logger.info('[worker] %d warmed up', _this.pid); + _this.emitter.emit(util.format('worker-%d-warmup', _this.pid)); //tell everyone warmup is done + + actualListenAfterWarmUp(); + }) + .otherwise(function(error){ + + _this.logger.info('[worker] %d warmup failure', _this.pid); + _this.emitter.emit(util.format('worker-%d-warmup-failure', _this.pid), error); //tell everyone warmup is done + + actualListenAfterWarmUp(); + }); + }); + }); + }; + + if(!debug){ //normal + run(); + } + else{ //debug fresh process, waiting for 'run' command + _this.emitter.once('run', run); + } + + return (wait > 0 ? timeout(_this.timeout, tillListen.promise) : tillListen.promise); +}; + +Worker.prototype.pause = function pause(){ + + var _this = this, + tillPaused = when.defer(); + + //pause is done by stopping the server completely, it works with #resume which recreates the server and start listening + //the assumption is that stop/start server is not expensive at all, the state of application is well capsuled by the express app + //and the user's own objects in the process's memory, as long as nothing is tied with the network server, resume would be really fast + if(!_this.server){ + tillPaused.reject(new Error('server not started')); + } + else{ + _this.server.close(function(){ + tillPaused.resolve(_this.server); + }); + _this.server = null; + } + + return tillPaused.promise; +}; + +Worker.prototype.resume = function resume(){ + + var _this = this, + app = _this.app, + port = _this.port, + tillResumed = when.defer(); + + //simply start listening on the `port` again, it should be very fast, for the immediate next request + //as we assume the state in the express app and users' space won't be impacted much by switching the server + if(!_this.server){ + _this.server = _this.createServer(app).listen(port, function(){ + + tillResumed.resolve({ + 'server': _this.server, + 'app': app, + 'port': port, + 'master': null, + 'worker': _this + }); + }); + } + else{ + tillResumed.reject(new Error('server already listening')); + } + + return tillResumed.promise; +}; + +Worker.prototype.run = function run(){ + + var _this = this, + runnable = _this.runnable, + debug = _this.debug, + wait = _this.timeout, + tillRun = when.defer(), + warmUpThenRun = function warmUpThenRun(){ + + when(_this.warmUp(runnable)).ensure(function(warmedUp){ //warm up app after listening + + _this.logger.debug('[worker] %d warmed up', _this.pid); + _this.emitter.emit(util.format('worker-%d-warmup', _this.pid)); //tell master i'm ready + + tillRun.resolve({ + 'runnable': runnable, + 'master': null, + 'worker': _this + }); + + runnable(); + }); + }; + + assert.ok(runnable); + + if(!debug){ + warmUpThenRun(); + } + else{ + _this.emitter.once('run', warmUpThenRun); + } + + return (wait > 0 ? timeout(_this.timeout, tillRun.promise) : tillRun.promise); +}; + +Worker.prototype.whenConnected = function whenConnected(conn) { + + var _this = this; + + _this.aliveConnections += 1; + _this.totalConnections += 1; + + conn.setTimeout(_this.timeout, _.bind(conn.destroy, conn)); + + conn.once('close', function() { + _this.aliveConnections -= 1; + }); +}; + +Worker.prototype.whenHeartbeat = function whenHeartbeat(){ + + //heartbeat to the master + var _this = this, + emitter = _this.emitter; + + try{ + usage.lookup(_this.pid, function(error, result){ + + _this.totalTransactions = _this.totalTransactions.plus(_this.transactions); + _this.totalDurations = _this.totalDurations.plus(_this.durations); + + var heartbeat = { + 'pid': process.pid, + 'uptime': process.uptime(), + 'cpu': result.cpu, + 'memory': result.memory, + 'aliveConnections': _this.aliveConnections, + 'totalConnections': _this.totalConnections, + 'transactions': _this.transactions, + 'durations': _this.durations, + 'tps': _this.transactions * 1000 / _this.heartbeatInterval, + 'totalTransactions': _this.totalTransactions.toString(16), + 'totalDurations': _this.totalDurations.toString(16), + 'gc': _this.gc, + 'uv': uvmon.getData(), //added uv monitor data for heartbeat + 'error': _this.error, + 'cycle': _this.heartbeatInterval + }; + + _this.status.setStatus('status.os', heartbeat); + emitter.emit('heartbeat', heartbeat); + + //cleanup after heartbeat + _this.durations = 0; + _this.transactions = 0; + _this.gc = { + 'incremental': 0, + 'full': 0, + 'pauseMS': 0 + }; + _this.error = { + 'count': 0, + 'fatal': 0 + }; + }); + } + finally{ + _this.nextHeartbeat = setTimeout( + _.bind(_this.whenHeartbeat, _this), _this.heartbeatInterval); + } +}; + +Worker.prototype.whenGC = function whenGC(usage, type){ + + var _this = this; + + _this.logger.debug('[worker] gc usage:%d, type:%s', usage, type); + + _this.gc[type] += 1; + _this.gc.pauseMS += usage; + + _this.emitter.to(['self']).emit('gc', usage, type); +}; + +Worker.prototype.beforeStop = function beforeStop(){ + + var _this = this; + + clearTimeout(_this.nextHeartbeat); + + //stop serving traffic, when 'disconnect' server should already take off traffic, this is to make sure resources get released + if(_.isFunction(_this.app.close)){ + + _this.app.close(); + } +}; + +Worker.prototype.whenStop = _.once(function whenStop(){ + + var _this = this; + + _this.beforeStop(); + + (function gracefully(){ + + if(_this.aliveConnections > 0){ + + _this.logger.debug('[worker] gracefully shutdown: %d pending on aliveConnections: %d', process.pid, _this.aliveConnections); + setTimeout(gracefully, 500); + } + else{ + + _this.logger.info('[worker] gracefully shutdown: %d', process.pid); + process.exit(0); + } + + })(); +}); + +Worker.prototype.whenExit = _.once(function whenExit(){ + + _this.beforeStop(); + + this.logger.warn('[worker] forced shutdown: %d', process.pid); + + process.exit(-1); +}); diff --git a/monitor.js b/monitor.js new file mode 100644 index 0000000..29e886e --- /dev/null +++ b/monitor.js @@ -0,0 +1,3 @@ +'use strict'; + +module.exports = require('./lib/monitor'); \ No newline at end of file diff --git a/package.json b/package.json index 0697851..6da952d 100644 --- a/package.json +++ b/package.json @@ -1,38 +1,52 @@ { - "author": "ql.io", + "author": "cubejs", "contributors": [{ - "name": "Subbu Allamaraju", - "email": "subbu@ebaysf.com" + "name": "Roy Zhou", + "email": "huzhou@ebay.com" }], - "name": "cluster2", - "version": "0.4.20", "repository": { "type": "git", - "url": "https://github.com/ql-io/cluster2" + "url": "https://github.com/cubejs/cluster2.git" }, + "name": "cluster2", + "version": "0.5.0-SNAPSHOT", "engines": { - "node": ">= 0.8.0" + "node": ">= 0.10.0" }, - "main": "lib/index.js", + "main": "index.js", "dependencies": { "underscore": "~1.4.4", - "express": "~2.5.11", + "usage": "~0.3.8", + "when": "~2.3.0", + "winston": "~0.7.2", + "graceful-fs": "~2.0.0", + "request": "~2.21.0", + "express": "~3.1.0", + "socket.io": "~0.9.16", "ejs": "~0.8.4", - "npm": "~1.3.0", - "when": "~2.4.0", - "memwatch": "~0.2.2", - "usage": "~0.3.8" + "node-inspector": "~0.4.0", + "gc-stats": "~0.0.1", + "nodefly-uvmon": "~0.0.7", + "bignumber.js": "~1.1.1", + "npm": "~1.3", + "optimist": "~0.6.0", + "axon": "~1.0.0" }, "devDependencies": { - "websocket": "~1.0.8", - "nodeunit": "~0.8.0", - "request": "~2.21.0", "mocha": "~1.11.0", "should": "~1.2.2", - "harbor": "~0.2.0" + "dustjs-linkedin": "~2.0.3", + "dustjs-helpers": "~1.1.1", + "consolidate": "~0.9.1" }, "scripts": { - "test": "nodeunit test" + "prestart": "rm -rf ./log; mkdir log; rm -rf ./pids; mkdir pids", + "start": "node ./examples/cluster-demo.js --port=9090 --monPort=9091 --noWorkers=2 --cache.enable --heartbeat.interval=5000 &", + "stop": "node shutdown.js", + "pretest": "rm -rf ./log; mkdir log; rm -rf ./pids; mkdir pids", + "test": "mocha --ui bdd --timeout 10s --reporter spec ./test/*-test.js" }, - "optionalDependencies": {} + "publishConfig": { + "registry": "https://registry.npmjs.org" + } } diff --git a/public/css/mon.css b/public/css/mon.css deleted file mode 100644 index 71be738..0000000 --- a/public/css/mon.css +++ /dev/null @@ -1,75 +0,0 @@ -body { - margin-left: 2%; - width: 95%; - font-family: 'Droid Sans', 'Helvetica', 'Arial', sans-serif; - color: #333333; - counter-reset: count-h1; -} - -/*table.header {*/ - /*float: right;*/ - /*font-size: 8pt;*/ - /*text-align: right;*/ -/*}*/ - -.header, .footer { - font-size: 8pt; -} - -div.header { - text-align: right; - font-size: 8pt; -} - -table { - font-size: 10pt; - border-spacing: 2px; -} - -table.master { - width: 90% -} - -table.worker { - width: 90% -} - -td { - padding: 2px; -} - -/*.odd {*/ - /*background: #E8EDFF;*/ -/*}*/ - -.range { - font-size: 8pt; -} - -.range-max { - margin-top: 8px; -} - -.range-min { - margin-bottom: 12px; -} - -table.in-flight { - border: 1px solid #d3d3d3; - border-collapse:collapse; - width: 400px; -} - -table.in-flight td { - border: 1px solid #d3d3d3; - vertical-align: top; - overflow-x: hidden; -} - -table.logs { - width: 100%; -} - -table.logs thead { - font-weight: bold; -} \ No newline at end of file diff --git a/public/scripts/jquery.sparkline.min.js b/public/scripts/jquery.sparkline.min.js deleted file mode 100644 index 628efd2..0000000 --- a/public/scripts/jquery.sparkline.min.js +++ /dev/null @@ -1,94 +0,0 @@ -/* jquery.sparkline 1.6 - http://omnipotent.net/jquery.sparkline/ -** Licensed under the New BSD License - see above site for details */ - -(function($){var defaults={common:{type:'line',lineColor:'#00f',fillColor:'#cdf',defaultPixelsPerValue:3,width:'auto',height:'auto',composite:false,tagValuesAttribute:'values',tagOptionsPrefix:'spark',enableTagOptions:false},line:{spotColor:'#f80',spotRadius:1.5,minSpotColor:'#f80',maxSpotColor:'#f80',lineWidth:1,normalRangeMin:undefined,normalRangeMax:undefined,normalRangeColor:'#ccc',drawNormalOnTop:false,chartRangeMin:undefined,chartRangeMax:undefined,chartRangeMinX:undefined,chartRangeMaxX:undefined},bar:{barColor:'#00f',negBarColor:'#f44',zeroColor:undefined,nullColor:undefined,zeroAxis:undefined,barWidth:4,barSpacing:1,chartRangeMax:undefined,chartRangeMin:undefined,chartRangeClip:false,colorMap:undefined},tristate:{barWidth:4,barSpacing:1,posBarColor:'#6f6',negBarColor:'#f44',zeroBarColor:'#999',colorMap:{}},discrete:{lineHeight:'auto',thresholdColor:undefined,thresholdValue:0,chartRangeMax:undefined,chartRangeMin:undefined,chartRangeClip:false},bullet:{targetColor:'red',targetWidth:3,performanceColor:'blue',rangeColors:['#D3DAFE','#A8B6FF','#7F94FF'],base:undefined},pie:{sliceColors:['#f00','#0f0','#00f']},box:{raw:false,boxLineColor:'black',boxFillColor:'#cdf',whiskerColor:'black',outlierLineColor:'#333',outlierFillColor:'white',medianColor:'red',showOutliers:true,outlierIQR:1.5,spotRadius:1.5,target:undefined,targetColor:'#4a2',chartRangeMax:undefined,chartRangeMin:undefined}};var VCanvas_base,VCanvas_canvas,VCanvas_vml;$.fn.simpledraw=function(width,height,use_existing){if(use_existing&&this[0].VCanvas){return this[0].VCanvas;} -if(width===undefined){width=$(this).innerWidth();} -if(height===undefined){height=$(this).innerHeight();} -if($.browser.hasCanvas){return new VCanvas_canvas(width,height,this);}else if($.browser.msie){return new VCanvas_vml(width,height,this);}else{return false;}};var pending=[];$.fn.sparkline=function(uservalues,userOptions){return this.each(function(){var options=new $.fn.sparkline.options(this,userOptions);var render=function(){var values,width,height;if(uservalues==='html'||uservalues===undefined){var vals=this.getAttribute(options.get('tagValuesAttribute'));if(vals===undefined||vals===null){vals=$(this).html();} -values=vals.replace(/(^\s*\s*$)|\s+/g,'').split(',');}else{values=uservalues;} -width=options.get('width')=='auto'?values.length*options.get('defaultPixelsPerValue'):options.get('width');if(options.get('height')=='auto'){if(!options.get('composite')||!this.VCanvas){var tmp=document.createElement('span');tmp.innerHTML='a';$(this).html(tmp);height=$(tmp).innerHeight();$(tmp).remove();}}else{height=options.get('height');} -$.fn.sparkline[options.get('type')].call(this,values,options,width,height);};if(($(this).html()&&$(this).is(':hidden'))||($.fn.jquery<"1.3.0"&&$(this).parents().is(':hidden'))||!$(this).parents('body').length){pending.push([this,render]);}else{render.call(this);}});};$.fn.sparkline.defaults=defaults;$.sparkline_display_visible=function(){for(var i=pending.length-1;i>=0;i--){var el=pending[i][0];if($(el).is(':visible')&&!$(el).parents().is(':hidden')){pending[i][1].call(el);pending.splice(i,1);}}};var UNSET_OPTION={};var normalizeValue=function(val){switch(val){case'undefined':val=undefined;break;case'null':val=null;break;case'true':val=true;break;case'false':val=false;break;default:var nf=parseFloat(val);if(val==nf){val=nf;}} -return val;};$.fn.sparkline.options=function(tag,userOptions){var extendedOptions;this.userOptions=userOptions=userOptions||{};this.tag=tag;this.tagValCache={};var defaults=$.fn.sparkline.defaults;var base=defaults.common;this.tagOptionsPrefix=userOptions.enableTagOptions&&(userOptions.tagOptionsPrefix||base.tagOptionsPrefix);var tagOptionType=this.getTagSetting('type');if(tagOptionType===UNSET_OPTION){extendedOptions=defaults[userOptions.type||base.type];}else{extendedOptions=defaults[tagOptionType];} -this.mergedOptions=$.extend({},base,extendedOptions,userOptions);};$.fn.sparkline.options.prototype.getTagSetting=function(key){var val,i,prefix=this.tagOptionsPrefix;if(prefix===false||prefix===undefined){return UNSET_OPTION;} -if(this.tagValCache.hasOwnProperty(key)){val=this.tagValCache.key;}else{val=this.tag.getAttribute(prefix+key);if(val===undefined||val===null){val=UNSET_OPTION;}else if(val.substr(0,1)=='['){val=val.substr(1,val.length-2).split(',');for(i=val.length;i--;){val[i]=normalizeValue(val[i].replace(/(^\s*)|(\s*$)/g,''));}}else if(val.substr(0,1)=='{'){var pairs=val.substr(1,val.length-2).split(',');val={};for(i=pairs.length;i--;){var keyval=pairs[i].split(':',2);val[keyval[0].replace(/(^\s*)|(\s*$)/g,'')]=normalizeValue(keyval[1].replace(/(^\s*)|(\s*$)/g,''));}}else{val=normalizeValue(val);} -this.tagValCache.key=val;} -return val;};$.fn.sparkline.options.prototype.get=function(key){var tagOption=this.getTagSetting(key);if(tagOption!==UNSET_OPTION){return tagOption;} -return this.mergedOptions[key];};$.fn.sparkline.line=function(values,options,width,height){var xvalues=[],yvalues=[],yminmax=[];for(var i=0;imaxy){maxy=normalRangeMax;}} -if(options.get('chartRangeMin')!==undefined&&(options.get('chartRangeClip')||options.get('chartRangeMin')maxy)){maxy=options.get('chartRangeMax');} -if(options.get('chartRangeMinX')!==undefined&&(options.get('chartRangeClipX')||options.get('chartRangeMinX')maxx)){maxx=options.get('chartRangeMaxX');} -var rangex=maxx-minx===0?1:maxx-minx;var rangey=maxy-miny===0?1:maxy-miny;var vl=yvalues.length-1;if(vl<1){this.innerHTML='';return;} -var target=$(this).simpledraw(width,height,options.get('composite'));if(target){var canvas_width=target.pixel_width;var canvas_height=target.pixel_height;var canvas_top=0;var canvas_left=0;var spotRadius=options.get('spotRadius');if(spotRadius&&(canvas_width<(spotRadius*4)||canvas_height<(spotRadius*4))){spotRadius=0;} -if(spotRadius){if(options.get('minSpotColor')||(options.get('spotColor')&&yvalues[vl]==miny)){canvas_height-=Math.ceil(spotRadius);} -if(options.get('maxSpotColor')||(options.get('spotColor')&&yvalues[vl]==maxy)){canvas_height-=Math.ceil(spotRadius);canvas_top+=Math.ceil(spotRadius);} -if(options.get('minSpotColor')||options.get('maxSpotColor')&&(yvalues[0]==miny||yvalues[0]==maxy)){canvas_left+=Math.ceil(spotRadius);canvas_width-=Math.ceil(spotRadius);} -if(options.get('spotColor')||(options.get('minSpotColor')||options.get('maxSpotColor')&&(yvalues[vl]==miny||yvalues[vl]==maxy))){canvas_width-=Math.ceil(spotRadius);}} -canvas_height--;var drawNormalRange=function(){if(normalRangeMin!==undefined){var ytop=canvas_top+Math.round(canvas_height-(canvas_height*((normalRangeMax-miny)/rangey)));var height=Math.round((canvas_height*(normalRangeMax-normalRangeMin))/rangey);target.drawRect(canvas_left,ytop,canvas_width,height,undefined,options.get('normalRangeColor'));}};if(!options.get('drawNormalOnTop')){drawNormalRange();} -var path=[];var paths=[path];var x,y,vlen=yvalues.length;for(i=0;imaxy){y=maxy;} -if(!path.length){path.push([canvas_left+Math.round((x-minx)*(canvas_width/rangex)),canvas_top+canvas_height]);} -path.push([canvas_left+Math.round((x-minx)*(canvas_width/rangex)),canvas_top+Math.round(canvas_height-(canvas_height*((y-miny)/rangey)))]);}} -var lineshapes=[];var fillshapes=[];var plen=paths.length;for(i=0;i2){path[0]=[path[0][0],path[1][1]];} -lineshapes.push(path);} -plen=fillshapes.length;for(i=0;imax)){max=options.get('chartRangeMax');} -var zeroAxis=options.get('zeroAxis');if(zeroAxis===undefined){zeroAxis=min<0;} -var range=max-min===0?1:max-min;var colorMapByIndex,colorMapByValue;if($.isArray(options.get('colorMap'))){colorMapByIndex=options.get('colorMap');colorMapByValue=null;}else{colorMapByIndex=null;colorMapByValue=options.get('colorMap');} -var target=$(this).simpledraw(width,height,options.get('composite'));if(target){var color,canvas_height=target.pixel_height,yzero=min<0&&zeroAxis?canvas_height-Math.round(canvas_height*(Math.abs(min)/range))-1:canvas_height-1;for(i=values.length;i--;){var x=i*(options.get('barWidth')+options.get('barSpacing')),y,val=values[i];if(val===null){if(options.get('nullColor')){color=options.get('nullColor');val=(zeroAxis&&min<0)?0:min;height=1;y=(zeroAxis&&min<0)?yzero:canvas_height-height;}else{continue;}}else{if(valmax){val=max;} -color=(val<0)?options.get('negBarColor'):options.get('barColor');if(zeroAxis&&min<0){height=Math.round(canvas_height*((Math.abs(val)/range)))+1;y=(val<0)?yzero:yzero-height;}else{height=Math.round(canvas_height*((val-min)/range))+1;y=canvas_height-height;} -if(val===0&&options.get('zeroColor')!==undefined){color=options.get('zeroColor');} -if(colorMapByValue&&colorMapByValue[val]){color=colorMapByValue[val];}else if(colorMapByIndex&&colorMapByIndex.length>i){color=colorMapByIndex[i];} -if(color===null){continue;}} -target.drawRect(x,y,options.get('barWidth')-1,height-1,color,color);}}else{this.innerHTML='';}};$.fn.sparkline.tristate=function(values,options,width,height){values=$.map(values,Number);width=(values.length*options.get('barWidth'))+((values.length-1)*options.get('barSpacing'));var colorMapByIndex,colorMapByValue;if($.isArray(options.get('colorMap'))){colorMapByIndex=options.get('colorMap');colorMapByValue=null;}else{colorMapByIndex=null;colorMapByValue=options.get('colorMap');} -var target=$(this).simpledraw(width,height,options.get('composite'));if(target){var canvas_height=target.pixel_height,half_height=Math.round(canvas_height/2);for(var i=values.length;i--;){var x=i*(options.get('barWidth')+options.get('barSpacing')),y,color;if(values[i]<0){y=half_height;height=half_height-1;color=options.get('negBarColor');}else if(values[i]>0){y=0;height=half_height-1;color=options.get('posBarColor');}else{y=half_height-1;height=2;color=options.get('zeroBarColor');} -if(colorMapByValue&&colorMapByValue[values[i]]){color=colorMapByValue[values[i]];}else if(colorMapByIndex&&colorMapByIndex.length>i){color=colorMapByIndex[i];} -if(color===null){continue;} -target.drawRect(x,y,options.get('barWidth')-1,height-1,color,color);}}else{this.innerHTML='';}};$.fn.sparkline.discrete=function(values,options,width,height){values=$.map(values,Number);width=options.get('width')=='auto'?values.length*2:width;var interval=Math.floor(width/values.length);var target=$(this).simpledraw(width,height,options.get('composite'));if(target){var canvas_height=target.pixel_height,line_height=options.get('lineHeight')=='auto'?Math.round(canvas_height*0.3):options.get('lineHeight'),pheight=canvas_height-line_height,min=Math.min.apply(Math,values),max=Math.max.apply(Math,values);if(options.get('chartRangeMin')!==undefined&&(options.get('chartRangeClip')||options.get('chartRangeMin')max)){max=options.get('chartRangeMax');} -var range=max-min;for(var i=values.length;i--;){var val=values[i];if(valmax){val=max;} -var x=(i*interval),ytop=Math.round(pheight-pheight*((val-min)/range));target.drawLine(x,ytop,x,ytop+line_height,(options.get('thresholdColor')&&val1){var canvas_width=target.pixel_width-Math.ceil(options.get('targetWidth')/2),canvas_height=target.pixel_height,min=Math.min.apply(Math,values),max=Math.max.apply(Math,values);if(options.get('base')===undefined){min=min<0?min:0;}else{min=options.get('base');} -var range=max-min;for(var i=2,vlen=values.length;i1){var canvas_width=target.pixel_width,canvas_height=target.pixel_height,radius=Math.floor(Math.min(canvas_width,canvas_height)/2),total=0,next=0,circle=2*Math.PI;for(var i=values.length;i--;){total+=values[i];} -if(options.get('offset')){next+=(2*Math.PI)*(options.get('offset')/360);} -var vlen=values.length;for(i=0;i0){end=next+(circle*(values[i]/total));} -target.drawPieSlice(radius,radius,radius,start,end,undefined,options.get('sliceColors')[i%options.get('sliceColors').length]);next=end;}}};var quartile=function(values,q){if(q==2){var vl2=Math.floor(values.length/2);return values.length%2?values[vl2]:(values[vl2]+values[vl2+1])/2;}else{var vl4=Math.floor(values.length/4);return values.length%2?(values[vl4*q]+values[vl4*q+1])/2:values[vl4*q];}};$.fn.sparkline.box=function(values,options,width,height){values=$.map(values,Number);width=options.get('width')=='auto'?'4.0em':width;var minvalue=options.get('chartRangeMin')===undefined?Math.min.apply(Math,values):options.get('chartRangeMin'),maxvalue=options.get('chartRangeMax')===undefined?Math.max.apply(Math,values):options.get('chartRangeMax'),target=$(this).simpledraw(width,height,options.get('composite')),vlen=values.length,lwhisker,loutlier,q1,q2,q3,rwhisker,routlier;if(target&&values.length>1){var canvas_width=target.pixel_width,canvas_height=target.pixel_height;if(options.get('raw')){if(options.get('showOutliers')&&values.length>5){loutlier=values[0];lwhisker=values[1];q1=values[2];q2=values[3];q3=values[4];rwhisker=values[5];routlier=values[6];}else{lwhisker=values[0];q1=values[1];q2=values[2];q3=values[3];rwhisker=values[4];}}else{values.sort(function(a,b){return a-b;});q1=quartile(values,1);q2=quartile(values,2);q3=quartile(values,3);var iqr=q3-q1;if(options.get('showOutliers')){lwhisker=undefined;rwhisker=undefined;for(var i=0;iq1-(iqr*options.get('outlierIQR'))){lwhisker=values[i];} -if(values[i]rwhisker){target.drawCircle((routlier-minvalue)*unitsize+canvas_left,canvas_height/2,options.get('spotRadius'),options.get('outlierLineColor'),options.get('outlierFillColor'));}} -target.drawRect(Math.round((q1-minvalue)*unitsize+canvas_left),Math.round(canvas_height*0.1),Math.round((q3-q1)*unitsize),Math.round(canvas_height*0.8),options.get('boxLineColor'),options.get('boxFillColor'));target.drawLine(Math.round((lwhisker-minvalue)*unitsize+canvas_left),Math.round(canvas_height/2),Math.round((q1-minvalue)*unitsize+canvas_left),Math.round(canvas_height/2),options.get('lineColor'));target.drawLine(Math.round((lwhisker-minvalue)*unitsize+canvas_left),Math.round(canvas_height/4),Math.round((lwhisker-minvalue)*unitsize+canvas_left),Math.round(canvas_height-canvas_height/4),options.get('whiskerColor'));target.drawLine(Math.round((rwhisker-minvalue)*unitsize+canvas_left),Math.round(canvas_height/2),Math.round((q3-minvalue)*unitsize+canvas_left),Math.round(canvas_height/2),options.get('lineColor'));target.drawLine(Math.round((rwhisker-minvalue)*unitsize+canvas_left),Math.round(canvas_height/4),Math.round((rwhisker-minvalue)*unitsize+canvas_left),Math.round(canvas_height-canvas_height/4),options.get('whiskerColor'));target.drawLine(Math.round((q2-minvalue)*unitsize+canvas_left),Math.round(canvas_height*0.1),Math.round((q2-minvalue)*unitsize+canvas_left),Math.round(canvas_height*0.9),options.get('medianColor'));if(options.get('target')){var size=Math.ceil(options.get('spotRadius'));target.drawLine(Math.round((options.get('target')-minvalue)*unitsize+canvas_left),Math.round((canvas_height/2)-size),Math.round((options.get('target')-minvalue)*unitsize+canvas_left),Math.round((canvas_height/2)+size),options.get('targetColor'));target.drawLine(Math.round((options.get('target')-minvalue)*unitsize+canvas_left-size),Math.round(canvas_height/2),Math.round((options.get('target')-minvalue)*unitsize+canvas_left+size),Math.round(canvas_height/2),options.get('targetColor'));}}else{this.innerHTML='';}};if($.browser.msie&&!document.namespaces.v){document.namespaces.add('v','urn:schemas-microsoft-com:vml','#default#VML');} -if($.browser.hasCanvas===undefined){var t=document.createElement('canvas');$.browser.hasCanvas=t.getContext!==undefined;} -VCanvas_base=function(width,height,target){};VCanvas_base.prototype={init:function(width,height,target){this.width=width;this.height=height;this.target=target;if(target[0]){target=target[0];} -target.VCanvas=this;},drawShape:function(path,lineColor,fillColor,lineWidth){alert('drawShape not implemented');},drawLine:function(x1,y1,x2,y2,lineColor,lineWidth){return this.drawShape([[x1,y1],[x2,y2]],lineColor,lineWidth);},drawCircle:function(x,y,radius,lineColor,fillColor){alert('drawCircle not implemented');},drawPieSlice:function(x,y,radius,startAngle,endAngle,lineColor,fillColor){alert('drawPieSlice not implemented');},drawRect:function(x,y,width,height,lineColor,fillColor){alert('drawRect not implemented');},getElement:function(){return this.canvas;},_insert:function(el,target){$(target).html(el);}};VCanvas_canvas=function(width,height,target){return this.init(width,height,target);};VCanvas_canvas.prototype=$.extend(new VCanvas_base(),{_super:VCanvas_base.prototype,init:function(width,height,target){this._super.init(width,height,target);this.canvas=document.createElement('canvas');if(target[0]){target=target[0];} -target.VCanvas=this;$(this.canvas).css({display:'inline-block',width:width,height:height,verticalAlign:'top'});this._insert(this.canvas,target);this.pixel_height=$(this.canvas).height();this.pixel_width=$(this.canvas).width();this.canvas.width=this.pixel_width;this.canvas.height=this.pixel_height;$(this.canvas).css({width:this.pixel_width,height:this.pixel_height});},_getContext:function(lineColor,fillColor,lineWidth){var context=this.canvas.getContext('2d');if(lineColor!==undefined){context.strokeStyle=lineColor;} -context.lineWidth=lineWidth===undefined?1:lineWidth;if(fillColor!==undefined){context.fillStyle=fillColor;} -return context;},drawShape:function(path,lineColor,fillColor,lineWidth){var context=this._getContext(lineColor,fillColor,lineWidth);context.beginPath();context.moveTo(path[0][0]+0.5,path[0][1]+0.5);for(var i=1,plen=path.length;i';this.canvas.insertAdjacentHTML('beforeEnd',groupel);this.group=$(this.canvas).children()[0];},drawShape:function(path,lineColor,fillColor,lineWidth){var vpath=[];for(var i=0,plen=path.length;i'+' ';this.group.insertAdjacentHTML('beforeEnd',vel);},drawCircle:function(x,y,radius,lineColor,fillColor){x-=radius+1;y-=radius+1;var stroke=lineColor===undefined?' stroked="false" ':' strokeWeight="1" strokeColor="'+lineColor+'" ';var fill=fillColor===undefined?' filled="false"':' fillColor="'+fillColor+'" filled="true" ';var vel='';this.group.insertAdjacentHTML('beforeEnd',vel);},drawPieSlice:function(x,y,radius,startAngle,endAngle,lineColor,fillColor){if(startAngle==endAngle){return;} -if((endAngle-startAngle)==(2*Math.PI)){startAngle=0.0;endAngle=(2*Math.PI);} -var startx=x+Math.round(Math.cos(startAngle)*radius);var starty=y+Math.round(Math.sin(startAngle)*radius);var endx=x+Math.round(Math.cos(endAngle)*radius);var endy=y+Math.round(Math.sin(endAngle)*radius);if(startx==endx&&starty==endy&&(endAngle-startAngle)'+' ';this.group.insertAdjacentHTML('beforeEnd',vel);},drawRect:function(x,y,width,height,lineColor,fillColor){return this.drawShape([[x,y],[x,y+height],[x+width,y+height],[x+width,y],[x,y]],lineColor,fillColor);}});})(jQuery); diff --git a/public/scripts/mon-client.js b/public/scripts/mon-client.js deleted file mode 100644 index 07f8736..0000000 --- a/public/scripts/mon-client.js +++ /dev/null @@ -1,79 +0,0 @@ -$(document).ready(function() { - var wsCtor = window['MozWebSocket'] ? MozWebSocket : WebSocket; - var currentMemoryUsage = [], hostCpu = [], pendingInRequests = [], pendingOutRequests = []; - - var uri = 'ws://' + document.domain; - uri = uri + ':' + (document.location.protocol === 'https:' ? 443 : document.location.port); - var socket = new wsCtor(uri, 'ql.io-mon'); - socket.onmessage = function(e) { - var json; - try { - json = JSON.parse(e.data); - $('#hostCpu').html(json.master.hostCpu); - hostCpu.push(json.master.hostCpu); - if(hostCpu.length > 120) { - hostCpu = hostCpu.slice(1); - } - - $('#currentMemoryUsage').html(json.master.currentMemoryUsage); - currentMemoryUsage.push(json.master.currentMemoryUsage); - if(currentMemoryUsage.length > 120) { - currentMemoryUsage = currentMemoryUsage.slice(1); - } - - var pendingOut = 0, pendingIn = 0; - $('#averageLoad').html(json.master.averageLoad); - $.each(json.master.workers, function(i, worker) { - $('#connectionsTotal' + '-' + i).html(worker.connectionsTotal); - $('#connectionsActive' + '-' + i).html(worker.connectionsActive); - $('#requestsTotal-' + i).html(worker.requestsTotal); - $('#inRequests-' + i).html(worker.inRequests); - $('#outResponses-' + i).html(worker.outResponses); - $('#outRequests-' + i).html(worker.outRequests); - $('#inResponses-' + i).html(worker.inResponses); - $('#activeInRequests-' + i).html(worker.inRequests - worker.outResponses); - $('#activeOutRequests-' + i).html(worker.outRequests - worker.inResponses); - pendingIn = pendingIn + worker.inRequests - worker.outResponses; - pendingOut = pendingOut + worker.outRequests - worker.inResponses; - }); - - $('#inRequests').html(json.master.inRequests); - $('#outResponses').html(json.master.outResponses); - $('#outRequests').html(json.master.outRequests); - $('#inResponses').html(json.master.inResponses); - - pendingInRequests.push(pendingIn); - if(pendingInRequests.length > 120) { - pendingInRequests = pendingInRequests.slice(1); - } - $('#master-in-pending').sparkline(pendingInRequests, { height: 100, width: 800, type:'bar', barColor:'red' }); - $('#master-pending-in-min').html(Math.min.apply(Math, pendingInRequests)); - $('#master-pending-in-max').html(Math.max.apply(Math, pendingInRequests)); - pendingOutRequests.push(pendingOut); - if(pendingOutRequests.length > 120) { - pendingOutRequests = pendingOutRequests.slice(1); - } - $('#master-out-pending').sparkline(pendingOutRequests, { height: 100, width: 800, type:'bar', barColor:'green' }); - $('#master-pending-out-min').html(Math.min.apply(Math, pendingOutRequests)); - $('#master-pending-out-max').html(Math.max.apply(Math, pendingOutRequests)); - } - catch(e) { - alert(e) - } - } - - function forMemoryNum(memory) { - var strMemory; - if(memory < 1024) { - strMemory = memory + ' Bytes'; - } - if(memory < 1024 * 1024) { - strMemory = (memory / 1024).toFixed(2) + ' KB'; - } - else { - strMemory = (memory / (1024 * 1024)).toFixed(2) + ' MB'; - } - return strMemory; - } -}); - diff --git a/public/views/footer.ejs b/public/views/footer.ejs deleted file mode 100644 index 44c205d..0000000 --- a/public/views/footer.ejs +++ /dev/null @@ -1,3 +0,0 @@ -
-

©2012 eBay Software Foundation.

-
\ No newline at end of file diff --git a/public/views/header.ejs b/public/views/header.ejs deleted file mode 100644 index 9eb557f..0000000 --- a/public/views/header.ejs +++ /dev/null @@ -1,19 +0,0 @@ - diff --git a/public/views/index.ejs b/public/views/index.ejs deleted file mode 100644 index 9c0bec4..0000000 --- a/public/views/index.ejs +++ /dev/null @@ -1,56 +0,0 @@ -

Master

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
Number of workers <%=master.noWorkers %> (<%=master.workersKilled%> killed)
Number of core used <%=master.coresUsed%>
Mem usage at startup <%=master.memoryUsageAtBoot%>
Current total mem usage <%=master.currentMemoryUsage%>
CPU usage <%=master.hostCpu%>%
Average load - <%=master.averageLoad%>
- -

Logs

- -Logs - -

Counters

- -
- - <% for(var pid in master.workers) { %> - - - - - <% for(var name in master.workers[pid]) { %> - - - - - <% } %> - - <% } %> -
Worker <%= pid %>
<%=name%><%=master.workers[pid][name]%>
-
- - diff --git a/public/views/layout.ejs b/public/views/layout.ejs deleted file mode 100644 index 041a422..0000000 --- a/public/views/layout.ejs +++ /dev/null @@ -1,99 +0,0 @@ - - - - Runtime Monitor - - - - - - - - - - - -<%- partial("header") %> - -
-<%- body %> - -
-<%- partial("footer") %> -
-
- - diff --git a/public/views/logs.ejs b/public/views/logs.ejs deleted file mode 100644 index d9a0e38..0000000 --- a/public/views/logs.ejs +++ /dev/null @@ -1,22 +0,0 @@ -

Log Files

- - - - - - - - - - - - <% for(var i = 0; i < logs.length; i++) { %> - - - - - - - <% } %> - -
FileSizeCreatedLast Modifled
<%=logs[i].filename%><%=logs[i].stats.size%><%=logs[i].stats.ctime%><%=logs[i].stats.mtime%>
diff --git a/shutdown.js b/shutdown.js new file mode 100644 index 0000000..9c8c586 --- /dev/null +++ b/shutdown.js @@ -0,0 +1,67 @@ +'use strict'; + +var utils = require('./lib/utils'), + readMasterPid = utils.readMasterPid, + readPids = utils.readPids, + safeKill = utils.safeKill, + getLogger = utils.getLogger, + optimist = require('optimist'), + when = require('when'), + path = require('path'), + _ = require('underscore'); + +var argv = optimist.argv, + timeout = argv.timeout || 60000,//user specified timeout or 1 min + pids = argv.pids || path.join(process.cwd(), '/pids'), + masterPid = argv.pid || readMasterPid(pids), + workerPids = _.filter(readPids(pids) || [], function(pid){ + return pid !== masterPid; + }), + logger = getLogger(); + +logger.info('[shutdown] SIGINT:%d monitor:%j', masterPid, workerPids); + +(function shutdown(begin){ + + if(!safeKill(masterPid, 'SIGINT', logger)){//master will handle 'SIGINT' only once + + if(Date.now() - begin >= timeout){ + + _.each(workerPids, function(wpid){ + //bruteforcely kill all workers + safeKill(wpid, 'SIGTERM', logger); + }); + + //bruteforcely kill master + safeKill(masterPid, 'SIGTERM', logger); + } + + setTimeout(_.bind(shutdown, null, begin), 1000);//check every seconds + } + else{//master is finally gone, we'll quickly check whether all workers are gone too + + when.map(workerPids, function(wpid){ + + var tillWorkerGone = when.defer(); + + if(!safeKill(wpid, 'SIGHUP', logger)){ + logger.warn('[shutdown] cleanup found dangling worker:%d', wpid); + tillWorkerGone.reject(new Error('pid:' + wpid + ' still lives')); + } + else{ + //worker already exit, check next + tillWorkerGone.resolve(true); + } + }) + .then(function(){ + //all workers exit normally + process.exit(0); + }) + .otherwise(function(){ + //some worker didn't exit though master already did + process.exit(-1); + }); + } + +})(Date.now()); + diff --git a/status.js b/status.js new file mode 100644 index 0000000..a0129a6 --- /dev/null +++ b/status.js @@ -0,0 +1,6 @@ +'use strict'; + +var cluster2 = process.cluster2 = process.cluster2 || {}; +cluster2.status = cluster2.status || require('./lib/status'); + +module.exports = cluster2.status; \ No newline at end of file diff --git a/test/api-test.js b/test/api-test.js new file mode 100644 index 0000000..7412904 --- /dev/null +++ b/test/api-test.js @@ -0,0 +1,52 @@ +'use strict'; + +var should = require('should'), + getLogger = require('../lib/utils').getLogger; + +describe('cluster2', function(){ + + before(function(done){ + + process.getLogger = getLogger; + + done(); + }); + + describe('#isMaster', function(){ + + it('should assert true', function(done){ + + var cluster2 = require('../index'); + cluster2.isMaster.should.equal(true); + cluster2.isWorker.should.equal(false); + cluster2.emitter.should.be.ok; + cluster2.cacheManager.should.be.ok; + cluster2.status.should.be.ok; + + done(); + + }); + + }); + + describe('#listen', function(){ + + it('should start listening app', function(done){ + + done(); + + }); + + }); + + describe('#run', function(){ + + it('should start running the given runnable', function(done){ + + done(); + + }); + + }); + +}); \ No newline at end of file diff --git a/test/cache-socket-test.js b/test/cache-socket-test.js new file mode 100644 index 0000000..aaf9e0b --- /dev/null +++ b/test/cache-socket-test.js @@ -0,0 +1,58 @@ +'use strict'; + +var should = require('should'); +var cacheSocketFactory = require('../lib/cache-socket/cache-socket-factory.js'); +var utils = require('../lib/utils.js'); +var cacheMgrSocket = cacheSocketFactory.getCacheSocket('manager', 'json'); +var cacheUsrSocket = cacheSocketFactory.getCacheSocket('user', 'json'); + +describe('Test cache socket', function () { + + before(function (done) { + utils.pickAvailablePorts(9190, 9290, 2).then(function (ports) { + console.log('pickup ports %j', ports); + cacheMgrSocket.listen(ports, function (error) { + if (error) { + return done(error); + }else { + cacheUsrSocket.connect(ports, function (error) { + if (error) { + return done(error); + }else { + return done(); + } + }); + } + }); + }, done); + }); + + after(function (done) { + cacheMgrSocket.close(); + cacheUsrSocket.close(); + done(); + }); + + it ('Manager should be able to get request and reply', function (done) { + cacheMgrSocket.on('message', function (msg, reply) { + msg.should.be.ok; + msg.hello.should.equal('hello'); + reply('world'); + }); + + cacheUsrSocket.send({hello: 'hello'}, function (back) { + back.should.equal('world'); + return done(); + }); + }); + + it ('User should be able to get notification', function (done) { + cacheUsrSocket.on('message', function (msg) { + msg.should.be.ok; + msg.hello.should.equal('notification'); + done(); + }); + + cacheMgrSocket.send({hello: 'notification'}); + }); +}); diff --git a/test/cache-test.js b/test/cache-test.js new file mode 100644 index 0000000..fbea155 --- /dev/null +++ b/test/cache-test.js @@ -0,0 +1,206 @@ +'use strict'; + +var should = require('should'), + getLogger = require('../lib/utils').getLogger, + logger = getLogger(__filename), + fs = require('fs'); + +describe('cache', function(){ + + + before(function(done){ + + process.getLogger = getLogger; + + require('../lib/cache').enable({ + 'enable': true + }).then(function (resolved) { + return done(); + }).otherwise(function (error) { + return done(error); + }); + }); + + after(function (done) { + fs.unlinkSync('cluster-cache-domain'); + process.cacheServer.close(function (err) { + if (err) { + return done(err); + } + return done(); + }); + }); + + describe('#cache-user', function(){ + + it('should auto connect to mgr', function(done){ + + this.timeout(3000); + + require('../lib/cache-usr.js').user() + .then(function(usr){ + + done(); + }); + }); + + it('should support all ACID operations', function(done){ + + this.timeout(5000); + + require('../lib/cache-usr').user() + .then(function(usr){ + + var namespace = 'ns-' + Date.now(); + logger.info('[test] using namespace:%s', namespace); + + usr.get(namespace, 'key') + .then(function(value){ + + logger.info('[test] first "key" get attempt should fail:%j', value); + should.not.exist(value); + + usr.watch(namespace, 'key', function(value, key){ + + value.should.equal('value'); + + logger.info('[test] watch key triggered'); + }); + + usr.watch(namespace, null, function(value, key){ + + key.should.equal('key'); + value.should.equal('value'); + + logger.info('[test] watch all triggered'); + + }); + + usr.set(namespace, 'key', 'value') + .then(function(set){ + + logger.info('[test] first "key" set attempt should succeed:%s', set); + set.should.equal(true); + + usr.get(namespace, 'key') + .then(function(value){ + + logger.info('[test] 2nd "key" get attempt should succeed:%j', value); + value.should.be.ok; + value.should.equal('value'); + + usr.inspect(namespace, 'key') + .then(function(inspection){ + + logger.info('[test] inspecting "key" got:%j', inspection); + inspection.should.be.ok; + inspection.length.should.equal(3); + + //value, persist, expire + inspection[0].should.equal('value'); + inspection[1].should.equal(false); + inspection[2].should.equal(0); + + var stat = usr.stat(namespace); + stat.should.be.ok; + stat.hit.should.equal(1); + stat.miss.should.equal(1); + stat.load.should.equal(0); + stat.error.should.equal(0); + + done(); + + }, done); + + }, done); + + }, done); + + }, done); + }); + }); + }); + + describe('#use', function(){ + + + it('should give a Cache interface back which hides the cache-usr behind', function(done){ + + this.timeout(5000); + + var namespace = 'use-ns-' + Date.now(), + persist = true, + expire = 3000; + + var cache = require('../lib/cache').use(namespace, { + 'persist': persist, + 'expire': expire + }); + + logger.info('[test] cache obtained:%j', cache); + + cache.should.be.ok; + cache.namespace.should.equal(namespace); //test namespace + + cache.meta().then(function(meta){ //test meta + + meta.should.be.ok; + meta.persist.should.equal(true); + + logger.info('[test] cache queries begin'); + + cache.get('key') //test get without loader + .then(function(value){ + + logger.info('[test] cache 1st "get" attempt should value:%j', value); + + should.not.exist(value); + + cache.get('key', function(){ //test get with loader + + return 'value'; + }) + .then(function(value){ + + logger.info('[test] cache 2nd "get" with loader attempt should succeed given value:%j', value); + + value.should.equal('value'); + + cache.keys().then(function(keys){ //test get keys + + keys.should.be.ok; + keys.should.include('key'); + + cache.stat().then(function(stat){ //test stats + + stat.should.be.ok; + stat.hit.should.equal(0); + stat.miss.should.equal(2); + stat.load.should.equal(1); + stat.error.should.equal(0); + + cache.watch('key', function(v, k){ //test watch + + done(); + + }) + .then(function(){ + + cache.set('key', 'value-updated'); //test set & watch + + }); + + }, done); + + }); + + }, done); + + }, done); + + }, done); + }); + + }); + +}); diff --git a/test/cluster-cache-test.js b/test/cluster-cache-test.js new file mode 100644 index 0000000..b33d593 --- /dev/null +++ b/test/cluster-cache-test.js @@ -0,0 +1,131 @@ +'use strict'; + +var request = require('request'); +var should = require('should'); +var fork = require('child_process').fork; +var when = require('when'); +var parallel = require('when/parallel'); +var util = require('util'); +var utils = require('../lib/utils.js'); + +describe('Cache Performance Test', function () { + + var childProc; + var key = 'cache-test-key'; + var value = 'cache-test-value'; + var port; + var writeTask = function () { + var deferred = when.defer(); + request.get(util.format('http://127.0.0.1:%d/set?key=%s&value=%s', port, key, value), function (err, res, body) { + if (!err && res.statusCode === 200 && body === value) { + deferred.resolve(body); + }else { + deferred.reject('cache set error'); + } + }); + return deferred.promise; + }; + var readTask = function () { + var deferred = when.defer(); + request.get(util.format('http://127.0.0.1:%d/get?key=%s', port, key), function (err, res, body) { + if (!err && res.statusCode === 200 && (body === value || body === 'cache-test')) { + deferred.resolve(body); + }else { + deferred.reject('cache get error'); + } + }); + return deferred.promise; + }; + + beforeEach(function (done) { + this.timeout(10000); + var token = 't-' + Date.now(); + utils.pickAvailablePorts(9090, 9190, 2).then(function (ports) { + port = ports[0]; + childProc = fork(require.resolve('./lib/cluster-cache-runtime.js'), ['--token=' + token], {env: {port: ports[0], monPort: ports[1]}}); + childProc.on('message', function (msg) { + if (msg.ready) { + return done(); + } + if (msg.err) { + return done(err); + } + }); + }).otherwise(function (err) { + return done(err); + }); + }); + + afterEach(function (done) { + this.timeout(5000); + childProc.kill('SIGTERM'); + require('fs').unlinkSync('cluster-cache-domain'); + setTimeout(done, 4000); + }); + + describe('# 90% read, %10 write', function () { + this.timeout(10000); + var tasks = []; + for (var i=0; i<20; i++) { + if (i % 10 === 0) { + tasks.push(writeTask); + }else { + tasks.push(readTask); + } + } + it('Should resolve all the promises', function (done) { + var startTime = Date.now(); + parallel(tasks).then(function (values) { + var duration = Date.now() - startTime; + console.log(duration / 1000); + done(); + }).otherwise(function (err) { + done(err); + }); + }); + }); + + describe('# 50% read, 50% write', function () { + this.timeout(10000); + var tasks = []; + for (var i=0; i<20; i++) { + if (i % 10 < 5) { + tasks.push(writeTask); + }else { + tasks.push(readTask); + } + } + it('Should resolve all the promises', function (done) { + var startTime = Date.now(); + parallel(tasks).then(function (values) { + var duration = Date.now() - startTime; + console.log(duration / 1000); + done(); + }).otherwise(function (err) { + done(err); + }); + }); + }); + + describe('# 10% read, 90% write', function () { + this.timeout(10000); + var tasks = []; + for (var i=0; i<20; i++) { + if (i % 10 < 9) { + tasks.push(writeTask); + }else { + tasks.push(readTask); + } + } + it('Should resolve all the promises', function (done) { + var startTime = Date.now(); + parallel(tasks).then(function (values) { + var duration = Date.now() - startTime; + console.log(duration / 1000); + done(); + }).otherwise(function (err) { + done(err); + }); + }); + }); +}); diff --git a/test/cluster-emitter-test.js b/test/cluster-emitter-test.js new file mode 100644 index 0000000..f0b6230 --- /dev/null +++ b/test/cluster-emitter-test.js @@ -0,0 +1,77 @@ +'use strict'; + +var should = require('should'), + _ = require('underscore'), + fork = require('child_process').fork, + getLogger = require('../lib/utils').getLogger; + +describe('cluster-emitter', function(){ + + before(function(done){ + + process.getLogger = getLogger; + done(); + }); + + describe('master-emitter', function(){ + + it('should work in none-cluster mode', function(done){ + + this.timeout(500); + + var emitter = require('../lib/emitter'); + + emitter.should.be.ok; + _.isFunction(emitter.emit).should.equal(true); + _.isFunction(emitter.on).should.equal(true); + _.isFunction(emitter.once).should.equal(true); + _.isFunction(emitter.removeListener).should.equal(true); + _.isFunction(emitter.removeAllListeners).should.equal(true); + + var event = 'event-' + Date.now(), + echo = 'echo-' + event; + + emitter.once(echo, function(){//test once + + done(); + }); + + emitter.on(event, function(){//test on + + emitter.emit(echo); + }); + + emitter.emit(event);//test emit + }); + + }); + + //now we'll need to verify the test in cluster mode, for master emitter the behavior should be: + //emit is to send the event to both master & all workers (we'll prepare a couple of workers at least 2 just to make sure) + //on/once should listen to events emitted by either master or workers + //removeListener/removeAllListeners should revoke all the listeners triggered above + + //for the slave emitter, the behavior should be: + //emit is to sent the event to both master & the worker itself + //on/once should listen to events emitted from either master or workers + //removeListener/removeAllListeners should revoke all the listeners triggered abovesss + + describe('cluster-emitter', function(){ + + it('should work in cluster mode', function(done){ + + this.timeout(3000); + + var token = 't-' + Date.now(), + clusterRuntime = fork(require.resolve('./lib/cluster-emitter-runtime'), ['--token=' + token]); + + clusterRuntime.on('message', function(msg){ + + done(msg.exit); + + }); + + }); + }); + +}); \ No newline at end of file diff --git a/test/cluster-pause-resume-test.js b/test/cluster-pause-resume-test.js new file mode 100644 index 0000000..a8b7953 --- /dev/null +++ b/test/cluster-pause-resume-test.js @@ -0,0 +1,73 @@ +'use strict'; + +var request = require('request'); +var should = require('should'); +var fork = require('child_process').fork; +var utils = require('../lib/utils'); + +describe('Test Pause and Resume the Worker', function () { + + var childProc; + var port; + + before(function (done) { + var token = 't-' + Date.now(); + utils.pickAvailablePorts(9090, 9190, 2).then(function (ports) { + port = ports[0] + childProc = fork(require.resolve('./lib/cluster-pause-resume-runtime.js'), ['--token=' + token], {env: {port: ports[0], monPort: ports[1]}}); + childProc.once('message', function (msg) { + if (msg.ready) { + return done(); + }else if (msg.err) { + console.log(msg.err); + return done(msg.err); + } + }); + }).otherwise(function (err) { + return done(err); + }); + }); + + after(function (done) { + childProc.kill('SIGTERM'); + done(); + }); + + it('Should pause the worker', function (done) { + this.timeout(5000); + childProc.send({operation: 'pause'}); + + childProc.once('message', function (msg) { + if (msg.paused) { + request.get({ + url: 'http://127.0.0.1:' + port + '/sayHello', + timeout: 4000 + }, function (err, res, body) { + if (err) { + done(); + }else { + done(new Error('Should be error here')); + } + }); + } + }); + }); + + it('should resume the worker', function (done) { + this.timeout(5000); + childProc.send({operation: 'resume'}); + + childProc.once('message', function (msg) { + if (msg.resumed) { + request.get('http://127.0.0.1:' + port + '/sayHello', function (err, res, body) { + if (err) { + done(err); + } + res.statusCode.should.equal(200); + body.should.equal('hello'); + done(); + }); + } + }); + }); +}); diff --git a/test/cluster-status-test.js b/test/cluster-status-test.js new file mode 100644 index 0000000..55f7b00 --- /dev/null +++ b/test/cluster-status-test.js @@ -0,0 +1,156 @@ +'use strict'; + +var should = require('should'), + fork = require('child_process').fork, + getLogger = require('../lib/utils').getLogger; + +describe('cluster-status', function(){ + + before(function(done){ + + process.getLogger = getLogger; + done(); + }); + + describe('#statuses', function(){ + + it('should get no status at the beginning', function(done){ + + var status = require('../lib/status'); + + status.statuses().should.be.ok; + status.statuses().length.should.equal(0); + + done(); + + }); + + }); + + describe('#register', function(){ + + it('should allow register of an immutable component', function(done){ + + var status = require('../lib/status'), + name = 'immutable-status-' + Date.now(), + view = 'view-' + name; + + status.getStatus(name).then(done, function(){ + + status.register(name, function(){ + return view; + }); + + status.getStatus(name).then(function(result){ + + result.should.be.ok; + result.length.should.equal(1); + + var stat = result.shift(); + stat.should.be.ok; + stat.pid.should.equal(process.pid); + stat.name.should.equal(name); + stat.status.should.equal(view); + + status.setStatus(name, 'noop').then(function(){ //as we didn't register update, setStatus should have no effect at all. + + status.getStatus(name).then(function(result){ + + result.should.be.ok; + result.length.should.equal(1); + + var stat = result.shift(); + stat.should.be.ok; + stat.pid.should.equal(process.pid); + stat.name.should.equal(name); + stat.status.should.equal(view); + + done(); + + }); + + }, done); + + }, done); + }); + }); + + it('should allow register of a mutable component', function(done){ + + var status = require('../lib/status'), + name = 'mutable-status-' + Date.now(), + view = 'view-' + name, + updated = 'updated-' + name; + + status.getStatus(name).then(done, function(){ + + status.register(name, function(){ + return view; + }, + function(update){ + view = update; + }); + + status.getStatus(name).then(function(result){ + + result.should.be.ok; + result.length.should.equal(1); + + var stat = result.shift(); + stat.should.be.ok; + stat.pid.should.equal(process.pid); + stat.name.should.equal(name); + stat.status.should.equal(view); + + + status.setStatus(name, updated).then(function(){ //as we didn't register update, setStatus should have no effect at all. + + status.getStatus(name).then(function(result){ + + result.should.be.ok; + result.length.should.equal(1); + + var stat = result.shift(); + stat.should.be.ok; + stat.pid.should.equal(process.pid); + stat.name.should.equal(name); + stat.status.should.equal(updated); + + done(); + + }); + + }, done); + + }, done); + }); + }); + + }); + + //need to add test for cluster mode. + //we'll need to verify that statuses, register, setStatus, getStatus all work from workers view, masters view + //worker should see components registered by itself + //master should see components registered by itself and all of its workers + + describe('cluster-status', function(){ + + it('should work in cluster mode', function(done){ + + this.timeout(5000); + + var token = 't-' + Date.now(), + clusterRuntime = fork(require.resolve('./lib/cluster-status-runtime'), ['--token=' + token]); + + clusterRuntime.once('message', function(msg){ + + console.log('[test] message:%j, exit:%j', msg, msg.exit); + + done(msg.exit); + + }); + + }); + }); + +}); \ No newline at end of file diff --git a/test/cluster-test.js b/test/cluster-test.js deleted file mode 100644 index c195a36..0000000 --- a/test/cluster-test.js +++ /dev/null @@ -1,635 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -var spawn = require('child_process').spawn, - request = require('request'), - fs = require('fs'), - os = require('os'), - EventEmitter = require('events').EventEmitter, - when = require('when'), - util = require('util'), - _ = require('underscore'), - harbor = require('harbor')(3000, 5000); - -var debug = false; -function log() { - if(debug) { - console.log.apply(null, (arguments || []).join('')); - } -} -var test = 't', - ith = 0, - port = 3000, - monPort = 10000 - port; - -module.exports = { - - setUp: function (callback) { - - harbor.claim(test + (ith += 1), function(err, p){ - - port = p; - - harbor.claim(test + (ith += 1), function(err, p){ - - monPort = p; - - callback(); - }); - }); - }, - - 'start and then stop': function(test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function() { - waitForStart(child, emitter, test, 0, 100); - }); - - emitter.on('started', function () { - stop(emitter); - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function() { - waitForStop.apply(null, [emitter, test, 0, 100]) - }); - - emitter.on('stopped', function() { - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function(err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }); - - emitter.emit('starting'); - }, - - 'start, check heartbeat and stop': function(test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function() { - waitForStart(child, emitter, test, 0, 100); - }); - - emitter.on('started', function () { - - var timeOut = setTimeout(function(){ - test.ok(false, "timeout and no heartbeat found"); - stop(emitter); - }, 5000); - - emitter.once('heartbeat', function(heartbeat){ - - console.log('got heartbeat!!!\j%j', heartbeat); - test.ok(heartbeat.pid); - //test.ok(heartbeat.uptime); - //test.ok(heartbeat.totalmem); - //test.ok(heartbeat.freemem); - - clearTimeout(timeOut); - stop(emitter); - }); - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function() { - waitForStop.apply(null, [emitter, test, 0, 100]) - }); - - emitter.on('stopped', function() { - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function(err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }); - - emitter.emit('starting'); - }, - - 'start, check ecv and stop': function(test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function() { - waitForStart(child, emitter, test, 0, 100); - }); - - emitter.on('started', function () { - setTimeout(function(){ - request(util.format('http://localhost:%d/ecv', port), function (error, response, body) { - // Regex to match the expected response. Tricky part is the IPv4 match. - // Very naive exp to check numbers 0 - 255. - // (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]? ) -> ( numbers 250 to 255 | numbers 200 to 249 | numbers 0 to 199) - // Same expression for each of the 4 IPs - var hostname = require('os').hostname(); - var re = new RegExp(util.format( - 'status=AVAILABLE&ServeTraffic=true&ip=(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)&hostname=%s&port=%d&time=.*', - hostname, - port)); - - test.ok(re.exec(body) !== null, - util.format('expected:status=AVAILABLE&ServeTraffic=true&ip=&hostname=%s&port=%d&time=.*&error=%s&body=%s', hostname, port, error, body)); - - stop(emitter); - }) - }, 1000); - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function() { - waitForStop.apply(null, [emitter, test, 0, 100]); - }); - - emitter.on('stopped', function() { - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function(err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }) - - emitter.emit('starting'); - }, - - 'start and then shutdown': function (test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function () { - waitForStart(child, emitter, test, 0, 100); - }); - - emitter.on('started', function () { - shutdown(emitter); - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function () { - waitForStop.apply(null, [emitter, test, 0, 100]) - }); - - emitter.on('stopped', function () { - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function (err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }); - - emitter.emit('starting'); - }, - - - 'start, send traffic, get responses, and then shutdown': function (test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function () { - waitForStart(child, emitter, test, 0, 100); - }); - - emitter.on('started', function () { - var respCount = 0; - for(var i = 0; i < 10; i++) { - request(util.format('http://localhost:%d', port), function (error, response, body) { - if(error) { - test.ok(false, 'got error from server') - } - else { - respCount++; - if(respCount === 10) { - return shutdown(emitter); - } - } - }); - } - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function () { - waitForStop.apply(null, [emitter, test, 0, 100]) - }); - - emitter.on('stopped', function () { - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function (err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }); - - emitter.emit('starting'); - }, - - 'start, graceful shutdown': function (test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function () { - waitForStart(child, emitter, test, 0, 100); - }); - - var respCount = 0; - - emitter.on('started', function () { - for(var i = 0; i < 2000; i++) { - request(util.format('http://localhost:%d', port), function (error, response, body) { - if(error) { - test.ok(false, 'got error from server') - } - else { - respCount++; - } - }); - } - // Send shutdown while requests are in-flight - setTimeout(function() { - shutdown(emitter); - }, 10); - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function () { - waitForStop.apply(null, [emitter, test, 0, 100]) - }); - - emitter.on('stopped', function () { - // Ensure that all in-flight requests are handled - test.equals(respCount, 2000); - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function (err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }); - - emitter.emit('starting'); - }, - - 'start, check recycle on threshold, shutdown': function (test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function () { - waitForStart(child, emitter, test, 0, 100); - }); - - var respCount = 0, errCount = 0; - emitter.on('started', function () { - var paths = fs.readdirSync('./pids'); - - // connThreshold is 10. So, sending 20+ requests without pooling would cause - // recycle - for(var i = 0; i < 100; i++) { - request({ - uri: util.format('http://localhost:%d', port), - headers: { - 'connection': 'close' - } - }, function (error, response, body) { - if(error) { - test.ok(false, error.message || 'got error from server') - } - else { - respCount++; - } - if(respCount === 100) { - // Wait for process recycling to complete - setTimeout(function() { - // Before shutting down, check the pids again - var pathsEnd = fs.readdirSync('./pids'); - test.ok(pathsEnd.length > paths.length, 'Expected more processes'); - shutdown(emitter); - }, 5000); - } - }); - } - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function () { - waitForStop.apply(null, [emitter, test, 0, 100]) - }); - - emitter.on('stopped', function () { - fs.readdir('./pids', function (err, paths) { - test.equal(paths.length, 0); - }); - test.ok(respCount + errCount, 2000); - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function (err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }); - - emitter.emit('starting'); - }, - - 'start, abrupt stop': function (test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function () { - waitForStart(child, emitter, test, 0, 100); - }); - - var respCount = 0, errCount = 0; - emitter.on('started', function () { - for(var i = 0; i < 2000; i++) { - request(util.format('http://localhost:%d', port), function (error, response, body) { - if(error) { - errCount++; - } - else { - respCount++; - } - }); - } - // Send shutdown while requests are in-flight - stop(emitter); - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function () { - waitForStop.apply(null, [emitter, test, 0, 100]) - }); - - emitter.on('stopped', function () { - // Ensure that all in-flight requests are handled - test.ok(respCount + errCount, 2000); - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function (err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }); - - emitter.emit('starting'); - }, - - 'start, disable, enable and stop': function(test) { - var emitter = new EventEmitter(), child = start(emitter); - - emitter.on('starting', function () { - waitForStart(child, emitter, test, 0, 100); - }); - - emitter.on('started', function () { - request(util.format('http://localhost:%d/ecv', port), function (error, response, body) { - // Regex to match the expected response. Tricky part is the IPv4 match. - // Very naive exp to check numbers 0 - 255. - // (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]? ) -> ( numbers 250 to 255 | numbers 200 to 249 | numbers 0 to 199) - // Same expression for each of the 4 IPs - var hostname = require('os').hostname(); - var re = new RegExp(util.format( - 'status=AVAILABLE&ServeTraffic=true&ip=(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)&hostname=%s&port=%d&time=.*', - hostname, - port)); - var result = re.exec(body); - test.ok(result !== null, - util.format('expected:status=AVAILABLE&ServeTraffic=true&ip=&hostname=%s&port=%d&time=.*', hostname, port)); - - request({uri: util.format('http://localhost:%d/ecv/disable', port), method: 'POST'}, function (error, response, body) { - // Wait for signal to propagate to workers - setTimeout(function() { - request(util.format('http://localhost:%d/ecv', port), function (error, response, body) { - re = new RegExp(util.format( - 'status=DISABLED&ServeTraffic=false&ip=(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)&hostname=%s&port=%d&time=.*', - hostname, - port)); - var result = re.exec(body); - test.ok(result !== null, - util.format('expected:status=AVAILABLE&ServeTraffic=false&ip=&hostname=%s&port=%d&time=.*', hostname, port)); - request({uri: util.format('http://localhost:%d/ecv/enable', port), method: 'POST'}, function (error, response, body) { - if(error) { - test.ok(false, 'could not enable again'); - } - setTimeout(function() { - request(util.format('http://localhost:%d/ecv', port), function (error, response, body) { - if(error) { - test.ok(false, 'ecv did not succeed'); - } - stop(emitter); - }); - }, 200); - }); - }); - - }, 200); - }); - }); - }); - - emitter.on('start failure', function (error) { - log('Failed to start ', error.stack || error); - test.ok(false, 'failed to start') - }); - - emitter.on('stopping', function () { - waitForStop.apply(null, [emitter, test, 0, 100]) - }); - - emitter.on('stopped', function () { - log('Stopped'); - // Assert that there are 0 pids. - fs.readdir('./pids', function (err, paths) { - test.equal(paths.length, 0); - }); - test.done(); - }) - - emitter.emit('starting'); - } -} - -// Start the cluster -function start(emitter) { - log('Starting'); - var env = {}; - _.extend(env, process.env); - _.extend(env, { - port:port, - monPort:monPort, - heartbeatInterval:100 - }); - var start = spawn('node', ['test/lib/server.js'], { - env: env, - stdio: ['pipe', 1, 2, 'ipc']//enable piped stdout, and ipc for messaging - }); - - start.on('message', function(message){ - - if(message.type === 'heartbeat'){ - emitter.emit(message.type, message); - } - }); - - start.on('exit', function (code, signal) { - log('Process exited with signal ', signal, ' and code ', code); - }); - - return start; -} - -function stop(emitter) { - log('Stopping'); - var stop = spawn('node', ['test/lib/stop.js']); - - stop.on('exit', function (code, signal) { - log('Process exited with signal ', signal, ' and code ', code); - }); - - stop.stdout.setEncoding('utf8'); - stop.stdout.on('data', function (data) { - log(data); - }); - emitter.emit('stopping'); -} - -function shutdown(emitter) { - log('Shutting down'); - var shutdown = spawn('node', ['test/lib/shutdown.js']); - shutdown.on('exit', function (code, signal) { - log('Process exited with signal ', signal, ' and code ', code); - }); - - shutdown.stdout.setEncoding('utf8'); - shutdown.stdout.on('data', function (data) { - log(data); - }); - emitter.emit('stopping'); -} - -/* -function waitForStart(child, emitter, test, current, max) { - current++; - if(current < max) { - request(util.format('http://localhost:%d', port), function (error, response, body) { - log('Waiting for server to start'); - if(error) { - log('Error: ', error.stack || error); - if(error.code === 'ECONNREFUSED') { - setTimeout(function () { - waitForStart.apply(null, [child, emitter, test, current, max]) - }, 100); - } - } - else { - emitter.emit('started'); - } - }); - } - else { - test.ok(false, 'Server did not start. Giving up'); - test.done(); - } -} -*/ - -function waitForStart(child, emitter, test) { - - var deferred = when.defer(); - var timeOut = setTimeout(function(){ - deferred.reject(new Error("timeout")); - }, 3000); - - var handler = function(message){ - if(message.ready){ - clearTimeout(timeOut); - child.removeListener("message", handler); - - setTimeout(function(){ - deferred.resolve(message); - }, 2000); - } - if(message.type === 'heartbeat'){ - emitter.emit('heartbeat', message); - } - }; - child.on("message", handler); - - deferred.promise - .then(function(){ - emitter.emit("started"); - }) - .otherwise(function(error){ - test.ok(false, error); - test.done(); - }); -} - -function waitForStop(emitter, test, current, max) { - current++; - if(current < max) { - request(util.format('http://localhost:%d', port), function (error, response, body) { - log('Waiting for server to stop'); - if(error) { - emitter.emit('stopped'); - } - else { - setTimeout(function () { - waitForStop.apply(null, [emitter, test, current, max]) - }, 100); - } - }); - } - else { - test.ok(false, 'Server did not start. Giving up'); - test.done(); - } -} - - diff --git a/test/component-status-test.js b/test/component-status-test.js deleted file mode 100644 index 05ed56e..0000000 --- a/test/component-status-test.js +++ /dev/null @@ -1,102 +0,0 @@ -// 'use strict'; - -// var componentStatus = require('../lib/component-status.js').componentStatus, -// should = require('should'); - -// describe('component-status', function(){ - -// describe('#register', function(){ - -// it('should allow registration', function(done){ - -// componentStatus.register('health', function(params){ -// return 'i am healthy\n'; -// }); - -// componentStatus.reducer('health', function(memoize, health){ -// return { -// value : memoize && health -// }; -// }); - -// componentStatus.getComponents().should.include('health'); - -// componentStatus.getStatus('health', { -// done : function(status){ -// status.should.be.ok; -// done(); -// } -// }); -// }); - -// it('should allow update of status of the updater is given', function(done){ -// var trafficEnabled = true; -// componentStatus.register('traffic-enabled', -// function(params){ -// return trafficEnabled; -// }, -// 'array', -// function(params, value){ -// trafficEnabled = value; -// }); - -// componentStatus.getStatus('traffic-enabled', { -// 'done': function(status){ -// status.should.be.ok; -// status[0].should.be.ok; - -// componentStatus.setStatus('traffic-enabled', { -// 'done': function(){ - -// componentStatus.getStatus('traffic-enabled', { -// 'done': function(status){ - -// status.should.be.ok; -// status[0].should.not.be.ok; - -// done(); -// } -// }); -// } -// }, false); -// } -// }); -// }); - -// it('should allow update of status of the updater is given from worker', function(done){ -// var trafficEnabled = true; -// componentStatus.register('traffic-enabled', -// function(params){ -// return trafficEnabled; -// }, -// 'array', -// function(params, value){ -// trafficEnabled = value; -// }); - -// componentStatus.getStatus('traffic-enabled', { -// 'worker': process.pid, -// 'done': function(status){ -// status.should.be.ok; -// status[0].should.be.ok; - -// componentStatus.setStatus('traffic-enabled', { -// 'worker': process.pid, -// 'done': function(){ - -// componentStatus.getStatus('traffic-enabled', { -// 'worker': process.pid, -// 'done': function(status){ - -// status.should.be.ok; -// status[0].should.not.be.ok; - -// done(); -// } -// }); -// } -// }, false); -// } -// }); -// }); -// }); diff --git a/test/ecv-test.js b/test/ecv-test.js new file mode 100644 index 0000000..e248603 --- /dev/null +++ b/test/ecv-test.js @@ -0,0 +1,192 @@ +'use strict'; + +var should = require('should'), + express = require('express'), + request = require('request'), + http = require('http'), + when = require('when'), + timeout = require('when/timeout'), + _ = require('underscore'), + EventEmitter = require('events').EventEmitter, + getLogger = require('../lib/utils').getLogger, + pickAvailablePort = require('../lib/utils').pickAvailablePort; + +function knock(port, path, assertions){ + + var deferred = when.defer(); + + request.get('http://127.0.0.1:' + port + path, function(error, response, body){ + + assertions = assertions || function(){}; + try{ + assertions(error, response, body); + deferred.resolve(null); + } + catch(e){ + console.trace(e); + deferred.reject(e); + } + }); + + return deferred.promise; +} + +describe('ecv', function(){ + + before(function(done){ + + process.getLogger = getLogger; + done(); + }); + + describe('#enable', function(){ + + it('should support control mode via urls', function(done){ + + this.timeout(5000); + + var ecv = require('../lib/ecv'), + app = express(), + server = http.createServer(app), + emitter = new EventEmitter(); + + emitter.to = function(targets){ + + return { + 'emit': function(){ + emitter.emit.apply(emitter, arguments); + } + }; + }; + + ecv.enable(app, { + 'root': '/ecv', + 'markUp': '/ecv/markUp', + 'markDown': '/ecv/markDown', + 'mode': 'control', + 'disabled': true, + 'emitter': emitter + }); + + pickAvailablePort(8000, 8099).then(function(port){ + + server.listen(port, function(){ + //server started; + + knock(port, '/ecv', function(error, response, body){ + + should.not.exist(error); + response.should.be.ok; + response.statusCode.should.equal(500);//yet markUp + }) + .then(function(){ + + var expectMarkUpAlert = when.defer(); + + emitter.once('markUp', function(target){ + console.log('[markUp] %j', target); + expectMarkUpAlert.resolve(target); + }); + + when.join(knock(port, '/ecv/markUp'), timeout(2000, expectMarkUpAlert.promise)).then(function(){ + console.log('[marked up]'); + knock(port, '/ecv', function(error, response, body){ + + should.not.exist(error); + response.should.be.ok; + response.statusCode.should.equal(200);//should have been marked up + }) + .then(function(){ + + var expectMarkDownAlert = when.defer(); + + emitter.once('markDown', function(target){ + expectMarkDownAlert.resolve(target); + }); + + when.join(knock(port, '/ecv/markDown'), timeout(2000, expectMarkDownAlert.promise)).then(function(){ + + knock(port, '/ecv', function(error, response, body){ + + should.not.exist(error); + response.should.be.ok; + response.statusCode.should.equal(500);//marked down again + }) + .then(done, done); //fail due to incorrect ecv after mark down + + }, done); //fail due to either mark down rejected or mark down status change event not received + + }, done); //fail due to incorrect ecv after mark up + + }, done); //fail due to either mark up rejected or mark up status change event not received + + }, done); //fail due to initial ecv check failed + + }); + + }, done); //fail due to all ports rejected + }); + + it('should support monitor with any validator', function(done){ + + this.timeout(3000); + + var ecv = require('../lib/ecv.js'), + app = express(), + server = http.createServer(app), + emitter = new EventEmitter(), + disabled = false; + + emitter.to = function(targets){ + + return { + 'emit': function(){ + emitter.emit.apply(emitter, arguments); + } + }; + }; + + pickAvailablePort(8000, 8099).then(function(port){ + + ecv.enable(app, { + 'root': '/ecv', + 'mode': 'monitor', + 'monitor': 'http://localhost:' + port + '/', + 'validator': function(error, response, body){ + + return !error && response.statusCode === 200; + }, + 'emitter': emitter + }); + + app.get('/', function(req, res){ + + res.send(disabled ? 500 : 200); + }); + + server.listen(port, function(){ + + knock(port, '/ecv', function(error, response, body){ + + should.not.exist(error); + response.should.be.ok; + response.statusCode.should.equal(200); + }) + .then(function(){ + + disabled = true; + + knock(port, '/ecv', function(error, response, body){ + + should.not.exist(error); + response.should.be.ok; + response.statusCode.should.equal(500); + }) + .then(done, done); + + }, done); + }); + }); + }); + }); +}); diff --git a/test/lib/cluster-cache-runtime.js b/test/lib/cluster-cache-runtime.js new file mode 100644 index 0000000..67ba6a0 --- /dev/null +++ b/test/lib/cluster-cache-runtime.js @@ -0,0 +1,71 @@ +'use strict'; + +var listen = require('../../index').listen; +var express = require('express'); +var app = express(); + +function configureApp() { + app.get('/set', function (req, res) { + var key = req.query.key; + var value = req.query.value; + if (!key || !value) { + res.send('hello', 200); + }else { + var cache = require('../../lib/cache.js').use('cache-test'); + cache.set(key, value).then(function (happens) { + if (happens) { + res.send(value, 200); + }else { + res.send('fail', 200); + } + }).otherwise(function (err) { + res.send(err, 404); + }); + } + }); + + app.get('/get', function (req, res) { + var key = req.query.key; + if (!key) { + res.send('hello', 200); + }else { + var cache = require('../../lib/cache.js').use('cache-test'); + cache.get(key, function () { + return 'cache-test'; + }).then(function (value) { + res.send(value, 200); + }).otherwise(function (err) { + res.send(err, 404); + }); + } + }); + + return app; +} + +//console.log('aaa: ' + process.env.port); +listen({ + 'noWorkers': 2, + 'createServer': require('http').createServer, + 'app': app, + 'port': parseInt(process.env.port) || 9090, + 'configureApp': configureApp, + 'cache': { + 'enable': true, + 'mode': 'standalone', + //'domainPath': './tmp/cluster-cache-domain-' + process.pid, + //'persistPath': 'tmp/cluster-cache-persist-' + process.pid + }, + 'ecv': { + 'mode': 'control', + 'root': '/ecv' + }, + 'monCreateServer': require('http').createServer, + 'monPort': parseInt(process.env.monPort) || 9091 +}).then(function (resolved) { + process.send({ + ready: true, + }); +}).otherwise(function (err) { + process.send({err: err}); +}); diff --git a/test/lib/cluster-emitter-runtime.js b/test/lib/cluster-emitter-runtime.js new file mode 100644 index 0000000..9429ebb --- /dev/null +++ b/test/lib/cluster-emitter-runtime.js @@ -0,0 +1,119 @@ +'use strict'; + +process.getLogger = require('../../lib/utils.js').getLogger; + +var should = require('should'), + cluster = require('cluster'), + optimist = require('optimist'), + when = require('when'), + timeout = require('when/timeout'), + _ = require('underscore'), + emitter = require('../../lib/emitter'), + logger = process.getLogger(__filename); + +if(cluster.isMaster){ + + var argv = optimist.argv, + token = argv.token, + noWorkers = argv.noWorkers || 2, + event = 'event-' + token, + echo = 'echo-' + event; + + cluster.setupMaster({ + 'args': ['--event=' + event] + }); + + logger.info('[master] exec with token:%s and will use event:%s and echo:%s to verify emitter with %d workers', token, event, echo, noWorkers); + + var workers = _.map(_.range(0, noWorkers), function(){ + return cluster.fork(); + }), + waitForWorkers = when.map(workers, function(w){ + var waitForOnline = when.defer(); + + w.once('online', function(){ + waitForOnline.resolve(w); + }); + + return timeout(2000, waitForOnline.promise); + }), + exit = function exit(error){ + + logger.info('[master] exiting with error:%j', error); + + process.send({ + 'exit': error + }); //tell the test it has exited with failure + + process.nextTick(function(){ //exit now. + _.invoke(workers, 'kill', 'SIGTERM'); //force all workers to exit before master itself + process.nextTick(function(){ + process.exit(error ? -1 : 0); + }); + }); + }; + + waitForWorkers + .then(function(){ + + logger.info('[master] got all workers online notifications'); + + var expects = _.map(workers, function(w){ + return w.process.pid;//all workers' pids + }) + .concat([process.pid]);//master included + + emitter.once(event, function(){ + + logger.info('[master] received event:%s, and will echo:%s with pid:%d', event, echo, process.pid); + emitter.to(['self']).emit(echo, process.pid); + }); + + emitter.on(echo, function(pid){ + + logger.info('[master] received echo:%s from process:%d', echo, pid); + + expects = _.without(expects, pid); + if(_.isEmpty(expects)){ + exit(); + } + }); + + logger.info('[master] emitting event:%s to all', event); + emitter.emit(event, echo); + + setTimeout(function(){ + + exit(new Error('timeout after 5s, remaining expects:' + expects)); + }, 5000); + }) + .otherwise(function(error){ + + logger.info('[master] did not receive all workers online notification in time'); + exit(error); + }); +} +else{ + + var argv = optimist.argv, + event = argv.event; + + emitter.on(event, function(echo){ + + logger.info('[worker:%d] received:%s and will echo:%s', process.pid, event, echo); + + var eventToMe = event + '/' + process.pid; + + emitter.once(eventToMe, function(){ + + logger.info('[worker:%d] received:%s and will emit:%s', process.pid, eventToMe, echo); + emitter.emit(echo, process.pid); + logger.info('[worker:%d] emitted:%s with payload:%s', process.pid, echo, process.pid); + }); + + emitter.to(['self']).emit(eventToMe); + logger.info('[worker:%d] emitted:%s', process.pid, eventToMe); + }); + + logger.info('[worker:%d] prepared to echo to event:%s', process.pid, event); +} diff --git a/test/lib/cluster-pause-resume-runtime.js b/test/lib/cluster-pause-resume-runtime.js new file mode 100644 index 0000000..651f692 --- /dev/null +++ b/test/lib/cluster-pause-resume-runtime.js @@ -0,0 +1,53 @@ +'use strict'; + +var listen = require('../../index').listen; +var express = require('express'); +var app = express(); + +function configureApp() { + app.get('/sayHello', function (req, res) { + res.send('hello', 200); + }); + + return app; +} + +listen({ + 'noWorkers': 1, + 'createServer': require('http').createServer, + 'app': app, + 'port': parseInt(process.env.port) || 9090, + 'configureApp': configureApp, + 'cache': { + 'enable': false + }, + 'ecv': { + 'mode': 'control', + 'root': '/ecv' + }, + 'monCreateServer': require('http').createServer, + 'monPort': parseInt(process.env.monPort) || 9091 +}).then(function (resolved) { + //console.log(resolved); + if (resolved.worker) { + return; + } + var master = resolved.master; + var workerPid = Object.keys(master.puppets)[0]; + process.on('message', function (msg) { + if (msg.operation === 'pause') { + master.pause(workerPid).then(function (resolved) { + process.send({paused: true}); + }); + } + if (msg.operation === 'resume') { + master.resume(workerPid).then(function (resolved) { + process.send({resumed: true}); + }); + } + }); + process.send({ready: true}); +}).otherwise(function (err) { + console.log(err); + process.send({err: err}); +}); diff --git a/test/lib/cluster-status-runtime.js b/test/lib/cluster-status-runtime.js new file mode 100644 index 0000000..ff40c1d --- /dev/null +++ b/test/lib/cluster-status-runtime.js @@ -0,0 +1,125 @@ +'use strict'; + +process.getLogger = require('../../lib/utils').getLogger; + +var should = require('should'), + cluster = require('cluster'), + optimist = require('optimist'), + when = require('when'), + timeout = require('when/timeout'), + _ = require('underscore'), + status = require('../../lib/status'), + emitter = require('../../lib/emitter'), + logger = process.getLogger(__filename); + +if(cluster.isMaster){ + + var argv = optimist.argv, + token = argv.token, + noWorkers = argv.noWorkers || 2, + statusName = 'status-' + token; + + cluster.setupMaster({ + 'args': ['--status=' + statusName] + }); + + logger.info('[master] exec with token:%s and will register status:%s and create %d workers', token, statusName, noWorkers); + + var onlineWorkers = 0, + waitForWorkers = when.defer(); + + emitter.on('worker-online', function(pid){ + + onlineWorkers += 1; + + if(onlineWorkers === noWorkers){ + + waitForWorkers.resolve(onlineWorkers); + } + }); + + var workers = _.map(_.range(0, noWorkers), function(){ + + return cluster.fork(); + }), + exit = function exit(error){ + + logger.info('[master] exiting with error:%j', error); + + process.send({ + 'exit': error ? new Error(error) : null + }); + + process.nextTick(function(){ //exit now. + + _.invoke(workers, 'kill', 'SIGTERM'); //force all workers to exit before master itself + + process.nextTick(function(){ + + process.exit(error ? -1 : 0); + + }); + }); + }; + + timeout(4000, waitForWorkers.promise) + .then(function(){ + + logger.info('[master] got all workers online notifications'); + + var expects = _.map(workers, function(w){ + return w.process.pid;//all workers' pids + }) + .concat([process.pid]);//master included + + var pid = process.pid; + + status.register(statusName, + function(){ + return pid; + }, + function(value){ + pid = value; + }); + + status.getStatus(statusName) + .then(function(result){ + + console.log('[cluster-status] got status result:%j', result); + + result.should.be.ok; + result.length.should.equal(expects.length); + + exit(); + + }) + .otherwise(exit); + }) + .otherwise(function(error){ + + logger.info('[master] did not receive all workers online notification in time'); + + exit(error); + }); +} +else{ + + var argv = optimist.argv, + statusName = argv.status, + pid = process.pid; + + logger.info('[worker:%d] forked and register status:%s', pid, statusName); + + status.register(statusName, + function(){ + return pid; + }, + function(value){ + pid = value; + }); + + logger.info('[worker:%d] registered status', process.pid, statusName); + + emitter.to(['master']).emit('worker-online', pid);//truely ready + +} diff --git a/test/lib/server.js b/test/lib/server.js deleted file mode 100644 index ac8ee7f..0000000 --- a/test/lib/server.js +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var Cluster = require('../../lib/index.js'), - express = require('express'); - -var server = express.createServer(); -var serving = true; -server.get('/', function(req, res) { - res.send('hello'); - if(!serving) { - req.connection.end(); - } -}); - -// test nanny feature -server.get('/nanny-feature-test', function (req, res) { - // the first worker that handles the request will run away - res.send(process.pid + ''); - console.log('worker ' + process.pid + ' will run away after 2s'); - setTimeout(function () { - console.log('worker ' + process.pid + ' runs away'); - clearInterval(process.heartbeat); - }, 2000); -}); - -server.on('close', function() { - serving = false; -}) - -var c = new Cluster({ - timeout: 300 * 1000, - port: process.env["port"] || 3000, - monPort: process.env["monPort"] || 10000 - process.env["port"] || 3001, - cluster: true, - noWorkers: process.env["noWorkers"] || 2, - connThreshold: 10, - ecv: { - control: true - }, - heartbeatInterval: process.env["heartbeatInterval"] || 1000, - maxHeartbeatDelay: process.env["maxHeartbeatDelay"] || 3000 -}); - -c.on('died', function(pid) { - //console.log('Worker ' + pid + ' died'); - process.send({ - pid: pid, - dead: true - }) -}); - -c.on('forked', function(pid) { - //console.log('Worker ' + pid + ' forked'); -}); - -c.on('listening', function(pid){ - //console.log('Worker ' + pid + ' listening'); - process.send({ - ready: true - }); -}); - -c.on('SIGKILL', function() { - //console.log('Got SIGKILL'); - process.send({ - 'signal':'SIGKILL' - }); -}); - -c.on('SIGTERM', function(event) { - //console.log('Got SIGTERM - shutting down'); - console.log(event); - process.send({ - 'signal':'SIGTERM' - }); -}); - -c.on('SIGINT', function() { - //console.log('Got SIGINT'); - process.send({ - 'signal':'SIGINT' - }); -}); - -c.on('heartbeat', function(heartbeat){ - - //console.log('Got HEARTBEAT:%j', heartbeat); - heartbeat.type = 'heartbeat'; - process.send(heartbeat); -}); - -c.listen(function(cb) { - cb(server); -}); diff --git a/test/lib/shutdown.js b/test/lib/shutdown.js deleted file mode 100644 index d122ae0..0000000 --- a/test/lib/shutdown.js +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var Cluster = require('../../lib/index.js'); -var c = new Cluster(); -c.shutdown(); \ No newline at end of file diff --git a/test/lib/stop.js b/test/lib/stop.js deleted file mode 100644 index febe6b9..0000000 --- a/test/lib/stop.js +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var Cluster = require('../../lib/index.js'); -var c = new Cluster(); -c.stop(); \ No newline at end of file diff --git a/test/lib/wsserver.js b/test/lib/wsserver.js deleted file mode 100644 index 4b36154..0000000 --- a/test/lib/wsserver.js +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2012 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var Cluster = require('../../lib/index.js'), - WebSocketServer = require('websocket').server, - http = require("http"); - -var server = http.createServer(function(request, response) { - console.log((new Date()) + ' Received request for ' + request.url); - response.writeHead(200); - response.end(); -}); - -var wsServer = new WebSocketServer({ - httpServer: server, - // You should not use autoAcceptConnections for production - // applications, as it defeats all standard cross-origin protection - // facilities built into the protocol and the browser. You should - // *always* verify the connection's origin and decide whether or not - // to accept it. - autoAcceptConnections: false -}); - -function originIsAllowed(origin) { - // put logic here to detect whether the specified origin is allowed. - return true; -} - -wsServer.on('request', function(request) { - if (!originIsAllowed(request.origin)) { - // Make sure we only accept requests from an allowed origin - request.reject(); - console.log((new Date()) + ' Connection from origin ' + request.origin + ' rejected.'); - return; - } - - var connection = request.accept('echo-protocol', request.origin); - console.log((new Date()) + ' Connection accepted.'); - connection.on('message', function(message) { - - if (message.type === 'utf8') { - console.log('Process:' + process.pid + ' Received Message: ' + message.utf8Data); - connection.sendUTF(message.utf8Data); - } - else if (message.type === 'binary') { - console.log('Process:' + process.pid + ' Received Binary Message of ' + message.binaryData.length + ' bytes'); - connection.sendBytes(message.binaryData); - } - }); - connection.on('close', function(reasonCode, description) { - console.log((new Date()) + ' Peer ' + connection.remoteAddress + ' disconnected.'); - - wsServer.close(); - }); -}); - -server.on('close', function() { - serving = false; -}) - -var c = new Cluster({ - timeout: 300 * 1000, - port: process.env["port"] || 3000, - monPort: process.env["monPort"] || 10000 - process.env["port"] || 3001, - cluster: true, - noWorkers: process.env["noWorkers"] || 2, - connThreshold: 10, - ecv: { - control: true - }, - heartbeatInterval: 1000 -}); - -c.on('died', function(pid) { - console.log('Worker ' + pid + ' died'); - process.send({ - dead: true - }) -}); - -c.on('forked', function(pid) { - console.log('Worker ' + pid + ' forked'); -}); - -c.on('listening', function(pid){ - console.log('Worker ' + pid + ' listening'); - process.send({ - ready: true - }); -}); - -c.on('SIGKILL', function() { - console.log('Got SIGKILL'); - process.send({ - 'signal':'SIGKILL' - }); -}); - -c.on('SIGTERM', function(event) { - console.log('Got SIGTERM - shutting down'); - console.log(event); - process.send({ - 'signal':'SIGTERM' - }); -}); - -c.on('SIGINT', function() { - console.log('Got SIGINT'); - process.send({ - 'signal':'SIGINT' - }); -}); - -c.on('heartbeat', function(heartbeat){ - - heartbeat.type = 'heartbeat'; - process.send(heartbeat); -}); - -c.listen(function(cb) { - cb(server); -}); diff --git a/test/master-test.js b/test/master-test.js new file mode 100644 index 0000000..b7774af --- /dev/null +++ b/test/master-test.js @@ -0,0 +1,114 @@ +'use strict'; + +var should = require('should'), + _ = require('underscore'), + util = require('util'), + express = require('express'), + request = require('request'), + EventEmitter = require('events').EventEmitter, + getLogger = require('../lib/utils').getLogger, + pickAvailablePorts = require('../lib/utils').pickAvailablePorts; + +describe('master', function(){ + + before(function(done){ + + process.getLogger = getLogger; + done(); + }); + + describe.skip('#construction', function(){ + + it('should create a master', function(done){ + + var logger = process.getLogger(), + Master = require('../lib/master').Master; + + pickAvailablePorts(7000, 7999, 5).then(function(ports){ + + logger.info('[test] ports picked:%j', ports); + + var emitter = new EventEmitter(), + app = express(), + master = new Master(process, { + 'emitter': emitter, + 'monCreateServer': require('http').createServer, + 'monConfigureApp': function(monApp){ + return monApp; + }, + 'monApp': app, + 'monPort': ports[0], + 'port': ports[1], + 'warmUpPort': ports[4], + 'noWorkers': 0, + 'debug': { + 'debugPort': ports[2], + 'webPort': ports[3] + }, + 'cache': { + + }, + 'gc': { + + }, + 'ecv': { + 'root': '/ecv' + } + }); + + logger.info('[test] master created'); + + app.get('/', function(req, res){ + + res.send(200); + }); + + logger.info('[test] app created'); + + master.should.be.ok; + master.isMaster.should.equal(true); + master.isWorker.should.equal(false); + master.pid.should.equal(process.pid); + master.status.should.be.ok; + master.gc.should.be.ok; + + _.isFunction(master.listen).should.equal(true); + _.isFunction(master.run).should.equal(true); + + master.listen().then(function(resolve){ + + resolve.should.be.ok; + resolve.server.should.be.ok; + resolve.app.should.equal(app); + resolve.port.should.equal(ports[0]); + should.not.exist(resolve.worker); + resolve.master.should.equal(master); + + var hit = util.format('http://localhost:%d/', ports[0]); + + request.get(hit, function(err, response, body){ + + should.not.exist(err); + response.should.be.ok; + response.statusCode.should.equal(200); + + done(); + }); + + }, done); + }); + }); + + }); + + describe.skip('nanny', function(){ + + it('should nanny the workers, and detect runaways', function(done){ + + done(); + + }); + + }); + +}); diff --git a/test/nanny-feature-test.js b/test/nanny-feature-test.js deleted file mode 100644 index a0d5d7b..0000000 --- a/test/nanny-feature-test.js +++ /dev/null @@ -1,72 +0,0 @@ -'use strict'; - -var should = require('should'), - request = require('request'), - _ = require('underscore'), - spawn = require('child_process').spawn, - nodeunit = require('nodeunit'), - EventEmitter = require('events').EventEmitter, - emitter = new EventEmitter(), - childProc; - -exports['Nanny Feature Test'] = { - setUp: function (callback) { - var env = {}; - _.extend(env, process.env); - _.extend(env, { - host: '127.0.0.1', - port: 3000, - monPort: 3001, - noWorkers: 2, - hearheatInterval: 500, - maxHeartbeatDelay: 1500 - }); - - childProc = spawn('node', ['test/lib/server.js'], { - env: env, - stdio: ['pipe', 1, 2, 'ipc'] - }); - - childProc.once('message', function (msg) { - if (msg.ready) { - //test.done(); - callback(); - } - }); - }, - - tearDown: function (callback) { - childProc.kill('SIGKILL'); - callback(); - }, - - 'worker runs away and gets killed & replacement has been created': function (test) { - var pid; - request.get('http://127.0.0.1:3000/nanny-feature-test', function (err, res, body) { - test.equal(err, null); - test.ok(body); - pid = parseInt(body); - console.log(pid); - }); - childProc.on('message', function (msg) { - if (msg.dead) { - console.log(msg.pid); - test.strictEqual(msg.pid, pid); - request.get('http://127.0.0.1:3001/ComponentStatus?component=worker', function (err, res, body) { - console.log(body); - var pids = body.substring(1, body.length - 1).split(','); - test.strictEqual(pids.length, 3); // master & 2 workers - test.done(); - }); - } - }); - }, - - /*'New worker created to replace the killed worker': function (test) { - request.get('http://127.0.0.1:3001/ComponentStatus?component=worker', function (err, res, body) { - var pids = body.substring(1, body.length - 1).split(','); - test.strictEqual(pids.length, 3); // master & 2 workers - test.done(); - }); - }*/ -}; diff --git a/test/performance/cluster-resume-performance-test.js b/test/performance/cluster-resume-performance-test.js new file mode 100644 index 0000000..8fc4e96 --- /dev/null +++ b/test/performance/cluster-resume-performance-test.js @@ -0,0 +1,77 @@ +'use strict'; + +var request = require('request'); +var should = require('should'); +var fork = require('child_process').fork; + +describe('Test Pause and Resume the Worker', function () { + var childProc; + before(function (done) { + var token = 't-' + Date.now(); + childProc = fork(require.resolve('../../../ebay-node-demo/server.js'), ['--token=' + token]); + childProc.once('message', function (msg) { + if (msg.ready) { + return done(); + }else if (msg.err) { + console.log(msg.err); + return done(msg.err); + } + }); + }); + + after(function (done) { + childProc.kill('SIGTERM'); + done(); + }); + + it('First request time', function (done) { + this.timeout(10000); + request.get({ + url: 'http://127.0.0.1:9090/ui-components', + timeout: 10000 + }, function (err, res, body) { + if (err) { + done(err); + } + res.statusCode.should.equal(200); + done(); + }); + }); + + /*it('Pause the woker', function (done) { + this.timeout(5000); + childProc.send({operation: 'pause'}); + + childProc.once('message', function (msg) { + if (msg.paused) { + request.get({ + url: 'http://127.0.0.1:9090', + timeout: 4000 + }, function (err, res, body) { + if (err) { + done(); + }else { + done(new Error('Should be error here')); + } + }); + } + }); + }); + + it('First request after resume', function (done) { + this.timeout(5000); + childProc.send({operation: 'resume'}); + + childProc.once('message', function (msg) { + if (msg.resumed) { + request.get('http://127.0.0.1:9090/ui-components', function (err, res, body) { + if (err) { + done(err); + } + res.statusCode.should.equal(200); + done(); + }); + } + }); + });*/ +}); diff --git a/test/puppet-test.js b/test/puppet-test.js new file mode 100644 index 0000000..3b5edc2 --- /dev/null +++ b/test/puppet-test.js @@ -0,0 +1,135 @@ +'use strict'; + +var should = require('should'), + _ = require('underscore'), + EventEmitter = require('events').EventEmitter, + Puppet = require('../lib/puppet').Puppet, + getLogger = require('../lib/utils').getLogger; + +describe('puppet', function(){ + + before(function(done){ + + process.getLogger = getLogger; + done(); + }); + + describe('#contructor', function(){ + + it('should create a puppet instance', function(done){ + + var emitter = new EventEmitter(); + emitter.to = function(targets){ + + return { + 'emit': function(){ + emitter.emit.apply(emitter, arguments); + } + }; + }; + + var pid = Math.floor(process.pid * (1 + Math.random())), + logger = getLogger(), + master = { + 'logger': logger, + 'emitter': emitter, + 'puppets': {}, + + 'fork': function(){ + + } + }, + worker = { + 'process': { + 'pid': pid + }, + + 'disconnect': function(){ + + process.nextTick(function(){ + + emitter.emit('dismiss', worker); + + process.nextTick(function(){ + + emitter.emit('exit', worker); + }); + }); + } + }, + puppet = new Puppet(master, worker, { + 'port':8080 + }, {}); + + master.puppets[pid] = puppet; + + puppet.should.be.ok; + + _.each(['dismiss', 'whenOnline', 'whenListening', 'whenExit', 'whenHeartbeat'], function(m){ + _.isFunction(puppet[m]).should.equal(true); + }); + + _.each(['forkedState', 'activeState', 'oldState', 'diedState'], function(s){ + puppet[s].should.be.ok; + }); + + emitter.once('online', function(worker){ + + puppet.whenOnline(); + }); + + emitter.once('listening', function(worker, address){ + + puppet.whenListening(address); + }); + + emitter.once('heartbeat', function(worker, heartbeat){ + + puppet.whenHeartbeat(); + }); + + emitter.once('dismiss', function(worker){ + + puppet.dismiss(); + }); + + emitter.once('exit', function(worker){ + + puppet.whenExit(); + }); + + puppet.state.should.equal(puppet.forkedState); + + emitter.once('online', function(worker){ + + puppet.worker.should.equal(worker); + puppet.state.should.equal(puppet.forkedState); + + emitter.once('listening', function(worker){ + + puppet.worker.should.equal(worker); + puppet.state.should.equal(puppet.activeState); + + emitter.once('exit', function(worker){ + + puppet.worker.should.equal(worker); + puppet.state.should.equal(puppet.diedState); + + done(); + }); + + emitter.emit('exit', worker); + }); + + emitter.emit('listening', worker, { + 'port': 8080 + }); + }); + + emitter.emit('online', worker); + + }); + + }); + +}); \ No newline at end of file diff --git a/test/utils-test.js b/test/utils-test.js new file mode 100644 index 0000000..8a7002c --- /dev/null +++ b/test/utils-test.js @@ -0,0 +1,558 @@ +'use strict'; + +var should = require('should'), + utils = require('../lib/utils'), + path = require('path'), + when = require('when'), + fs = require('graceful-fs'), + _ = require('underscore'); + +describe('utils', function(){ + + describe('#ensureDir', function(){ + + it('should mkdir if it does not exist', function(done){ + + this.timeout(1000); + + var ensureDir = utils.ensureDir, + dir = path.join(__dirname, '/ensureDir-' + Date.now()); + + fs.existsSync(dir).should.equal(false); + + ensureDir(dir); + fs.existsSync(dir).should.equal(true); + + ensureDir(dir); + fs.existsSync(dir).should.equal(true); + + var touch = path.join(dir, 'touch.txt'); + + fs.writeFileSync(touch, ''); + fs.existsSync(touch).should.equal(true); + + ensureDir(dir); + fs.existsSync(touch).should.equal(true); + + ensureDir(dir, true); + fs.existsSync(touch).should.equal(false); + + fs.unlink(dir, function(){ + done(); + }); + + }); + + }); + + describe('#writePid', function(){ + + it('should writePid to a given dir', function(done){ + + this.timeout(1000); + + var writePid = utils.writePid, + dir = path.join(__dirname, '/writePid-' + Date.now()), + pid = process.pid; + + writePid(pid, dir); + fs.existsSync(dir).should.equal(true); + + var written = fs.readdirSync(dir); + written.should.be.ok; + + _.some(_.map(written, function(filename){ + + if(/master\.([\d]+)\.pid/.test(filename)){ + //filename matched, further verifyt the pid + var verifyPid = fs.readFileSync(path.join(dir, filename), {'encoding':'utf-8'}); + return verifyPid && parseInt(verifyPid, 10) === pid; + } + else{ + //not matched + return false; + } + + })).should.equal(true); + + done(); + }); + }); + + describe('#readPids', function(){ + + it('should read all pids from a given dir', function(done){ + + this.timeout(1000); + + var writePid = utils.writePid, + readPids = utils.readPids, + dir = path.join(__dirname, '/readPids-' + Date.now()), + pid = process.pid; + + writePid(pid, dir); + fs.existsSync(dir).should.equal(true); + + var pids = readPids(dir); + pids.should.be.ok; + pids.should.include(pid); + + done(); + }); + }); + + describe('#readMasterPid', function(){ + + it('should read master pid from a given dir', function(done){ + + this.timeout(1000); + + var writePid = utils.writePid, + readMasterPid = utils.readMasterPid, + dir = path.join(__dirname, '/readMasterPid-' + Date.now()), + pid = process.pid; + + writePid(pid, dir); + fs.existsSync(dir).should.equal(true); + + var masterPid = readMasterPid(dir); + masterPid.should.be.ok; + masterPid.should.equal(pid); + + done(); + }); + }); + + describe('#getNodeInspectorPath', function(){ + + it('should get the path of node-inspector', function(done){ + + this.timeout(500); + + var nodeInspectorPath = utils.getNodeInspectorPath(); + nodeInspectorPath.should.be.ok; + + var stat = fs.statSync(nodeInspectorPath); + stat.should.be.ok; + stat.isFile().should.equal(true); + + done(); + }); + }); + + describe('#assertOld', function(){ + + it('should use the heuristic to determine when gc is hurting tps', function(done){ + + //whenever tps grow up, whether or not other metrics goes up/down assertion should be false. + var pid = Math.floor(process.pid * (1 + Math.random())), + assertOld = utils.assertOld(0);//0 seconds is old + + _.each(_.range(0, 1000), function(ith){ + + assertOld({ + 'pid': pid, + 'tps': ith, + 'cpu': ith, + 'memory': ith, + 'gc': { + 'incremental': ith, + 'full': ith + } + }).should.equal(false); + }); + + done(); + }); + }); + + describe('#assertBadGC', function(){ + + it('should use the heuristic to determine when gc is hurting tps', function(done){ + + //whenever tps grow up, whether or not other metrics goes up/down assertion should be false. + var pid = Math.floor(process.pid * (1 + Math.random())), + assertBadGC = utils.assertBadGC(); + + _.each(_.range(0, 1000), function(ith){ + + assertBadGC({ + 'pid': pid, + 'tps': ith, + 'cpu': ith, + 'memory': ith, + 'gc': { + 'pauseMS': ith + } + }).should.equal(false); + }); + + done(); + }); + + it('should assert true whenever a degradation of more than 10% happens', function(done){ + + var pid = Math.floor(process.pid * (1 + Math.random())), + assertBadGC = utils.assertBadGC(); + + assertBadGC({ + 'pid': pid, + 'tps': 50, + 'cpu': 50, + 'memory': 1000000000, + 'gc': { + 'pauseMS': 100 + } + }).should.equal(false); + + assertBadGC({ + 'pid': pid, + 'tps': 40,//over 10% + 'cpu': 55,//higher + 'memory': 1100000000, + 'gc': { + 'pauseMS': 110 + } + }).should.equal(true); + + done(); + }); + + it('should give 10% margin for tolerance', function(done){ + + var pid = Math.floor(process.pid * (1 + Math.random())), + assertBadGC = utils.assertBadGC(); + + assertBadGC({ + 'pid': pid, + 'tps': 50, + 'cpu': 50, + 'memory': 1000000000, + 'gc': { + 'pauseMS': 100 + } + }).should.equal(false); + + assertBadGC({ + 'pid': pid, + 'tps': 48,//less than 10% + 'cpu': 55,//higher + 'memory': 1100000000, + 'gc': { + 'pauseMS': 110 + } + }).should.equal(false); + + done(); + }); + + }); + + describe('#deathQueue', function(){ + + it('should let the suicide worker die if it is the 1st one in the queue', function(done){ + + var deathQueue = utils.deathQueue, + queue = [], + emitter = new (require('events').EventEmitter)(); + + emitter.to = function(targets){ + + return { + 'emit': function(){ + emitter.emit.apply(emitter, arguments); + } + }; + }; + + var pid = Math.floor(process.pid * (1 + Math.random())), + util = require('util'); + + emitter.once('dismiss', function(suicide){ + + suicide.should.equal(pid); + + process.nextTick(function(){ + emitter.emit(util.format('worker-%d-died', suicide)); + }); + + done(); + }); + + deathQueue(queue, pid, emitter, function(){ + + var successor = pid + 1; + process.nextTick(function(){ + emitter.emit(util.format('worker-%d-listening', successor)); + }); + + return { + 'process': { + 'pid': successor + } + } + }); + + }); + + it('should let us queue the suicide workers one after another', function(done){ + + var deathQueue = utils.deathQueue, + queue = [], + emitter = new (require('events').EventEmitter)(); + + emitter.to = function(targets){ + + return { + 'emit': function(){ + emitter.emit.apply(emitter, arguments); + } + }; + }; + + var pid = Math.floor(process.pid * (1 + Math.random())), + util = require('util'), + expects = _.map(_.range(0, 10), function(ith){return pid + ith * 2;}); + + emitter.on('dismiss', function(suicide){ + + suicide.should.equal(expects.shift()); + + process.nextTick(function(){ + emitter.emit(util.format('worker-%d-died', suicide)); + }); + + if(!expects.length){ + done(); + } + }); + + _.each(_.range(0, 10), function(ith){ + + var ithPid = pid + ith * 2, + prevPid = ithPid - 2, + ithSuccessor = ithPid + 1; + + deathQueue(queue, ithPid, emitter, function(){ + + process.nextTick(function(){ + + //because we queued the deaths, at the time this ith worker is to suicide, the i - 1 th worker should have been gone! + _.contains(expects, prevPid).should.equal(false); + emitter.emit(util.format('worker-%d-listening', ithSuccessor)); + }); + + return { + 'process': { + 'pid': ithSuccessor + } + } + }); + }); + }); + + it('should let us queue the suicide workers one after another even if the process exit does not work normally', function(done){ + + this.timeout(20000);//testing creates 10 processes, each needs 1s to timeout + + var deathQueue = utils.deathQueueGenerator({ + 'timeout': 1000 + }),//1sec timeout for the testing purpose. + queue = [], + emitter = new (require('events').EventEmitter)(); + + emitter.to = function(targets){ + + return { + 'emit': function(){ + emitter.emit.apply(emitter, arguments); + } + }; + }; + + var pid = Math.floor(process.pid * (1 + Math.random())), + util = require('util'), + expects = _.map(_.range(0, 10), function(ith){return pid + ith * 2;}); + + emitter.on('dismiss', function(suicide){ + + suicide.should.equal(expects.shift()); + + /* this is commented out deliberately, to test when the exit doesn't work normally + * the expectation is that the #safeKill will kickin and emit the 'worker-%d-died' event + process.nextTick(function(){ + emitter.emit(util.format('worker-%d-died', suicide)); + }); + */ + + if(!expects.length){ + done(); + } + }); + + _.each(_.range(0, 10), function(ith){ + + var ithPid = pid + ith * 2, + prevPid = ithPid - 2, + ithSuccessor = ithPid + 1; + + deathQueue(queue, ithPid, emitter, function(){ + + process.nextTick(function(){ + + //because we queued the deaths, at the time this ith worker is to suicide, the i - 1 th worker should have been gone! + _.contains(expects, prevPid).should.equal(false); + emitter.emit(util.format('worker-%d-listening', ithSuccessor)); + }); + + return { + 'process': { + 'pid': ithSuccessor + } + } + }); + }); + }); + + }); + + describe('#gcstats', function(){ + + //NOTE, we were using node-gc module, and it couldn't work together with express, socket.io, request etc. + //we received an error 'Bus error: 10' and node program exit abnormally + //we switched memwatch and nodefly-gcinfo modules but they were not emitting gc events at all + //now we're using gc-stats module, which is very rough, the index.js doesn't seem to be a correct one + //but the binary works, we wrap it in our utils and leverage only the binary part of it. + it('should collect gc stats', function(done){ + + this.timeout(30000); + + var gc = utils.gcstats, + min = 1, + bigger = [0], + nextGrowth; + + gc.on('stats', function onStats(stats){ + + console.log('[gcstats] %d %j', process.pid, stats); + + stats.should.be.ok; + /*{ + "pause":5181203, + "pauseMS":5, + "before":{ + "totalHeapSize":17603072, + "totalHeapExecutableSize":3145728, + "usedHeapSize":10838176, + "heapSizeLimit":1535115264 + }, + "after":{ + "totalHeapSize":18635008, + "totalHeapExecutableSize":3145728, + "usedHeapSize":8770888, + "heapSizeLimit":1535115264 + }, + "diff":{ + "totalHeapSize":1031936, + "totalHeapExecutableSize":0, + "usedHeapSize":-2067288, + "heapSizeLimit":0 + } + }*/ + stats.pause.should.be.ok; + + if((min -= 1) <= 0){ + + clearTimeout(nextGrowth); + + gc.removeListener('stats', onStats); + + done(); + } + }); + + (function grow(){ + + bigger.push(bigger); + + nextGrowth = setTimeout(grow, 1); + + })(); + + }); + + }); + + describe('#uvmon', function(){ + + it('should collect uv stats', function(done){ + + var uvmon = require('nodefly-uvmon'), + stats = uvmon.getData(); + //{"count":0,"sum_ms":0,"slowest_ms":0} + console.log('[uvmon] %d %j', process.pid, stats); + stats.should.be.ok; + + (function tick(countdown){ + + if(!countdown){ + return done(); + } + + stats = uvmon.getData(); + console.log('[uvmon] %d %j', process.pid, stats); + + setTimeout(function(){ + tick(countdown - 1) + }, 10); + + })(5); + }); + }); + + describe('#ls', function(){ + + it('should list all the deps', function(done){ + + utils.npmls.then(function(deps){ + + done(); + + }, done); + }); + + }); + + after(function(done){ + + var rmPatterns = [ + /ensureDir-[\d]+/, + /writePid-[\d]+/, + /readPids-[\d]+/, + /readMasterPid-[\d]+/ + ]; + + fs.readdir(__dirname, function(err, files){ + + _.each(files, function(f){ + + if(_.some(_.invoke(rmPatterns, 'test', f))){ + + var rm = path.join(__dirname, f), + touches = fs.readdirSync(rm) || []; + + _.each(touches, function(t){ + + fs.unlinkSync(path.join(rm, t)); + }); + + fs.rmdirSync(rm); + } + }); + + done(); + }); + + }); + +}); \ No newline at end of file diff --git a/test/worker-test.js b/test/worker-test.js new file mode 100644 index 0000000..612cc07 --- /dev/null +++ b/test/worker-test.js @@ -0,0 +1,153 @@ +'use strict'; + +var should = require('should'), + express = require('express'), + request = require('request'), + util = require('util'), + _ = require('underscore'), + getLogger = require('../lib/utils').getLogger, + pickAvailablePort = require('../lib/utils').pickAvailablePort, + Worker = require('../lib/worker').Worker, + EventEmitter = require('events').EventEmitter; + +describe('worker', function(){ + + before(function(done){ + + process.getLogger = getLogger; + done(); + }); + + describe('#construction', function(){ + + it('should create a worker instance', function(done){ + + this.timeout(10000); + + var logger = process.getLogger(); + + pickAvailablePort(7000, 7999).then(function(port){ + + logger.info('[test] port picked:%d', port); + + var emitter = new EventEmitter(); + emitter.to = function(targets){ + + return { + 'emit': function(){ + emitter.emit.apply(emitter, arguments); + } + }; + }; + + var app = express(), + configured = false, + warmed = false, + worker = new Worker(process, { + 'emitter': emitter, + 'createServer': require('http').createServer, + 'app': app, + 'port': port, + 'warmUpPort': port + 1, + 'configureApp': function(app){ + configured = true; + }, + 'warmUp': function(){ + warmed = true; + }, + 'gc': { + 'monitor': true + } + }); + + logger.info('[test] worker created'); + + var memory = []; + app.get('/', function(req, res){ + + memory.push(memory);//this should let GC kick in quickly + + res.send(200); + }); + + logger.info('[test] app created'); + + worker.should.be.ok; + worker.isMaster.should.equal(false); + worker.isWorker.should.equal(true); + worker.pid.should.equal(process.pid); + worker.debug.should.not.be.ok; + worker.aliveConnections.should.equal(0); + worker.totalConnections.should.equal(0); + worker.status.should.be.ok; + worker.gc.should.be.ok; + worker.gc.incremental.should.equal(0); + worker.gc.full.should.equal(0); + worker.error.should.be.ok; + worker.error.fatal.should.equal(0); + worker.error.count.should.equal(0); + + _.isFunction(worker.listen).should.equal(true); + _.isFunction(worker.run).should.equal(true); + _.isFunction(worker.whenGC).should.equal(true); + _.isFunction(worker.whenHeartbeat).should.equal(true); + _.isFunction(worker.whenStop).should.equal(true); + _.isFunction(worker.whenExit).should.equal(true); + + worker.listen().then(function(resolve){ + + resolve.should.be.ok; + resolve.server.should.be.ok; + resolve.app.should.equal(app); + resolve.port.should.equal(port); + should.not.exist(resolve.master); + resolve.worker.should.equal(worker); + configured.should.equal(true); + warmed.should.equal(true); + + var hit = util.format('http://localhost:%d/', port); + + request.get(hit, function(err, response, body){ + + should.not.exist(err); + response.should.be.ok; + response.statusCode.should.equal(200); + + //now we'll verify gc + emitter.on('gc', function(usage, type){ + + logger.info('[test] gc happended'); + + type.should.be.ok; + + numOfGCs += 1; + }); + + var numOfGCs = 0, + load = function load(){ + request.get(hit, function(err, response, body){ + + should.not.exist(err); + response.should.be.ok; + response.statusCode.should.equal(200); + + if(numOfGCs < 2){ + load(); + } + else{ + //now we'll verify heartbeat + done(); + } + }); + }; + + load(); + }); + + }, done); + }); + + }); + }); + +}); \ No newline at end of file diff --git a/test/wscluster-test.js b/test/wscluster-test.js deleted file mode 100644 index d2cb241..0000000 --- a/test/wscluster-test.js +++ /dev/null @@ -1,234 +0,0 @@ -// /* -// * Copyright 2012 eBay Software Foundation -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ - -// 'use strict'; - -// var spawn = require('child_process').spawn, -// request = require('request'), -// fs = require('fs'), -// os = require('os'), -// EventEmitter = require('events').EventEmitter, -// util = require('util'), -// _ = require('underscore'), -// when = require('when'), -// WebSocketClient = require("websocket").client; - -// var debug = false; -// function log() { -// if(debug) { -// console.log.apply(null, (arguments || []).join('')); -// } -// } -// var port = 3000, -// monPort = 10000 - port; - -// module.exports = { - -// setUp: function (callback) { -// //to ensure that occupying ports won't cause all test cases to fail -// fs.exists("./ports", function(exists){ -// if(!exists){ -// fs.writeFileSync("./ports", port); -// } -// fs.readFile("./ports", -// function(err, data){ -// port = parseInt(data, 10) + 1; -// if(port >= 5000){ -// port = 3000; -// } -// monPort = 10000 - port; -// fs.writeFile("./ports", "" + port, { -// encoding : "utf8" -// }, -// function(){ -// callback(); -// }); -// }); -// }); -// }, - -// tearDown: function(callback){ -// callback(); -// }, - -// 'start and then stop': function(test) { -// var emitter = new EventEmitter(), child = start(emitter); - -// emitter.on('starting', function() { -// waitForStart(child, emitter, test, 0, 100); -// }); - -// emitter.on('started', function () { -// //create 10 clients and verify that the websocket has been opened and served by different cluster workers. -// _.each(_.range(0, 10), function(e){ -// var client = new WebSocketClient(); -// client.on('connectFailed', function(error) { -// console.log('Connect Error: ' + error.toString()); -// }); - -// client.on('connect', function(connection) { -// console.log('WebSocket client connected'); -// connection.on('error', function(error) { -// console.log("Connection Error: " + error.toString()); -// }); -// connection.on('close', function() { -// console.log('echo-protocol Connection Closed'); -// }); -// connection.on('message', function(message) { -// if (message.type === 'utf8') { -// console.log("Received: '" + message.utf8Data + "'"); -// } -// }); - -// var now = new Date().getTime(); -// function sendNumber() { -// if (connection.connected) { -// var number = Math.round(Math.random() * 0xFFFFFF); -// connection.sendUTF(number.toString()); -// } -// if(new Date().getTime() - now < 10000){ -// setTimeout(sendNumber, 1000); -// } -// else{ -// stop(emitter); -// } -// } -// sendNumber(); -// }); - -// client.connect('ws://localhost:' + port + '/', 'echo-protocol'); -// }); -// }); - -// emitter.on('start failure', function (error) { -// log('Failed to start ', error.stack || error); -// test.ok(false, 'failed to start') -// }); - -// emitter.on('stopping', function() { -// waitForStop.apply(null, [emitter, test, 0, 100]) -// }); - -// emitter.on('stopped', function() { -// log('Stopped'); -// // Assert that there are 0 pids. -// fs.readdir('./pids', function(err, paths) { -// //test.equal(paths.length, 0); -// }); - -// test.done(); -// }); - -// emitter.emit('starting'); -// } -// } - -// // Start the cluster -// function start(emitter) { -// log('Starting'); -// var env = {}; -// _.extend(env, process.env); -// _.extend(env, { -// port:port, -// monPort:monPort -// }); -// var start = spawn('node', ['test/lib/wsserver.js'], { -// env: env, -// stdio: ['pipe', 1, 2, 'ipc']//enable piped stdout, and ipc for messaging -// }); -// start.on('exit', function (code, signal) { -// log('Process exited with signal ', signal, ' and code ', code); -// }); - -// return start; -// } - -// function stop(emitter) { -// log('Stopping'); -// var stop = spawn('node', ['test/lib/stop.js']); -// stop.on('exit', function (code, signal) { -// log('Process exited with signal ', signal, ' and code ', code); -// }); - -// stop.stdout.setEncoding('utf8'); -// stop.stdout.on('data', function (data) { -// log(data); -// }); -// emitter.emit('stopping'); -// } - -// function shutdown(emitter) { -// log('Shutting down'); -// var shutdown = spawn('node', ['test/lib/shutdown.js']); -// shutdown.on('exit', function (code, signal) { -// log('Process exited with signal ', signal, ' and code ', code); -// }); - -// shutdown.stdout.setEncoding('utf8'); -// shutdown.stdout.on('data', function (data) { -// log(data); -// }); -// emitter.emit('stopping'); -// } - -// function waitForStart(child, emitter, test) { - -// var deferred = when.defer(); -// var timeOut = setTimeout(function(){ -// deferred.reject(new Error("timeout")); -// }, 3000); - -// child.on("message", function(message){ -// if(message.ready){ -// clearTimeout(timeOut); -// deferred.resolve(); -// } -// if(message.type === 'heartbeat'){ -// emitter.emit('heartbeat', message); -// } -// }); - -// deferred.promise.then(function(){ -// emitter.emit("started"); -// }) -// .otherwise(function(error){ -// test.ok(false, error); -// test.done(); -// }); -// } - -// function waitForStop(emitter, test, current, max) { -// current++; -// if(current < max) { -// request(util.format('http://localhost:%d', port), function (error, response, body) { -// log('Waiting for server to stop'); -// if(error) { -// emitter.emit('stopped'); -// } -// else { -// setTimeout(function () { -// waitForStop.apply(null, [emitter, test, current, max]) -// }, 100); -// } -// }); -// } -// else { -// test.ok(false, 'Server did not start. Giving up'); -// test.done(); -// } -// } - -