Test case modified to wait longer after bringing primary down, now produces a different error.

mongoFailoverTest2

/* DEPS */

var MongoClient = require('mongodb').MongoClient;
var async = require('async');
var exec = require('child_process').exec;

// GLOBALS FOR TESTING
var servers = ["127.0.0.1:27017","127.0.0.1:27027","127.0.0.1:27037"];
var username = 'USERNAME';
var password = 'PASSWORD';
var replset = 'test';
var database = 'test';
var collection = 'test';

/* 
 *
 * SETUP 
 *
 */

bringUpReplMember(27017, '/data/test-0');
bringUpReplMember(27027, '/data/test-1');
bringUpReplMember(27037, '/data/test-2');

// wait 20 secs for repl set to be up and running
setTimeout( function(){ main() }, 20000 );

function main()
{
	var authString = username + ':' + password + '@';

	var rtServerString = new Array();

	for ( i = 0; i < servers.length; i++ )
	{
		rtServerString[i] = servers[i] ;
	}

	rtServerString = authString + rtServerString.join();

	var connectStringPre = "mongodb://" + rtServerString + "/";
	var connectStringEnd = "?w=0&readPreference=nearest&authSource=admin&maxPoolSize=1&replicaSet=" + replset;

	console.log( "Will connect to: " + connectStringPre + database + connectStringEnd );

	MongoClient.connect( connectStringPre + database + connectStringEnd, function(err, db) 
	{
		if ( err )
		{
			console.log( "Error connecting to db", err );
		}
		else
		{
			var testCollection = db.collection( collection );

			async.series([

				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					console.log("Bringing down primary");
					killReplMember( 27017 );
					setTimeout( function() { return callback() }, 30000 );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					setTimeout( function() { return callback() }, 3000 );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					console.log("Bringing primary back up");
					bringUpReplMember(27017, '/data/test-0');
					setTimeout( function() { return callback() }, 30000 );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					setTimeout( function() { return callback() }, 3000 );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				},
				function( callback )
				{
					doFind( testCollection, callback );
				}
			],
			function( err, results ) {});	
		}	

	} );
	
}


function bringUpReplMember( port, path )
{
	var command = 'mongod --replSet ' + replset +  ' --keyFile ~/keyFile --port ' + port + ' --dbpath ' + path;
	
	exec(command);
}

function killReplMember( port )
{
	var member_kill = "ps ax|grep mongod|grep " + port + " |awk '{print $1}'| xargs kill";
	exec(member_kill);
}

function doFind( collection, callback )
{
	collection.findOne({}, 
	function(err, item) {
		if ( err )
		{
			console.log( "Error getting data from mongodb: ", err );
		}
		else
		{
			console.log( "Successful findOne()" );
		} 

		return callback();
	});	
}