Merge branch 'feature/mongo_2.6.5_accumulators' into feature/mongo_2.6.5_accumulators_Accumulator

Conflicts:
	lib/pipeline/accumulators/Accumulator.js

lib/Errors.js

@@ -0,0 +1,65 @@
+"use strict";
+
+/*
+ * This file defines valid error codes used by mongo
+ * mongo/base/error_codes.err
+ **/
+
+// Error codes
+var ErrorCodes = {
+	OK                           : "OK"                           ,
+	INTERNAL_ERROR               : "INTERNAL_ERROR"               ,
+	BAD_VALUE                    : "BAD_VALUE"                    ,
+	DUPLICATE_KEY                : "DUPLICATE_KEY"                ,
+	NO_SUCH_KEY                  : "NO_SUCH_KEY"                  ,
+	GRAPH_CONTAINS_CYCLE         : "GRAPH_CONTAINS_CYCLE"         ,
+	HOST_UNREACHABLE             : "HOST_UNREACHABLE"             ,
+	HOST_NOT_FOUND               : "HOST_NOT_FOUND"               ,
+	UNKNOWN_ERROR                : "UNKNOWN_ERROR"                ,
+	FAILED_TO_PARSE              : "FAILED_TO_PARSE"              ,
+	CANNOT_MUTATE_OBJECT         : "CANNOT_MUTATE_OBJECT"         ,
+	USER_NOT_FOUND               : "USER_NOT_FOUND"               ,
+	UNSUPPORTED_FORMAT           : "UNSUPPORTED_FORMAT"           ,
+	UNAUTHORIZED                 : "UNAUTHORIZED"                 ,
+	TYPE_MISMATCH                : "TYPE_MISMATCH"                ,
+	OVERFLOW                     : "OVERFLOW"                     ,
+	INVALID_LENGTH               : "INVALID_LENGTH"               ,
+	PROTOCOL_ERROR               : "PROTOCOL_ERROR"               ,
+	AUTHENTICATION_FAILED        : "AUTHENTICATION_FAILED"        ,
+	CANNOT_REUSE_OBJECT          : "CANNOT_REUSE_OBJECT"          ,
+	ILLEGAL_OPERATION            : "ILLEGAL_OPERATION"            ,
+	EMPTY_ARRAY_OPERATION        : "EMPTY_ARRAY_OPERATION"        ,
+	INVALID_B_S_O_N              : "INVALID_B_S_O_N"              ,
+	ALREADY_INITIALIZED          : "ALREADY_INITIALIZED"          ,
+	LOCK_TIMEOUT                 : "LOCK_TIMEOUT"                 ,
+	REMOTE_VALIDATION_ERROR      : "REMOTE_VALIDATION_ERROR"      ,
+	NAMESPACE_NOT_FOUND          : "NAMESPACE_NOT_FOUND"          ,
+	INDEX_NOT_FOUND              : "INDEX_NOT_FOUND"              ,
+	PATH_NOT_VIABLE              : "PATH_NOT_VIABLE"              ,
+	NON_EXISTENT_PATH            : "NON_EXISTENT_PATH"            ,
+	INVALID_PATH                 : "INVALID_PATH"                 ,
+	ROLE_NOT_FOUND               : "ROLE_NOT_FOUND"               ,
+	ROLES_NOT_RELATED            : "ROLES_NOT_RELATED"            ,
+	PRIVILEGE_NOT_FOUND          : "PRIVILEGE_NOT_FOUND"          ,
+	CANNOT_BACKFILL_ARRAY        : "CANNOT_BACKFILL_ARRAY"        ,
+	USER_MODIFICATION_FAILED     : "USER_MODIFICATION_FAILED"     ,
+	REMOTE_CHANGE_DETECTED       : "REMOTE_CHANGE_DETECTED"       ,
+	FILE_RENAME_FAILED           : "FILE_RENAME_FAILED"           ,
+	FILE_NOT_OPEN                : "FILE_NOT_OPEN"                ,
+	FILE_STREAM_FAILED           : "FILE_STREAM_FAILED"           ,
+	CONFLICTING_UPDATE_OPERATORS : "CONFLICTING_UPDATE_OPERATORS" ,
+	FILE_ALREADY_OPEN            : "FILE_ALREADY_OPEN"            ,
+	LOG_WRITE_FAILED             : "LOG_WRITE_FAILED"             ,
+	CURSOR_NOT_FOUND             : "CURSOR_NOT_FOUND"             ,
+	KEY_NOT_FOUND                : "KEY_NOT_FOUND"                ,
+},
+
+// Classes of errors
+ErrorClass = {
+	NETWORK_ERROR: ["HOST_UNREACHABLE", "HOST_NOT_FOUND"],
+};
+
+module.exports = {
+	ErrorCodes: ErrorCodes,
+	ErrorClass: ErrorClass
+};

lib/index.js

@@ -42,20 +42,28 @@ exports = module.exports = function aggregate(pipeline, ctx, inputs, callback) {
 				src = inputs;
 			}else{
 				try{
-					pipelineInst.collectionName = inputs;	//NOTE: use the given `inputs` directly; not really a "name" but we don't really have collection names in mungedb-aggregate
-					src = exports.pipeline.PipelineD.prepareCursorSource(pipelineInst, pipelineInst.ctx);
+					ctx.ns = inputs;	//NOTE: use the given `inputs` directly; hacking so that the cursor source will be our inputs instead of the context namespace
+					src = exports.pipeline.PipelineD.prepareCursorSource(pipelineInst, ctx);
 				}catch(err){
 					return callback(err);
 				}
 			}
 
-			// run the pipeline against the input src
-			var results = pipelineInst.run(src, callback === exports.SYNC_CALLBACK ? undefined : function aggregated(err, results){
+			var runCallback;
+			if (!callback) {
+				runCallback = exports.SYNC_CALLBACK;
+				pipelineInst.SYNC_MODE = true;
+			} else {
+				runCallback = function aggregated(err, results){
 				if(err) return callback(err);
 				return callback(null, results.result);
-			});
-			pipelineInst = null; // unset so that subsequent calls can rebuild the pipeline
-			return results;
+		};
+			}
+
+			// run the pipeline against
+			pipelineInst.stitch();
+			var results = pipelineInst.run(runCallback);
+			return results ? results.result : undefined;
 		};
 	if(inputs) return aggregator(ctx, inputs, callback);
 	return aggregator;
@@ -75,5 +83,8 @@ exports.Cursor = require("./Cursor");
 exports.pipeline = require("./pipeline/");
 
 // version info
-exports.version = "r2.4.0-rc0";
-exports.gitVersion = "cb8efcd6a2f05d35655ed9f9b947cc4a99ade8db";
+exports.version = "r2.5.4";
+exports.gitVersion = "ffd52e5f46cf2ba74ba931c78da62d4a7f480d8e";
+
+// error code constants
+exports.ERRORS = require('./Errors.js');

lib/pipeline/Pipeline.js

@@ -1,5 +1,4 @@
 "use strict";
-var async = require("async");
 
 /**
  * mongodb "commands" (sent via db.$cmd.findOne(...)) subclass to make a command.  define a singleton object for it.
@@ -10,11 +9,11 @@ var async = require("async");
  **/
 // CONSTRUCTOR
 var Pipeline = module.exports = function Pipeline(theCtx){
-	this.collectionName = null;
-	this.sourceVector = null;
+	this.sources = null;
 	this.explain = false;
 	this.splitMongodPipeline = false;
 	this.ctx = theCtx;
+	this.SYNC_MODE = false;
 }, klass = Pipeline, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 var DocumentSource = require("./documentSources/DocumentSource"),
@@ -24,24 +23,156 @@ var DocumentSource = require("./documentSources/DocumentSource"),
 	SkipDocumentSource = require('./documentSources/SkipDocumentSource'),
 	UnwindDocumentSource = require('./documentSources/UnwindDocumentSource'),
 	GroupDocumentSource = require('./documentSources/GroupDocumentSource'),
+	OutDocumentSource = require('./documentSources/OutDocumentSource'),
+	GeoNearDocumentSource = require('./documentSources/GeoNearDocumentSource'),
+	RedactDocumentSource = require('./documentSources/RedactDocumentSource'),
 	SortDocumentSource = require('./documentSources/SortDocumentSource');
 
 klass.COMMAND_NAME = "aggregate";
 klass.PIPELINE_NAME = "pipeline";
 klass.EXPLAIN_NAME = "explain";
 klass.FROM_ROUTER_NAME = "fromRouter";
-klass.SPLIT_MONGOD_PIPELINE_NAME = "splitMongodPipeline";
 klass.SERVER_PIPELINE_NAME = "serverPipeline";
 klass.MONGOS_PIPELINE_NAME = "mongosPipeline";
 
 klass.stageDesc = {};//attaching this to the class for test cases
+klass.stageDesc[GeoNearDocumentSource.geoNearName] = GeoNearDocumentSource.createFromJson;
+klass.stageDesc[GroupDocumentSource.groupName] = GroupDocumentSource.createFromJson;
 klass.stageDesc[LimitDocumentSource.limitName] = LimitDocumentSource.createFromJson;
 klass.stageDesc[MatchDocumentSource.matchName] = MatchDocumentSource.createFromJson;
+klass.stageDesc[OutDocumentSource.outName] = OutDocumentSource.createFromJson;
 klass.stageDesc[ProjectDocumentSource.projectName] = ProjectDocumentSource.createFromJson;
+klass.stageDesc[RedactDocumentSource.redactName] = ProjectDocumentSource.createFromJson;
 klass.stageDesc[SkipDocumentSource.skipName] = SkipDocumentSource.createFromJson;
-klass.stageDesc[UnwindDocumentSource.unwindName] = UnwindDocumentSource.createFromJson;
-klass.stageDesc[GroupDocumentSource.groupName] = GroupDocumentSource.createFromJson;
 klass.stageDesc[SortDocumentSource.sortName] = SortDocumentSource.createFromJson;
+klass.stageDesc[UnwindDocumentSource.unwindName] = UnwindDocumentSource.createFromJson;
+klass.nStageDesc = Object.keys(klass.stageDesc).length;
+
+klass.optimizations = {};
+klass.optimizations.local = {};
+
+/**
+ * Moves $match before $sort when they are placed next to one another
+ * @static
+ * @method moveMatchBeforeSort
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.moveMatchBeforeSort = function moveMatchBeforeSort(pipelineInst) {
+	var sources = pipelineInst.sources;
+	for(var srcn = sources.length, srci = 1; srci < srcn; ++srci) {
+		var source = sources[srci];
+		if(source.constructor === MatchDocumentSource) {
+			var previous = sources[srci - 1];
+			if(previous && previous.constructor === SortDocumentSource) { //Added check that previous exists
+				/* swap this item with the previous */
+				sources[srci] = previous;
+				sources[srci-1] = source;
+			}
+		}
+	}
+};
+
+/**
+ * Moves $limit before $skip when they are placed next to one another
+ * @static
+ * @method moveLimitBeforeSkip
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.moveLimitBeforeSkip = function moveLimitBeforeSkip(pipelineInst) {
+	var sources = pipelineInst.sources;
+	if(sources.length === 0) return;
+	for(var i = sources.length - 1; i >= 1 /* not looking at 0 */; i--) {
+		var limit = sources[i].constructor === LimitDocumentSource ? sources[i] : undefined,
+			skip = sources[i-1].constructor === SkipDocumentSource ? sources[i-1] : undefined;
+		if(limit && skip) {
+			limit.setLimit(limit.getLimit() + skip.getSkip());
+			sources[i-1] = limit;
+			sources[i] = skip;
+
+			// Start at back again. This is needed to handle cases with more than 1 $limit
+			// (S means skip, L means limit)
+			//
+			// These two would work without second pass (assuming back to front ordering)
+			// SL   -> LS
+			// SSL  -> LSS
+			//
+			// The following cases need a second pass to handle the second limit
+			// SLL  -> LLS
+			// SSLL -> LLSS
+			// SLSL -> LLSS
+			i = sources.length; // decremented before next pass
+		}
+	}
+};
+
+/**
+ * Attempts to coalesce every pipeline stage into the previous pipeline stage, starting after the first
+ * @static
+ * @method coalesceAdjacent
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.coalesceAdjacent = function coalesceAdjacent(pipelineInst) {
+	var sources = pipelineInst.sources;
+	if(sources.length === 0) return;
+
+	// move all sources to a temporary list
+	var moveSrc = sources.pop(),
+		tempSources = [];
+	while(moveSrc) {
+		tempSources.unshift(moveSrc);
+		moveSrc = sources.pop();
+	}
+
+	// move the first one to the final list
+	sources.push(tempSources[0]);
+
+	// run through the sources, coalescing them or keeping them
+	for(var tempn = tempSources.length, tempi = 1; tempi < tempn; ++tempi) {
+		// If we can't coalesce the source with the last, then move it
+		// to the final list, and make it the new last.  (If we succeeded,
+		// then we're still on the same last, and there's no need to move
+		// or do anything with the source -- the destruction of tempSources
+		// will take care of the rest.)
+		var lastSource = sources[sources.length-1],
+			tempSrc = tempSources[tempi];
+		if(!(lastSource && tempSrc)) {
+			throw new Error('Must have a last and current source'); // verify(lastSource && tempSrc);
+		}
+		if(!lastSource.coalesce(tempSrc)) sources.push(tempSrc);
+	}
+};
+
+/**
+ * Iterates over sources in the pipelineInst, optimizing each
+ * @static
+ * @method optimizeEachDocumentSource
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.optimizeEachDocumentSource = function optimizeEachDocumentSource(pipelineInst) {
+	var sources = pipelineInst.sources;
+	for(var srci = 0, srcn = sources.length; srci < srcn; ++srci) {
+		sources[srci].optimize();
+	}
+};
+
+/**
+ * Auto-places a $match before a $redact when the $redact is the first item in a pipeline
+ * @static
+ * @method duplicateMatchBeforeInitalRedact
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.duplicateMatchBeforeInitalRedact = function duplicateMatchBeforeInitalRedact(pipelineInst) {
+	var sources = pipelineInst.sources;
+	if(sources.length >= 2 && sources[0].constructor === RedactDocumentSource) {
+		if(sources[1].constructor === MatchDocumentSource) {
+			var match = sources[1],
+				redactSafePortion = match.redactSafePortion();
+			if(Object.keys(redactSafePortion).length > 0) {
+				sources.shift(MatchDocumentSource.createFromJson(redactSafePortion, pipelineInst.ctx));
+			}
+		}
+	}
+};
 
 /**
  * Create an `Array` of `DocumentSource`s from the given JSON pipeline
@@ -52,7 +183,7 @@ klass.stageDesc[SortDocumentSource.sortName] = SortDocumentSource.createFromJson
  * @returns {Array}  The parsed `DocumentSource`s
  **/
 klass.parseDocumentSources = function parseDocumentSources(pipeline, ctx){
-	var sourceVector = [];
+	var sources = [];
 	for (var nSteps = pipeline.length, iStep = 0; iStep < nSteps; ++iStep) {
 		// pull out the pipeline element as an object
 		var pipeElement = pipeline[iStep];
@@ -70,11 +201,14 @@ klass.parseDocumentSources = function parseDocumentSources(pipeline, ctx){
 
 		// Parse the stage
 		var stage = desc(stageSpec, ctx);
-		if (!stage) throw new Error("Stage must not be undefined!");
-		stage.setPipelineStep(iStep);
-		sourceVector.push(stage);
+		if (!stage) throw new Error("Stage must not be undefined!"); // verify(stage)
+		sources.push(stage);
+
+		if(stage.constructor === OutDocumentSource && iStep !== nSteps - 1) {
+			throw new Error("$out can only be the final stage in the pipeline; code 16435");
+		}
 	}
-	return sourceVector;
+	return sources;
 };
 
 /**
@@ -99,12 +233,15 @@ klass.parseCommand = function parseCommand(cmdObj, ctx){
 	var pipeline;
 	for(var fieldName in cmdObj){
 		var cmdElement = cmdObj[fieldName];
-		if(fieldName == klass.COMMAND_NAME)						pipelineInst.collectionName = cmdElement;		//look for the aggregation command
+		if(fieldName[0] == "$")									continue;
+		else if(fieldName == "cursor")							continue;
+		else if(fieldName == klass.COMMAND_NAME)				continue;										//look for the aggregation command
 		else if(fieldName == klass.PIPELINE_NAME)				pipeline = cmdElement;							//check for the pipeline of JSON doc srcs
 		else if(fieldName == klass.EXPLAIN_NAME)				pipelineInst.explain = cmdElement;				//check for explain option
-		else if(fieldName == klass.FROM_ROUTER_NAME)			pipelineInst.fromRouter = cmdElement;			//if the request came from the router, we're in a shard
-		else if(fieldName == klass.SPLIT_MONGOD_PIPELINE_NAME)	pipelineInst.splitMongodPipeline = cmdElement;	//check for debug options
-		// NOTE: DEVIATION FROM MONGO: Not implementing: "Ignore $auth information sent along with the command. The authentication system will use it, it's not a part of the pipeline."
+		else if(fieldName == klass.FROM_ROUTER_NAME)			ctx.inShard = cmdElement;						//if the request came from the router, we're in a shard
+		else if(fieldName == "allowDiskUsage") {
+			if(typeof cmdElement !== 'boolean') throw new Error("allowDiskUsage must be a bool, not a " + typeof allowDiskUsage+ "; uassert code 16949");
+		}
 		else throw new Error("unrecognized field " + JSON.stringify(fieldName));
 	}
 
@@ -113,64 +250,13 @@ klass.parseCommand = function parseCommand(cmdObj, ctx){
 	 * Set up the specified document source pipeline.
 	 **/
 	// NOTE: DEVIATION FROM MONGO: split this into a separate function to simplify and better allow for extensions (now in parseDocumentSources)
-	var sourceVector = pipelineInst.sourceVector = Pipeline.parseDocumentSources(pipeline, ctx);
-
-	/* if there aren't any pipeline stages, there's nothing more to do */
-	if (!sourceVector.length) return pipelineInst;
-
-	/* Move filters up where possible.
-	CW TODO -- move filter past projections where possible, and noting corresponding field renaming.
-	*/
-
-	/*
-	Wherever there is a match immediately following a sort, swap them.
-	This means we sort fewer items.  Neither changes the documents in the stream, so this transformation shouldn't affect the result.
-	We do this first, because then when we coalesce operators below, any adjacent matches will be combined.
-	*/
-	for(var srcn = sourceVector.length, srci = 1; srci < srcn; ++srci) {
-		var source = sourceVector[srci];
-		if (source instanceof MatchDocumentSource) {
-			var previous = sourceVector[srci - 1];
-			if (previous instanceof SortDocumentSource) {
-				/* swap this item with the previous */
-				sourceVector[srci - 1] = source;
-				sourceVector[srci] = previous;
-			}
-		}
-	}
+	var sources = pipelineInst.sources = Pipeline.parseDocumentSources(pipeline, ctx);
 
-	/*
-	Coalesce adjacent filters where possible.  Two adjacent filters are equivalent to one filter whose predicate is the conjunction of the two original filters' predicates.
-	For now, capture this by giving any DocumentSource the option to absorb it's successor; this will also allow adjacent projections to coalesce when possible.
-	Run through the DocumentSources, and give each one the opportunity to coalesce with its successor.  If successful, remove the successor.
-	Move all document sources to a temporary list.
-	*/
-	var tempVector = sourceVector.slice(0);
-	sourceVector.length = 0;
-
-	// move the first one to the final list
-	sourceVector.push(tempVector[0]);
-
-	// run through the sources, coalescing them or keeping them
-	for(var tempn = tempVector.length, tempi = 1; tempi < tempn; ++tempi) {
-		/*
-		If we can't coalesce the source with the last, then move it to the final list, and make it the new last.
-		(If we succeeded, then we're still on the same last, and there's no need to move or do anything with the source -- the destruction of tempVector will take care of the rest.)
-		*/
-		var lastSource = sourceVector[sourceVector.length - 1],
-			temp = tempVector[tempi];
-		if (!temp || !lastSource) throw new Error("null document sources found");
-		if (!lastSource.coalesce(temp)){
-			sourceVector.push(temp);
-		}
-	}
-
-	// optimize the elements in the pipeline
-	for(var i = 0, l = sourceVector.length; i<l; i++) {
-		var iter = sourceVector[i];
-		if (!iter) throw new Error("Pipeline received empty document as argument");
-		iter.optimize();
-	}
+	klass.optimizations.local.moveMatchBeforeSort(pipelineInst);
+	klass.optimizations.local.moveLimitBeforeSkip(pipelineInst);
+	klass.optimizations.local.coalesceAdjacent(pipelineInst);
+	klass.optimizations.local.optimizeEachDocumentSource(pipelineInst);
+	klass.optimizations.local.duplicateMatchBeforeInitalRedact(pipelineInst);
 
 	return pipelineInst;
 };
@@ -186,86 +272,154 @@ function ifError(err) {
 }
 
 /**
- * Run the pipeline
+ * Gets the initial $match query when $match is the first pipeline stage
  * @method run
  * @param	inputSource		{DocumentSource}	The input document source for the pipeline
  * @param	[callback]		{Function}			Optional callback function if using async extensions
+ * @return {Object}	An empty object or the match spec
 **/
-proto.run = function run(inputSource, callback){
-	if (inputSource && !(inputSource instanceof DocumentSource)) throw new Error("arg `inputSource` must be an instance of DocumentSource");
-	if (!callback) callback = klass.SYNC_CALLBACK;
-	var self = this;
-	if (callback === klass.SYNC_CALLBACK) { // SYNCHRONOUS MODE
-		inputSource.setSource(undefined, ifError);	//TODO: HACK: temp solution to the fact that we need to initialize our source since we're using setSource as a workaround for the lack of real async cursors
-		var source = inputSource;
-		for(var i = 0, l = self.sourceVector.length; i < l; i++){
-			var temp = self.sourceVector[i];
-			temp.setSource(source, ifError);
-			source = temp;
-		}
-		/*
-		Iterate through the resulting documents, and add them to the result.
-		We do this even if we're doing an explain, in order to capture the document counts and other stats.
-		However, we don't capture the result documents for explain.
-		*/
-		var resultArray = [];
-		try{
-			for(var hasDoc = !source.eof(); hasDoc; hasDoc = source.advance()) {
-				var document = source.getCurrent();
-				resultArray.push(document);	// add the document to the result set
-				//Commenting out this assertion for munge.  MUHAHAHA!!!
-				// object will be too large, assert. the extra 1KB is for headers
-				//if(resultArray.len() < BSONObjMaxUserSize - 1024) throw new Error("aggregation result exceeds maximum document size (" + BSONObjMaxUserSize / (1024 * 1024) + "MB); code 16389");
+proto.getInitialQuery = function getInitialQuery() {
+	var sources = this.sources;
+	if(sources.length === 0) {
+		return {};
+	}
+
+	/* look for an initial $match */
+	var match = sources[0].constructor === MatchDocumentSource ? sources[0] : undefined;
+	if(!match) return {};
+
+	return match.getQuery();
+};
+
+/**
+ * Creates the JSON representation of the pipeline
+ * @method run
+ * @param	inputSource		{DocumentSource}	The input document source for the pipeline
+ * @param	[callback]		{Function}			Optional callback function if using async extensions
+ * @return {Object}	An empty object or the match spec
+**/
+proto.serialize = function serialize() {
+	var serialized = {},
+		array = [];
+
+	// create an array out of the pipeline operations
+	this.sources.forEach(function(source) {
+		source.serializeToArray(array);
+	});
+
+	serialized[klass.COMMAND_NAME] = this.ctx && this.ctx.ns && this.ctx.ns.coll ? this.ctx.ns.coll : '';
+	serialized[klass.PIPELINE_NAME] = array;
+
+	if(this.explain) serialized[klass.EXPLAIN_NAME] = this.explain;
+
+	return serialized;
+};
+
+/**
+ * Points each source at its previous source
+ * @method stitch
+**/
+proto.stitch = function stitch() {
+	if(this.sources.length <= 0) throw new Error("should not have an empty pipeline; massert code 16600");
+
+	/* chain together the sources we found */
+	var prevSource = this.sources[0];
+	for(var srci = 1, srcn = this.sources.length; srci < srcn; srci++) {
+		var tempSource = this.sources[srci];
+		tempSource.setSource(prevSource);
+		prevSource = tempSource;
+	}
+};
+
+/**
+ * Run the pipeline
+ * @method run
+ * @param callback {Function} Optional. Run the pipeline in async mode; callback(err, result)
+ * @return result {Object} The result of executing the pipeline
+**/
+proto.run = function run(callback) {
+	// should not get here in the explain case
+	if(this.explain) throw new Error("Should not be running a pipeline in explain mode!");
+
+	/* NOTE: DEVIATION FROM MONGO SOURCE. WE'RE SUPPORTING SYNC AND ASYNC */
+	if(this.SYNC_MODE) {
+		callback();
+		return this._runSync();
+	} else {
+		return this._runAsync(callback);
+	}
+};
+
+/**
+ * Get the last document source in the pipeline
+ * @method _getFinalSource
+ * @return {Object}		The DocumentSource at the end of the pipeline
+ * @private
+**/
+proto._getFinalSource = function _getFinalSource() {
+	return this.sources[this.sources.length - 1];
+};
+
+/**
+ * Run the pipeline synchronously
+ * @method _runSync
+ * @return {Object}		The results object {result:resultArray}
+ * @private
+**/
+proto._runSync = function _runSync(callback) {
+	var resultArray = [],
+		finalSource = this._getFinalSource(),
+		handleErr = function(err) {
+			if(err) throw err;
+		},
+		next;
+	while((next = finalSource.getNext(handleErr)) !== DocumentSource.EOF) {
+		resultArray.push(next);
+	}
+	return {result:resultArray};
+};
+
+/**
+ * Run the pipeline asynchronously
+ * @method _runAsync
+ * @param callback {Function} callback(err, resultObject)
+ * @private
+**/
+proto._runAsync = function _runAsync(callback) {
+	var resultArray = [],
+		finalSource = this._getFinalSource(),
+		gotNext = function(err, doc) {
+			if(err) return callback(err);
+			if(doc !== DocumentSource.EOF) {
+				resultArray.push(doc);
+				return setImmediate(function() { //setImmediate to avoid callstack size issues
+					finalSource.getNext(gotNext);
+				});
+			} else {
+				return callback(null, {result:resultArray});
 			}
-		} catch (err) {
-			return callback(err);
-		}
-		var result = {
-			result: resultArray
-//			,ok: true;	//not actually in here... where does this come from?
 		};
-		return callback(null, result);
-	} else {	// ASYNCHRONOUS MODE	//TODO: move this up to a higher level package?
-		return inputSource.setSource(undefined, function(err){	//TODO: HACK: temp solution to the fact that we need to initialize our source since we're using setSource as a workaround for the lack of real async cursors
-			if (err) return callback(err);
-			// chain together the sources we found
-			var source = inputSource;
-			async.eachSeries(
-				self.sourceVector,
-				function eachSrc(temp, next){
-					temp.setSource(source, function(err){
-						if (err) return next(err);
-						source = temp;
-						return next();
-					});
-				},
-				function doneSrcs(err){ //source is left pointing at the last source in the chain
-					if (err) return callback(err);
-					/*
-					Iterate through the resulting documents, and add them to the result.
-					We do this even if we're doing an explain, in order to capture the document counts and other stats.
-					However, we don't capture the result documents for explain.
-					*/
-					// the array in which the aggregation results reside
-					var resultArray = [];
-					try{
-						for(var hasDoc = !source.eof(); hasDoc; hasDoc = source.advance()) {
-							var document = source.getCurrent();
-							resultArray.push(document);	// add the document to the result set
-							//Commenting out this assertion for munge.  MUHAHAHA!!!
-							// object will be too large, assert. the extra 1KB is for headers
-							//if(resultArray.len() < BSONObjMaxUserSize - 1024) throw new Error("aggregation result exceeds maximum document size (" + BSONObjMaxUserSize / (1024 * 1024) + "MB); code 16389");
-						}
-					} catch (err) {
-						return callback(err);
-					}
-					var result = {
-						result: resultArray
-	//					,ok: true;	//not actually in here... where does this come from?
-					};
-					return callback(null, result);
-				}
-			);
-		});
-	}
+	finalSource.getNext(gotNext);
+};
+
+/**
+ * Get the pipeline explanation
+ * @method writeExplainOps
+ * @return {Array}	An array of source explanations
+**/
+proto.writeExplainOps = function writeExplainOps() {
+	var array = [];
+	this.sources.forEach(function(source) {
+		source.serializeToArray(array, /*explain=*/true);
+	});
+	return array;
+};
+
+/**
+ * Set the source of documents for the pipeline
+ * @method addInitialSource
+ * @param source {DocumentSource}
+**/
+proto.addInitialSource = function addInitialSource(source) {
+	this.sources.unshift(source);
 };

lib/pipeline/PipelineD.js

@@ -20,6 +20,7 @@ var DocumentSource = require('./documentSources/DocumentSource'),
  * Create a Cursor wrapped in a DocumentSourceCursor, which is suitable to be the first source for a pipeline to begin with.
  * This source will feed the execution of the pipeline.
  *
+ * //NOTE: Not doing anything here, as we don't use any of these cursor source features
  * //NOTE: DEVIATION FROM THE MONGO: We don't have special optimized cursors; You could support something similar by overriding `Pipeline#run` to call `DocumentSource#coalesce` on the `inputSource` if you really need it.
  *
  * This method looks for early pipeline stages that can be folded into
@@ -32,48 +33,62 @@ var DocumentSource = require('./documentSources/DocumentSource'),
  * @param ctx       {Object}    Context for expressions
  * @returns	{CursorDocumentSource}	the cursor that was created
 **/
-klass.prepareCursorSource = function prepareCursorSource(pipeline, /*dbName,*/ expCtx){
+klass.prepareCursorSource = function prepareCursorSource(pipeline, expCtx){
 
-	var sources = pipeline.sourceVector;
+	var sources = pipeline.sources;
 
-	//NOTE: SKIPPED: look for initial match
-	//NOTE: SKIPPED: create a query object
+	// NOTE: SKIPPED: look for initial match
+	// NOTE: SKIPPED: create a query object
 
-	//Look for an initial simple project; we'll avoid constructing Values for fields that won't make it through the projection
+	// Look for an initial simple project; we'll avoid constructing Values for fields that won't make it through the projection
 	var projection = {};
-	var deps = [];
+	var dependencies;
+	var deps = {};
 	var status = DocumentSource.GetDepsReturn.SEE_NEXT;
-	for (var i=0; i < sources.length && status != DocumentSource.GetDepsReturn.EXHAUSTIVE; i++) {
+	for (var i=0; i < sources.length && status !== DocumentSource.GetDepsReturn.EXHAUSTIVE; i++) {
 		status = sources[i].getDependencies(deps);
+		if(Object.keys(deps).length === 0) {
+			status = DocumentSource.GetDepsReturn.NOT_SUPPORTED;
+		}
 	}
-	if (status == DocumentSource.GetDepsReturn.EXHAUSTIVE) {
+	if (status === DocumentSource.GetDepsReturn.EXHAUSTIVE) {
 		projection = DocumentSource.depsToProjection(deps);
+		dependencies = DocumentSource.parseDeps(deps);
 	}
 
-	//NOTE: SKIPPED: Look for an initial sort
-	//NOTE: SKIPPED: Create the sort object
+	// NOTE: SKIPPED: Look for an initial sort
+	// NOTE: SKIPPED: Create the sort object
 
-//	//get the full "namespace" name
-//	var fullName = dbName + "." + pipeline.collectionName;
+	//get the full "namespace" name
+	// var fullName = dbName + "." + pipeline.collectionName;
 
-	//NOTE: SKIPPED: if(DEV) log messages
+	// NOTE: SKIPPED: if(DEV) log messages
 
-	//Create the necessary context to use a Cursor
-	//NOTE: SKIPPED: pSortedCursor bit
-	//NOTE: SKIPPED: pUnsortedCursor bit
-	var cursorWithContext = new CursorDocumentSource.CursorWithContext(/*fullName*/);
+	// Create the necessary context to use a Cursor
+	// NOTE: SKIPPED: pSortedCursor bit
+	// NOTE: SKIPPED: pUnsortedCursor bit
 
-	// Now add the Cursor to cursorWithContext
-	cursorWithContext._cursor = new Cursor( pipeline.collectionName );
+	// NOTE: Deviating from mongo here. We're passing in a source or set of documents instead of collection name in the ctx.ns field
+	var source;
+	if(expCtx.ns instanceof DocumentSource){
+		source = expCtx.ns;
+	} else {
+		var cursorWithContext = new CursorDocumentSource.CursorWithContext(/*fullName*/);
 
-	// wrap the cursor with a DocumentSource and return that
-	var source = new CursorDocumentSource( cursorWithContext, expCtx );
+		// Now add the Cursor to cursorWithContext
+		cursorWithContext._cursor = new Cursor( expCtx.ns );	//NOTE: collectionName will likely be an array of documents in munge
 
-//	source.namespace = fullName;
+		// wrap the cursor with a DocumentSource and return that
+		source = new CursorDocumentSource( cursorWithContext, expCtx );
 
-	//NOTE: SKIPPED: Note the query and sort
+		// NOTE: SKIPPED: Note the query and sort
 
-	if (Object.keys(projection).length) source.setProjection(projection);
+		if (Object.keys(projection).length) source.setProjection(projection, dependencies);
 
-	return source;
+		while(sources.length > 0 && source.coalesce(sources[0])) { //Note: Attempting to coalesce into the cursor source
+			sources.shift();
+		}
+	}
+
+	pipeline.addInitialSource(source);
 };

lib/pipeline/Value.js

@@ -79,67 +79,6 @@ klass.coerceToString = function coerceToString(value) {
 		throw new Error("can't convert from BSON type " + typeof(value) + " to String; uassert code 16007");
 	}
 };
-
-
-klass.canonicalize = function canonicalize(x) {
-	var xType = typeof(x);
-	if(xType == "object") xType = x === null ? "null" : x.constructor.name;
-	switch (xType) {
-		case "MinKey":
-			return -1;
-		case "MaxKey":
-			return 127;
-		case "EOO":
-		case "undefined":
-		case undefined:
-			return 0;
-		case "jstNULL":
-		case "null":
-		case "Null":
-			return 5;
-		case "NumberDouble":
-		case "NumberInt":
-		case "NumberLong":
-		case "number":
-			return 10;
-		case "Symbol":
-		case "string":
-			return 15;
-		case "Object":
-			return 20;
-		case "Array":
-			return 25;
-		case "Binary":
-			return 30;
-		case "ObjectId":
-			return 35;
-		case "ObjectID":
-			return 35;
-		case "boolean":
-		case "Boolean":
-			return 40;
-		case "Date":
-		case "Timestamp":
-			return 45;
-		case "RegEx":
-		case "RegExp":
-			return 50;
-		case "DBRef":
-			return 55;
-		case "Code":
-			return 60;
-		case "CodeWScope":
-			return 65;
-		default:
-			// Default value for Object
-			return 20;  
-	}
-};
-
-klass.cmp = function cmp(l, r){
-	return l < r ? -1 : l > r ? 1 : 0;
-};
-
 //TODO:	klass.coerceToTimestamp = ...?
 
 /**
@@ -170,7 +109,22 @@ klass.compare = function compare(l, r) {
 		if (isNaN(r)) return 1;
 		return klass.cmp(l,r);
 	}
-
+	// Compare MinKey and MaxKey cases
+	if(l.constructor && l.constructor.name in {'MinKey':1,'MaxKey':1} ){
+		if(l.constructor.name == r.constructor.name) { 
+			return 0; 
+		} else if (l.constructor.name === 'MinKey'){
+			return -1;
+		} else {
+			return 1; // Must be MaxKey, which is greater than everything but MaxKey (which r cannot be)
+		}	
+	}
+	// hack: These should really get converted to their BSON type ids and then compared, we use int vs object in queries
+	if (lt === "number" && rt === "object"){
+		return -1;
+	} else if (lt === "object" && rt === "number") {
+		return 1;
+	}
 	// CW TODO for now, only compare like values
 	if (lt !== rt) throw new Error("can't compare values of BSON types [" + lt + " " + l.constructor.name + "] and [" + rt + ":" + r.constructor.name + "]; code 16016");
 	// Compare everything else

lib/pipeline/accumulators/Accumulator.js

@@ -7,7 +7,7 @@
  * @module mungedb-aggregate
  * @constructor
  **/
-var Accumulator = module.exports = function Accumulator() {
+var Accumulator = module.exports = function Accumulator(){
 	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
 }, klass = Accumulator, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});

lib/pipeline/accumulators/AddToSetAccumulator.js

@@ -1,21 +1,28 @@
 "use strict";
 
-/** 
+/**
  * Create an expression that finds the sum of n operands.
- * @class AddSoSetAccumulator
+ * @class AddToSetAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
  * @module mungedb-aggregate
  * @constructor
 **/
 var AddToSetAccumulator = module.exports = function AddToSetAccumulator(/* ctx */){
 	if (arguments.length !== 0) throw new Error("zero args expected");
-	this.set = {};
+	this.set = [];
 	//this.itr = undefined; /* Shoudln't need an iterator for the set */
 	//this.ctx = undefined; /* Not using the context object currently as it is related to sharding */
 	base.call(this);
 }, klass = AddToSetAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// PROTOTYPE MEMBERS
+// NOTE: Skipping the create function, using the constructor instead
+
+// DEPENDENCIES
+var Value = require("../Value");
+
+
+// MEMBER FUNCTIONS
+
 proto.getOpName = function getOpName(){
 	return "$addToSet";
 };
@@ -24,17 +31,29 @@ proto.getFactory = function getFactory(){
 	return klass;	// using the ctor rather than a separate .create() method
 };
 
-proto.evaluate = function evaluate(doc) {
-	if (arguments.length !== 1) throw new Error("One and only one arg expected");
-	var rhs = this.operands[0].evaluate(doc);
-	if (rhs === undefined) return;
-	this.set[JSON.stringify(rhs)] = rhs;
+
+proto.contains = function contains(value) {
+	var set = this.set;
+	for (var i = 0, l = set.length; i < l; ++i) {
+		if (Value.compare(set[i], value) === 0) {
+			return true;
+		}
+	}
+	return false;
 };
 
-proto.getValue = function getValue() {
-	var setValues = [];
-	for (var setKey in this.set) {
-		setValues.push(this.set[setKey]);
+proto.processInternal = function processInternal(input, merging) {
+	if (! this.contains(input)) {
+		this.set.push(input);
 	}
-	return setValues;
 };
+
+proto.getValue = function getValue(toBeMerged) {
+	return this.set;
+};
+
+proto.reset = function reset() {
+	this.set = [];
+};
+
+

lib/pipeline/accumulators/AvgAccumulator.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * A class for constructing accumulators to calculate avg.
  * @class AvgAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
@@ -11,23 +11,54 @@ var AvgAccumulator = module.exports = function AvgAccumulator(){
 	this.subTotalName = "subTotal";
 	this.countName = "count";
 	this.totalIsANumber = true;
+	this.total = 0;
+	this.count = 0;
 	base.call(this);
-}, klass = AvgAccumulator, SumAccumulator = require("./SumAccumulator"), base = SumAccumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = AvgAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
+// NOTE: Skipping the create function, using the constructor instead
+
+// DEPENDENCIES
+var Value = require("../Value");
+
+// MEMBER FUNCTIONS
+proto.processInternal = function processInternal(input, merging) {
+	if (!merging) {
+		if (typeof input !== "number") {
+			return;
+		}
+		this.total += input;
+		this.count += 1;
+	} else {
+		Value.verifyDocument(input);
+		this.total += input[this.subTotalName];
+		this.count += input[this.countName];
+	}
 };
 
-proto.getValue = function getValue(){
-	if (this.totalIsANumber && this.count > 0) {
-		return this.total / this.count;
-	} else if (this.count === 0) {
-		return 0;
+proto.getValue = function getValue(toBeMerged){
+	if (!toBeMerged) {
+		if (this.totalIsANumber && this.count > 0) {
+			return this.total / this.count;
+		} else if (this.count === 0) {
+			return 0;
+		} else {
+			throw new Error("$sum resulted in a non-numeric type");
+		}
 	} else {
-		throw new Error("$sum resulted in a non-numeric type");
+		var ret = {};
+		ret[this.subTotalName] = this.total;
+		ret[this.countName] = this.count;
+
+		return ret;
 	}
 };
 
+proto.reset = function reset() {
+	this.total = 0;
+	this.count = 0;
+};
+
 proto.getOpName = function getOpName(){
 	return "$avg";
 };

lib/pipeline/accumulators/FirstAccumulator.js

@@ -1,18 +1,22 @@
 "use strict";
 
-/** 
- * Constructor for FirstAccumulator, wraps SingleValueAccumulator's constructor and adds flag to track whether we have started or not
+/**
+ * Constructor for FirstAccumulator, wraps Accumulator and adds flag to track whether we have started or not
  * @class FirstAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
  * @module mungedb-aggregate
  * @constructor
  **/
 var FirstAccumulator = module.exports = function FirstAccumulator(){
+	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-	this.started = 0; //TODO: hack to get around falsy values making us keep going
-}, klass = FirstAccumulator, base = require("./SingleValueAccumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	this._haveFirst = false;
+	this._first = undefined;
+}, klass = FirstAccumulator, base = require("./Accumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// PROTOTYPE MEMBERS
+// NOTE: Skipping the create function, using the constructor instead
+
+// MEMBER FUNCTIONS
 proto.getOpName = function getOpName(){
 	return "$first";
 };
@@ -21,19 +25,23 @@ proto.getFactory = function getFactory(){
 	return klass;	// using the ctor rather than a separate .create() method
 };
 
-/** 
- * Takes a document and returns the first value in the document
- * @param {Object} doc the document source
- * @return the first value
- **/
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
 
-	// only remember the first value seen
-	if (!base.prototype.getValue.call(this) && this.started === 0) {
-		this.value = this.operands[0].evaluate(doc);
-		this.started = 1;
+proto.processInternal = function processInternal(input, merging) {
+	/* only remember the first value seen */
+	if (!this._haveFirst) {
+		// can't use pValue.missing() since we want the first value even if missing
+		this._haveFirst = true;
+		this._first = input;
+		//this._memUsageBytes = sizeof(*this) + input.getApproximateSize() - sizeof(Value);
 	}
+};
+
+proto.getValue = function getValue(toBeMerged) {
+	return this._first;
+};
 
-	return this.value;
+proto.reset = function reset() {
+	this._haveFirst = false;
+	this._first = undefined;
+	this._memUsageBytes = 0;
 };

lib/pipeline/accumulators/LastAccumulator.js

@@ -9,13 +9,24 @@
  **/
 var LastAccumulator = module.exports = function LastAccumulator(){
 	base.call(this);
-}, klass = LastAccumulator, SingleValueAccumulator = require("./SingleValueAccumulator"), base = SingleValueAccumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	this.value = undefined;
+}, klass = LastAccumulator, base = require("./Accumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
-	this.value = this.operands[0].evaluate(doc);
+// NOTE: Skipping the create function, using the constructor instead
+
+// MEMBER FUNCTIONS
+proto.processInternal = function processInternal(input, merging){
+	this.value = input;
+};
+
+proto.getValue = function getValue() {
+	return this.value;
 };
 
 proto.getOpName = function getOpName(){
 	return "$last";
 };
+
+proto.reset = function reset() {
+	this.value = undefined;
+};

lib/pipeline/accumulators/MinMaxAccumulator.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * Constructor for MinMaxAccumulator, wraps SingleValueAccumulator's constructor and adds flag to track whether we have started or not
  * @class MinMaxAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
@@ -12,12 +12,14 @@ var MinMaxAccumulator = module.exports = function MinMaxAccumulator(sense){
 	base.call(this);
 	this.sense = sense; /* 1 for min, -1 for max; used to "scale" comparison */
 	if (this.sense !== 1 && this.sense !== -1) throw new Error("this should never happen");
-}, klass = MinMaxAccumulator, base = require("./SingleValueAccumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = MinMaxAccumulator, base = require("./Accumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// NOTE: Skipping the create function, using the constructor instead
 
 // DEPENDENCIES
 var Value = require("../Value");
 
-// PROTOTYPE MEMBERS
+// MEMBER FUNCTIONS
 proto.getOpName = function getOpName(){
 	if (this.sense == 1) return "$min";
 	return "$max";
@@ -31,22 +33,22 @@ klass.createMax = function createMax(){
 	return new MinMaxAccumulator(-1);
 };
 
-/** 
- * Takes a document and returns the first value in the document
- * @param {Object} doc the document source
- * @return the first value
- **/
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
-	var prhs = this.operands[0].evaluate(doc);
+proto.reset = function reset() {
+	this.value = undefined;
+};
+
+proto.getValue = function getValue(toBeMerged) {
+	return this.value;
+};
 
+proto.processInternal = function processInternal(input, merging) {
 	// if this is the first value, just use it
 	if (!this.hasOwnProperty('value')) {
-		this.value = prhs;
+		this.value = input;
 	} else {
 		// compare with the current value; swap if appropriate
-		var cmp = Value.compare(this.value, prhs) * this.sense;
-		if (cmp > 0) this.value = prhs;
+		var cmp = Value.compare(this.value, input) * this.sense;
+		if (cmp > 0) this.value = input;
 	}
 
 	return this.value;

lib/pipeline/accumulators/PushAccumulator.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * Constructor for PushAccumulator. Pushes items onto an array.
  * @class PushAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
@@ -12,17 +12,40 @@ var PushAccumulator = module.exports = function PushAccumulator(){
 	base.call(this);
 }, klass = PushAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
-	var v = this.operands[0].evaluate(doc);
-	if (v !== undefined) this.values.push(v);
-	return null;
-};
+// NOTE: Skipping the create function, using the constructor instead
 
-proto.getValue = function getValue(){
+// MEMBER FUNCTIONS
+proto.getValue = function getValue(toBeMerged){
 	return this.values;
 };
 
 proto.getOpName = function getOpName(){
 	return "$push";
 };
+
+proto.getFactory = function getFactory(){
+	return klass;	// using the ctor rather than a separate .create() method
+};
+
+
+proto.processInternal = function processInternal(input, merging) {
+	if (!merging) {
+		if (input !== undefined) {
+			this.values.push(input);
+			//_memUsageBytes += input.getApproximateSize();
+		}
+	}
+	else {
+		// If we're merging, we need to take apart the arrays we
+		// receive and put their elements into the array we are collecting.
+		// If we didn't, then we'd get an array of arrays, with one array
+		// from each merge source.
+		if (!(input instanceof Array)) throw new Error("input is not an Array during merge in PushAccumulator:35");
+
+		this.values = this.values.concat(input);
+
+		//for (size_t i=0; i < vec.size(); i++) {
+			//_memUsageBytes += vec[i].getApproximateSize();
+		//}
+	}
+};

lib/pipeline/accumulators/SingleValueAccumulator.js

@@ -1,24 +0,0 @@
-"use strict";
-
-/**
- * This isn't a finished accumulator, but rather a convenient base class
- * for others such as $first, $last, $max, $min, and similar.  It just
- * provides a holder for a single Value, and the getter for that.  The
- * holder is protected so derived classes can manipulate it.
- *
- * @class SingleValueAccumulator
- * @namespace mungedb-aggregate.pipeline.accumulators
- * @module mungedb-aggregate
- * @constructor
-**/
-var SingleValueAccumulator = module.exports = function SingleValueAccumulator(){
-	if (arguments.length > 1) throw new Error("expects a single value");
-	base.call(this);
-}, klass = SingleValueAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
-
-// DEPENDENCIES
-var Value = require("../Value");
-
-proto.getValue = function getValue(){
-	return this.value;
-};

lib/pipeline/accumulators/SumAccumulator.js

@@ -14,22 +14,20 @@ var SumAccumulator = module.exports = function SumAccumulator(){
 	base.call(this);
 }, klass = SumAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
-	var v = this.operands[0].evaluate(doc);
+// NOTE: Skipping the create function, using the constructor instead
 
-	if (typeof v !== "number") { // do nothing with non-numeric types
-		return 0;
-	} else {
+// MEMBER FUNCTIONS
+proto.processInternal = function processInternal(input, merging) {
+	if(typeof input === "number"){ // do nothing with non-numeric types
 		this.totalIsANumber = true;
-		this.total += v;
+		this.total += input;
 	}
 	this.count++;
 
 	return 0;
 };
 
-proto.getValue = function getValue(){
+proto.getValue = function getValue(toBeMerged){
 	if (this.totalIsANumber) {
 		return this.total;
 	}

lib/pipeline/documentSources/CursorDocumentSource.js

@@ -1,5 +1,12 @@
 "use strict";
 
+var DocumentSource = require('./DocumentSource'),
+	LimitDocumentSource = require('./LimitDocumentSource');
+
+// Mimicking max memory size from mongo/db/query/new_find.cpp
+// Need to actually decide some size for this?
+var MAX_BATCH_DOCS = 150;
+
 /**
  * Constructs and returns Documents from the objects produced by a supplied Cursor.
  * An object of this type may only be used by one thread, see SERVER-6123.
@@ -30,6 +37,10 @@ var CursorDocumentSource = module.exports = CursorDocumentSource = function Curs
 	this._projection = null;
 
 	this._cursorWithContext = cursorWithContext;
+	this._curIdx = 0;
+	this._currentBatch = [];
+	this._limit = undefined;
+	this._docsAddedToBatches = 0;
 
 	if (!this._cursorWithContext || !this._cursorWithContext._cursor) throw new Error("CursorDocumentSource requires a valid cursorWithContext");
 
@@ -59,6 +70,43 @@ klass.CursorWithContext = (function (){
  **/
 proto.dispose = function dispose() {
 	this._cursorWithContext = null;
+	this._currentBatch = [];
+	this._curIdx = 0;
+};
+
+proto.getSourceName = function getSourceName() {
+	return "$cursor";
+};
+
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	if (this._currentBatch.length <= this._curIdx) {
+		this.loadBatch();
+
+		if (this._currentBatch.length <= this._curIdx) {
+			callback(null, DocumentSource.EOF);
+			return DocumentSource.EOF;
+		}
+	}
+
+	// Don't unshift. It's expensiver.
+	var out = this._currentBatch[this._curIdx];
+	this._curIdx++;
+
+	callback(null, out);
+	return out;
+};
+
+proto.coalesce = function coalesce(nextSource) {
+	if (this._limit) {
+		return this._limit.coalesce(nextSource);
+	} else if (nextSource instanceof LimitDocumentSource) {
+		this._limit = nextSource;
+		return this._limit;
+	} else {
+		return false;
+	}
 };
 
 ///**
@@ -78,9 +126,10 @@ proto.dispose = function dispose() {
 // * @method	setQuery
 // * @param	{Object}	pBsonObj	the query to record
 // **/
-//proto.setQuery = function setQuery(pBsonObj) {};
-//
-//
+proto.setQuery = function setQuery(query) {
+	this._query = query;
+};
+
 ///**
 // * Record the sort that was specified for the cursor this wraps, if any.
 // * This should be captured after any optimizations are applied to
@@ -98,7 +147,7 @@ proto.dispose = function dispose() {
  * @method	setProjection
  * @param	{Object}	projection
  **/
-proto.setProjection = function setProjection(projection) {
+proto.setProjection = function setProjection(projection, deps) {
 
 	if (this._projection){
 		throw new Error("projection is already set");
@@ -113,42 +162,10 @@ proto.setProjection = function setProjection(projection) {
 //	this.cursor().fields = this._projection;
 
 	this._projection = projection;  //just for testing
+	this._dependencies = deps;
 };
 
 //----------------virtuals from DocumentSource--------------
-/**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	if (!this.current) this.findNext(); // if we haven't gotten the first one yet, do so now
-	return (this.current === null);
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-	if (!this.current) this.findNext(); // if we haven't gotten the first one yet, do so now
-	this.findNext();
-	return (this.current !== null);
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	if (!this.current) this.findNext(); // if we haven't gotten the first one yet, do so now
-	return this.current;
-};
 
 /**
  * Set the underlying source this source should use to get Documents
@@ -165,74 +182,76 @@ proto.getCurrent = function getCurrent() {
  * @param source   {DocumentSource}  the underlying source to use
  * @param callback  {Function}        a `mungedb-aggregate`-specific extension to the API to half-way support reading from async sources
  **/
-proto.setSource = function setSource(theSource, callback) {
+proto.setSource = function setSource(theSource) {
 	if (theSource) throw new Error("CursorDocumentSource doesn't take a source"); //TODO: This needs to put back without the if once async is fully and properly supported
-	if (callback) return setTimeout(callback, 0);
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToBsonArray()
- * to add this object to a pipeline being represented in BSON.
- *
- * @method	sourceToJson
- * @param	{Object} pBuilder	BSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(pBuilder, explain) {
-	/* this has no analog in the BSON world, so only allow it for explain */
-	//if (explain){
-	////we are not currently supporting explain in mungedb-aggregate
-	//}
+proto.serialize = function serialize(explain) {
+	if (!explain)
+		return null;
+
+	if (!this._cursorWithContext)
+		throw new Error("code 17135; Cursor deleted.");
+
+	// A stab at what mongo wants
+	return {
+		query: this._query,
+		sort: this._sort ? this._sort : null,
+		limit: this._limit ? this._limit : null,
+		fields: this._projection ? this._projection : null,
+		indexonly: false,
+		cursorType: this._cursorWithContext ? "cursor" : null
+	};
 };
 
-//----------------private--------------
+// LimitDocumentSource has the setLimit function which trickles down to any documentsource
+proto.getLimit = function getLimit() {
+	return this._limit ? this._limit.getLimit() : -1;
+};
 
-proto.findNext = function findNext(){
+//----------------private--------------
 
-	if ( !this._cursorWithContext ) {
-		this.current = null;
-		return;
-	}
+//proto.chunkMgr = function chunkMgr(){};
 
-	for( ; this.cursor().ok(); this.cursor().advance() ) {
+//proto.canUseCoveredIndex = function canUseCoveredIndex(){};
 
-		//yieldSometimes();
-//		if ( !this.cursor().ok() ) {
-//			// The cursor was exhausted during the yield.
-//			break;
-//		}
+//proto.yieldSometimes = function yieldSometimes(){};
 
-//		if ( !this.cursor().currentMatches() || this.cursor().currentIsDup() )
-//			continue;
+proto.loadBatch = function loadBatch() {
+	var nDocs = 0,
+		cursor = this._cursorWithContext ? this._cursorWithContext._cursor : null;
 
+	if (!cursor)
+		return this.dispose();
 
-		// grab the matching document
-		var documentObj;
-//		if (this.canUseCoveredIndex()) { ...  Dont need any of this, I think
+	for(;cursor.ok(); cursor.advance()) {
+		if (!cursor.ok())
+			break;
 
-		documentObj = this.cursor().current();
-		this.current = documentObj;
-		this.cursor().advance();
-		return;
-	}
+		// these methods do not exist
+		// if (!cursor.currentMatches() || cursor.currentIsDup())
+		// continue;
 
-	// If we got here, there aren't any more documents.
-	// The CursorWithContext (and its read lock) must be released, see SERVER-6123.
-	this.dispose();
-	this.current = null;
-};
+		var next = cursor.current();
+		this._currentBatch.push(this._projection ? base.documentFromJsonWithDeps(next, this._dependencies) : next);
 
-proto.cursor = function cursor(){
-	if( this._cursorWithContext && this._cursorWithContext._cursor){
-		return this._cursorWithContext._cursor;
-	}
-	throw new Error("cursor not defined");
-};
+		if (this._limit) {
+			this._docsAddedToBatches++;
+			if (this._docsAddedToBatches == this._limit.getLimit())
+				break;
 
-//proto.chunkMgr = function chunkMgr(){};
+			if (this._docsAddedToBatches >= this._limit.getLimit()) {
+				throw new Error("added documents to the batch over limit size");
+			}
+		}
 
-//proto.canUseCoveredIndex = function canUseCoveredIndex(){};
+		// Mongo uses number of bytes, but that doesn't make sense here. Yield when nDocs is over a threshold
+		if (nDocs > MAX_BATCH_DOCS) {
+			this._curIdx++; // advance the deque
+			nDocs++;
+			return;
+		}
+	}
 
-//proto.yieldSometimes = function yieldSometimes(){};
+	this._cursorWithContext = undefined;	//NOTE: Trying to emulate erasing the cursor; not exactly how mongo does it
+};

lib/pipeline/documentSources/DocumentSource.js

@@ -37,6 +37,22 @@ var DocumentSource = module.exports = function DocumentSource(expCtx){
 
 }, klass = DocumentSource, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
+/**
+ * Use EOF as boost::none for document sources to signal the end of their document stream.
+ **/
+klass.EOF = (function() {
+	/**
+	 * Represents a non-value in a document stream
+	 * @class EOF
+	 * @namespace mungedb-aggregate.pipeline.documentSources.DocumentSource
+	 * @module mungedb-aggregate
+	 * @constructor
+	 **/
+	var klass = function EOF(){
+	};
+	return klass;
+})();
+
 /*
 class DocumentSource :
 public IntrusiveCounterUnsigned,
@@ -67,32 +83,13 @@ proto.getPipelineStep = function getPipelineStep() {
 };
 
 /**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	throw new Error("not implemented");
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
+ * Returns the next Document if there is one or DocumentSource.EOF if at EOF.
  *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	//pExpCtx->checkForInterrupt(); // might not return
-	return false;
-};
-
-/**
  * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
+ * @method	getNext
  * @returns	{Document}	the current Document without advancing
  **/
-proto.getCurrent = function getCurrent() {
+proto.getNext = function getNext(callback) {
 	throw new Error("not implemented");
 };
 
@@ -136,10 +133,9 @@ proto.getSourceName = function getSourceName() {
  * @method	setSource
  * @param	{DocumentSource}	source	the underlying source to use
  **/
-proto.setSource = function setSource(theSource, callback) {
+proto.setSource = function setSource(theSource) {
 	if (this.source) throw new Error("It is an error to set the source more than once");
 	this.source = theSource;
-	if (callback) return setTimeout(callback, 0);
 };
 
 /**
@@ -198,62 +194,139 @@ proto.getDependencies = function getDependencies(deps) {
  * @returns	{Object}	JSONObj
  **/
 klass.depsToProjection = function depsToProjection(deps) {
-	var bb = {};
+	var needId = false,
+		bb = {};
 	if (deps._id === undefined)
 		bb._id = 0;
 
 	var last = "";
 	Object.keys(deps).sort().forEach(function(it){
-		if (last !== "" && it.slice(0, last.length) === last){
-			// we are including a parent of *it so we don't need to
-			// include this field explicitly. In fact, due to
-			// SERVER-6527 if we included this field, the parent
-			// wouldn't be fully included.
+		if (it.indexOf('_id') === 0 && (it.length === 3 || it[3] === '.')) {
+			needId = true;
 			return;
+		} else {
+			if (last !== "" && it.slice(0, last.length) === last){
+				// we are including a parent of *it so we don't need to
+				// include this field explicitly. In fact, due to
+				// SERVER-6527 if we included this field, the parent
+				// wouldn't be fully included.
+				return;
+			}
 		}
 		last = it + ".";
 		bb[it] = 1;
 	});
 
+	if (needId) // we are explicit either way
+		bb._id = 1;
+	else
+		bb._id = 0;
+
+
 	return bb;
 };
 
-/**
- * Add the DocumentSource to the array builder.
- * The default implementation calls sourceToJson() in order to
- * convert the inner part of the object which will be added to the
- * array being built here.
- *
- * @method	addToJsonArray
- * @param	{Array} pBuilder	JSONArrayBuilder: the array builder to add the operation to.
- * @param	{Boolean}	explain	create explain output
- * @returns	{Object}
- **/
-proto.addToJsonArray = function addToJsonArray(pBuilder, explain) {
-	pBuilder.push(this.sourceToJson({}, explain));
+proto._serialize = function _serialize(explain) {
+	throw new Error("not implemented");
+};
+
+proto.serializeToArray = function serializeToArray(array, explain) {
+	var entry = this.serialize(explain);
+	if (entry) {
+		array.push(entry);
+	}
+};
+
+klass.parseDeps = function parseDeps(deps) {
+	var md = {};
+
+	var last,
+		depKeys = Object.keys(deps);
+	for (var i = 0; i < depKeys.length; i++) {
+		var it = depKeys[i],
+			value = deps[it];
+
+		if (!last && it.indexOf(last) >= 0)
+			continue;
+		last = it + '.';
+		md[it] = true;
+	}
+	return md;
 };
 
 /**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
+ * A function compatible as a getNext for document sources.
+ * Does nothing except pass the documents through. To use,
+ * Attach this function on a DocumentSource prototype.
  *
- * @method	sourceToJson
- * @param	{Object} pBuilder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
+ * @method GET_NEXT_PASS_THROUGH
+ * @param callback {Function}
+ * @param callback.err {Error} An error or falsey
+ * @param callback.doc {Object} The source's next object or DocumentSource.EOF
  **/
-proto.sourceToJson = function sourceToJson(pBuilder, explain) {
-	throw new Error("not implemented");
+klass.GET_NEXT_PASS_THROUGH = function GET_NEXT_PASS_THROUGH(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var out;
+	this.source.getNext(function(err, doc) {
+		out = doc;
+		return callback(err, doc);
+	});
+	return out; // For the sync people in da house
 };
 
-/**
- * Convert the DocumentSource instance to it's JSON Object representation; Used by the standard JSON.stringify() function
- * @method toJSON
- * @return {String} a JSON-encoded String that represents the DocumentSource
- **/
-proto.toJSON = function toJSON(){
-	var obj = {};
-	this.sourceToJson(obj);
-	return obj;
+klass.documentFromJsonWithDeps = function documentFromJsonWithDeps(bson, neededFields) {
+	var arrayHelper = function(bson, neededFields) {
+		var values = [];
+
+		var bsonKeys = Object.keys(bson);
+		for (var i = 0; i < bsonKeys.length; i++) {
+			var key = bsonKeys[i],
+				bsonElement = bson[key];
+
+			if (bsonElement instanceof Object) {
+				var sub = klass.documentFromJsonWithDeps(bsonElement, isNeeded);
+				values.push(sub);
+			}
+
+			if (bsonElement instanceof Array) {
+				values.push(arrayHelper(bsonElement, neededFields));
+			}
+		}
+
+		return values;
+	};
+
+	var md = {};
+
+	var bsonKeys = Object.keys(bson);
+	for (var i = 0; i < bsonKeys.length; i++) {
+		var fieldName = bsonKeys[i],
+			bsonElement = bson[fieldName],
+			isNeeded = neededFields ? neededFields[fieldName] : null;
+
+		if (!isNeeded)
+			continue;
+
+		if (typeof(isNeeded) === 'boolean') {
+			md[fieldName] = bsonElement;
+			continue;
+		}
+
+		if (!isNeeded instanceof Object)
+			throw new Error("instanceof should be an instance of Object");
+
+		if (bsonElement instanceof Object) {
+			var sub = klass.documentFromJsonWithDeps(bsonElement, isNeeded);
+
+			md[fieldName] = sub;
+		}
+
+		if (bsonElement instanceof Array) {
+			md[fieldName] = arrayHelper(bsonElement, isNeeded);
+		}
+	}
+
+	return md;
+
 };

lib/pipeline/documentSources/FilterBaseDocumentSource.js

@@ -1,114 +0,0 @@
-"use strict";
-
-/**
- * A base class for filter document sources
- * @class FilterBaseDocumentSource
- * @namespace mungedb-aggregate.pipeline.documentSources
- * @module mungedb-aggregate
- * @constructor
- * @param [ctx] {ExpressionContext}
- **/
-var FilterBaseDocumentSource = module.exports = function FilterBaseDocumentSource(ctx){
-	if (arguments.length > 1) throw new Error("up to one arg expected");
-	base.call(this, ctx);
-	this.unstarted = true;
-	this.hasNext = false;
-	this.current = null;
-}, klass = FilterBaseDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
-
-//TODO: Need to implement coalesce()
-//TODO: Need to implement optimize()
-
-/**
- * Find the next acceptable source document, if there are any left.
- * @method findNext
- **/
-proto.findNext = function findNext() {
-	/* only do this the first time */
-	if (this.unstarted) {
-		this.hasNext = !this.source.eof();
-		this.unstarted = false;
-	}
-
-	while(this.hasNext) {
-		var document = this.source.getCurrent();
-		this.hasNext = this.source.advance();
-
-		if (this.accept(document)) {
-			this.current = document;
-			return;
-		}
-	}
-
-	this.current = null;
-};
-
-/**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	if (this.unstarted)
-		this.findNext();
-	return (this.current === null);
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-
-	if (this.unstarted)
-		this.findNext();
-
-	/**
-	* This looks weird after the above, but is correct.  Note that calling
-	* getCurrent() when first starting already yields the first document
-	* in the collection.  Calling advance() without using getCurrent()
-	* first will skip over the first item.
-	**/
-	this.findNext();
-	return (this.current !== null);
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	if (this.unstarted)
-		this.findNext();
-	if (this.current === null) throw new Error("This should never happen");
-	return this.current;
-};
-
-/**
-* Test the given document against the predicate and report if it should be accepted or not.
-* @param {object} document the document to test
-* @returns {bool} true if the document matches the filter, false otherwise
-**/
-proto.accept = function accept(document) {
-	throw new Error("not implemented");
-};
-
-/**
-* Create a JSONObj suitable for Matcher construction.
-*
-* This is used after filter analysis has moved as many filters to
-* as early a point as possible in the document processing pipeline.
-* See db/Matcher.h and the associated wiki documentation for the
-* format.  This conversion is used to move back to the low-level
-* find() Cursor mechanism.
-*
-* @param builder the builder to write to
-**/
-proto.toMatcherJson = function toMatcherJson(builder) {
-	throw new Error("not implemented");
-};

lib/pipeline/documentSources/GeoNearDocumentSource.js

@@ -0,0 +1,131 @@
+"use strict";
+
+var async = require('async'),
+	DocumentSource = require('./DocumentSource'),
+	FieldPath = require('../FieldPath');
+
+/**
+ * @class GeoNearDocumentSource
+ * @namespace mungedb-aggregate.pipeline.documentSources
+ * @module mungedb-aggregate
+ * @constructor
+ * @param [ctx] {ExpressionContext}
+ **/
+var GeoNearDocumentSource = module.exports = function GeoNearDocumentSource(ctx) {
+	if (arguments.length > 1) throw new Error("up to one arg expected");
+	base.call(this, ctx);
+	// mongo defaults
+	this.coordsIsArray = false;
+	this.limit = 100;
+	this.maxDistance = -1.0;
+	this.spherical = false;
+	this.distanceMultiplier = 1.0;
+	this.uniqueDocs = true;
+}, klass = GeoNearDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+klass.geoNearName = "$geoNear";
+
+proto.getSourceName = function() {
+	return klass.geoNearName;
+};
+
+proto.getNext = DocumentSource.GET_NEXT_PASS_THROUGH;
+
+proto.setSource = function(docSource) {
+	throw new Error('code 16602; $geoNear is only allowed as the first pipeline stage');
+};
+
+proto.isValidInitialSource = function() {
+	return true;
+};
+
+proto.serialize = function(explain) {
+	var result = {};
+
+	if (this.coordsIsArray)
+		result.near = this.near;
+	else
+		result.near = [this.near];
+
+	// not in buildGeoNearCmd
+	result.distanceField = this.distanceField.getPath(false);
+
+	result.limit = this.limit;
+
+	if (this.maxDistance > 0)
+		result.maxDistance = this.maxDistance;
+
+	if (this.query)
+		result.query = this.query;
+
+	if (this.spherical)
+		result.spherical = this.spherical;
+
+	if (this.distanceMultiplier)
+		result.distanceMultiplier = this.distanceMultiplier;
+
+	if (this.includeLocs) {
+		if (typeof this.includeLocs !== 'string')
+			throw new Error('code 16607; $geoNear requires that \'includeLocs\' option is a String');
+		result.includeLocs = this.includeLocs.getPath(false);
+	}
+
+	if (this.uniqueDocs)
+		result.uniqueDocs = this.uniqueDocs;
+
+	var sourceName = this.getSourceName(),
+		returnObj = {};
+	returnObj[sourceName] = result;
+
+	return returnObj;
+};
+
+klass.createFromJson = function(jsonElement, ctx) {
+	var out = new GeoNearDocumentSource(ctx);
+	out.parseOptions(jsonElement);
+	return out;
+};
+
+proto.parseOptions = function(options) {
+
+	// near and distanceField are required
+	if (!options.near || !Array.isArray(options.near))
+		throw new Error('code 16605; $geoNear requires a \'near\' option as an Array');
+	this.coordsIsArray = options.near instanceof Array;
+
+	if (typeof options.distanceField !== 'string')
+		throw new Error('code 16606: $geoNear  a \'distanceNear\' option as a String');
+	this.distanceField = new FieldPath(options.distanceField);
+
+	// remaining fields are optional
+
+	// num and limits are synonyms
+	if (typeof options.limit === 'number')
+		this.limit = options.limit;
+	if (typeof options.num === 'number')
+		this.limit = options.num;
+
+	if (typeof options.maxDistance === 'number')
+		this.maxDistance = options.maxDistance;
+
+	if (options.query instanceof Object)
+		this.query = options.query;
+
+	if (options.spherical)
+		this.spherical = options.spherical;
+
+	if (typeof options.distanceMultiplier === 'number')
+		this.distanceMultiplier = options.distanceMultiplier;
+
+	if (options.includeLocs) {
+		if (typeof options.includeLocs !== 'string')
+			throw new Error('code 16607; $geoNear requires that \'includeLocs\' option is a String');
+		this.includeLocs = new FieldPath(options.includeLocs);
+	}
+
+	if (options.uniqueDocs)
+		this.uniqueDocs;
+};
+
+
+

lib/pipeline/documentSources/GroupDocumentSource.js

@@ -4,16 +4,20 @@ var DocumentSource = require("./DocumentSource"),
 	Document = require("../Document"),
 	Expression = require("../expressions/Expression"),
 	ConstantExpression = require("../expressions/ConstantExpression"),
-	FieldPathExpression = require("../expressions/FieldPathExpression");
-
+	FieldPathExpression = require("../expressions/FieldPathExpression"),
+	Variables = require("../expressions/Variables"),
+	VariablesIdGenerator = require("../expressions/VariablesIdGenerator"),
+	VariablesParseState = require("../expressions/VariablesParseState"),
+	async = require("async");
 
 /**
  * A class for grouping documents together
+ *
  * @class GroupDocumentSource
  * @namespace mungedb-aggregate.pipeline.documentSources
  * @module mungedb-aggregate
  * @constructor
- * @param [ctx] {ExpressionContext}
+ * @param [expCtx] {ExpressionContext}
  **/
 var GroupDocumentSource = module.exports = function GroupDocumentSource(expCtx) {
 	if (arguments.length > 1) throw new Error("up to one arg expected");
@@ -24,7 +28,7 @@ var GroupDocumentSource = module.exports = function GroupDocumentSource(expCtx)
 	this.groups = {}; // GroupsType Value -> Accumulators[]
 	this.groupsKeys = []; // This is to faciliate easier look up of groups
 	this.originalGroupsKeys = []; // This stores the original group key un-hashed/stringified/whatever
-
+	this._variables = null;
 	this.fieldNames = [];
 	this.accumulatorFactories = [];
 	this.expressions = [];
@@ -38,7 +42,7 @@ klass.groupOps = {
 	"$avg": Accumulators.Avg,
 	"$first": Accumulators.First,
 	"$last": Accumulators.Last,
-	"$max": Accumulators.MinMax.createMax,
+	"$max": Accumulators.MinMax.createMax, // $min and $max have special constructors because they share base features
 	"$min": Accumulators.MinMax.createMin,
 	"$push": Accumulators.Push,
 	"$sum": Accumulators.Sum
@@ -46,43 +50,142 @@ klass.groupOps = {
 
 klass.groupName = "$group";
 
+/**
+ * Factory for making GroupDocumentSources
+ *
+ * @method create
+ * @static
+ * @param [expCtx] {ExpressionContext}
+ **/
+klass.create = function create(expCtx) {
+	return new GroupDocumentSource(expCtx);
+};
+
+/**
+ * Factory for making GroupDocumentSources
+ *
+ * @method getSourceName
+ * @return {GroupDocumentSource}
+ **/
 proto.getSourceName = function getSourceName() {
 	return klass.groupName;
 };
 
 /**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
+ * Gets the next document or DocumentSource.EOF if none
  *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
+ * @method getNext
+ * @return {Object}
  **/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	var idExp = this.idExpression,
-		insides = {
-			_id: idExp ? idExp.toJSON() : {}
+proto.getNext = function getNext(callback) {
+	var self = this;
+	async.series([
+		function(next) {
+			if (!self.populated)
+				self.populate(function(err) {
+					return next(err);
+				});
+			else
+				return next();
 		},
-		aFac = this.accumulatorFactories,
-		aFacLen = aFac.length;
+		function(next) {
+			if(Object.keys(self.groups).length === 0) {
+				return next(null, DocumentSource.EOF);
+			}
+
+			//Note: Skipped the spilled logic
+
+			if(self.currentGroupsKeysIndex === self.groupsKeys.length) {
+				return next(null, DocumentSource.EOF);
+			}
+
+			var id = self.groupsKeys[self.currentGroupsKeysIndex],
+				accumulators = self.groups[id],
+				out = self.makeDocument(id, accumulators /*,mergeableOutput*/);
+
+			if(++self.currentGroupsKeysIndex === self.groupsKeys.length) {
+				self.dispose();
+			}
+
+			return next(null, out);
+		}
+	], function(err, results) {
+		callback(err, results[1]);
+	});
+};
+
+/**
+ * Sets this source as apparently empty
+ *
+ * @method dispose
+ **/
+proto.dispose = function dispose() {
+	//NOTE: Skipped 'freeing' our resources; at best we could remove some references to things, but our parent will probably forget us anyways!
 
-	for(var i=0; i < aFacLen; ++i) {
-		var acc = new aFac[i](/*pExpCtx*/);
-		acc.addOperand(this.expressions[i]);
+	// make us look done
+	this.currentGroupsKeysIndex = this.groupsKeys.length;
 
-		insides[this.fieldNames[i]] = acc.toJSON(true);
+	// free our source's resources
+	this.source.dispose();
+};
+
+/**
+ * Optimizes the expressions in the group
+ * @method optimize
+ **/
+proto.optimize = function optimize() {
+	var self = this;
+	self.idExpression = self.idExpression.optimize();
+	self.expressions.forEach(function(expression, i) {
+		self.expressions[i] = expression.optimize();
+	});
+};
+
+/**
+ * Create an object that represents the document source.  The object
+ * will have a single field whose name is the source's name.
+ *
+ * @method	serialize
+ * @param explain {Boolean} Create explain output
+ **/
+proto.serialize = function serialize(explain) {
+	var insides = {};
+
+	// add the _id
+	insides._id = this.idExpression.serialize(explain);
+
+	//add the remaining fields
+	var aFacs = this.accumulatorFactories,
+		aFacLen = aFacs.length;
+
+	for(var i=0; i < aFacLen; i++) {
+		var aFac = aFacs[i](),
+			serialExpression = this.expressions[i].serialize(explain), //Get the accumulator's expression
+			serialAccumulator = {}; //Where we'll put the expression
+		serialAccumulator[aFac.getOpName()] = serialExpression;
+		insides[this.fieldNames[i]] = serialAccumulator;
 	}
 
-	builder[this.getSourceName()] = insides;
+	var serialSource = {};
+	serialSource[this.getSourceName()] = insides;
+	return serialSource;
 };
 
-klass.createFromJson = function createFromJson(groupObj, ctx) {
-	if (!(groupObj instanceof Object && groupObj.constructor === Object)) throw new Error("a group's fields must be specified in an object");
+/**
+ * Creates a GroupDocumentSource from the given elem
+ *
+ * @method	createFromJson
+ * @param elem {Object} The group specification object; the right hand side of the $group
+ **/
+klass.createFromJson = function createFromJson(elem, expCtx) {
+	if (!(elem instanceof Object && elem.constructor === Object)) throw new Error("a group's fields must be specified in an object");
+
+	var group = GroupDocumentSource.create(expCtx),
+		idSet = false;
 
-	var idSet = false,
-		group = new GroupDocumentSource(ctx);
+	var groupObj = elem,
+		idGenerator = new VariablesIdGenerator(),
+		vps = new VariablesParseState(idGenerator);
 
 	for (var groupFieldName in groupObj) {
 		if (groupObj.hasOwnProperty(groupFieldName)) {
@@ -93,106 +196,168 @@ klass.createFromJson = function createFromJson(groupObj, ctx) {
 				if(idSet) throw new Error("15948 a group's _id may only be specified once");
 
 				if (groupField instanceof Object && groupField.constructor === Object) {
+					/*
+						Use the projection-like set of field paths to create the
+						group-by key.
+					*/
 					var objCtx = new Expression.ObjectCtx({isDocumentOk:true});
-					group.idExpression = Expression.parseObject(groupField, objCtx);
+					group.setIdExpression(Expression.parseObject(groupField, objCtx, vps));
 					idSet = true;
 
 				} else if (typeof groupField === "string") {
-					if (groupField[0] !== "$") {
-						group.idExpression = new ConstantExpression(groupField);
-					} else {
-						var pathString = Expression.removeFieldPrefix(groupField);
-						group.idExpression = new FieldPathExpression(pathString);
-					}
-					idSet = true;
-
-				} else {
-					var typeStr = group._getTypeStr(groupField);
-					switch (typeStr) {
-						case "number":
-						case "string":
-						case "boolean":
-						case "Object":
-						case "object": // null returns "object" Xp
-						case "Array":
-							group.idExpression = new ConstantExpression(groupField);
-							idSet = true;
-							break;
-						default:
-							throw new Error("a group's _id may not include fields of type " + typeStr  + "");
+					if (groupField[0] === "$") {
+						group.setIdExpression(FieldPathExpression.parse(groupField, vps));
+						idSet = true;
 					}
 				}
 
+				if (!idSet) {
+					// constant id - single group
+					group.setIdExpression(ConstantExpression.create(groupField));
+					idSet = true;
+				}
 
 			} else {
+				/*
+					Treat as a projection field with the additional ability to
+					add aggregation operators.
+				*/
 				if (groupFieldName.indexOf(".") !== -1) throw new Error("16414 the group aggregate field name '" + groupFieldName + "' cannot contain '.'");
 				if (groupFieldName[0] === "$") throw new Error("15950 the group aggregate field name '" + groupFieldName + "' cannot be an operator name");
 				if (group._getTypeStr(groupFieldName) === "Object") throw new Error("15951 the group aggregate field '" + groupFieldName + "' must be defined as an expression inside an object");
 
-				var subFieldCount = 0;
-				for (var subFieldName in groupField) {
-					if (groupField.hasOwnProperty(subFieldName)) {
-						var subField = groupField[subFieldName],
-							op = klass.groupOps[subFieldName];
-						if (!op) throw new Error("15952 unknown group operator '" + subFieldName + "'");
+				var subElementCount = 0;
+				for (var subElementName in groupField) {
+					if (groupField.hasOwnProperty(subElementName)) {
+						var subElement = groupField[subElementName],
+							op = klass.groupOps[subElementName];
+						if (!op) throw new Error("15952 unknown group operator '" + subElementName + "'");
 
 						var groupExpression,
-							subFieldTypeStr = group._getTypeStr(subField);
-						if (subFieldTypeStr === "Object") {
-							var subFieldObjCtx = new Expression.ObjectCtx({isDocumentOk:true});
-							groupExpression = Expression.parseObject(subField, subFieldObjCtx);
-						} else if (subFieldTypeStr === "Array") {
-							throw new Error("15953 aggregating group operators are unary (" + subFieldName + ")");
-						} else {
-							groupExpression = Expression.parseOperand(subField);
+							subElementTypeStr = group._getTypeStr(subElement);
+						if (subElementTypeStr === "Object") {
+							var subElementObjCtx = new Expression.ObjectCtx({isDocumentOk:true});
+							groupExpression = Expression.parseObject(subElement, subElementObjCtx, vps);
+						} else if (subElementTypeStr === "Array") {
+							throw new Error("15953 aggregating group operators are unary (" + subElementName + ")");
+						} else { /* assume its an atomic single operand */
+							groupExpression = Expression.parseOperand(subElement, vps);
 						}
-						group.addAccumulator(groupFieldName,op, groupExpression);
+						group.addAccumulator(groupFieldName, op, groupExpression);
 
-						++subFieldCount;
+						++subElementCount;
 					}
 				}
-				if (subFieldCount != 1) throw new Error("15954 the computed aggregate '" + groupFieldName + "' must specify exactly one operator");
+				if (subElementCount !== 1) throw new Error("15954 the computed aggregate '" + groupFieldName + "' must specify exactly one operator");
 			}
 		}
 	}
 
 	if (!idSet) throw new Error("15955 a group specification must include an _id");
 
+	group._variables = new Variables(idGenerator.getIdCount());
+
 	return group;
 };
 
-proto._getTypeStr = function _getTypeStr(obj) {
-	var typeofStr = typeof obj,
-		typeStr = (typeofStr == "object" && obj !== null) ? obj.constructor.name : typeofStr;
-	return typeStr;
-};
+/**
+ * Populates the GroupDocumentSource by grouping all of the input documents at once.
+ *
+ * @method populate
+ * @param callback {Function} Required. callback(err) when done populating.
+ * @async
+ **/
+proto.populate = function populate(callback) {
+	var numAccumulators = this.accumulatorFactories.length;
+	if(numAccumulators !== this.expressions.length) {
+		callback(new Error("Must have equal number of accumulators and expressions"));
+	}
 
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // Check for interupts ????
-	if(!this.populated) this.populate();
+	var input,
+		self = this;
+	async.whilst(
+		function() {
+			return input !== DocumentSource.EOF;
+		},
+		function(cb) {
+			self.source.getNext(function(err, doc) {
+				if(err) return cb(err);
+				if(doc === DocumentSource.EOF) {
+					input = doc;
+					return cb(); //Need to stop now, no new input
+				}
 
-	//verify(this.currentGroupsKeysIndex < this.groupsKeys.length);
+				input = doc;
+				self._variables.setRoot(input);
 
-	++this.currentGroupsKeysIndex;
-	if (this.currentGroupsKeysIndex >= this.groupsKeys.length) {
-		this.currentDocument = null;
-		return false;
-	}
+				/* get the _id value */
+				var id = self.idExpression.evaluate(self._variables);
 
-	this.currentDocument = this.makeDocument(this.currentGroupsKeysIndex);
-	return true;
-};
+				if(undefined === id) id = null;
+
+				var groupKey = JSON.stringify(id),
+					group = self.groups[JSON.stringify(id)];
+
+				if(!group) {
+					self.groupsKeys.push(groupKey);
+					group = [];
+					self.groups[groupKey] = group;
+					// Add the accumulators
+					for(var afi = 0; afi<self.accumulatorFactories.length; afi++) {
+						group.push(self.accumulatorFactories[afi]());
+					}
+				}
+				//NOTE: Skipped memory usage stuff for case when group already existed
+
+				if(numAccumulators !== group.length) {
+					throw new Error('Group must have one of each accumulator');
+				}
+
+				//NOTE: passing the input to each accumulator
+				for(var gi=0; gi<group.length; gi++) {
+					group[gi].process(self.expressions[gi].evaluate(self._variables /*, doingMerge*/));
+				}
+
+				// We are done with the ROOT document so release it.
+				self._variables.clearRoot();
 
-proto.eof = function eof() {
-	if (!this.populated) this.populate();
-	return this.currentGroupsKeysIndex === this.groupsKeys.length;
+				//NOTE: Skipped the part about sorted files
+
+				return cb();
+			});
+		},
+		function(err) {
+			if(err) return callback(err);
+
+			self.populated = true;
+
+			return callback();
+		}
+	);
 };
 
-proto.getCurrent = function getCurrent() {
-	if (!this.populated) this.populate();
-	return this.currentDocument;
+/**
+ * Get the type of something. Handles objects specially to return their true type; i.e. their constructor
+ *
+ * @method populate
+ * @param obj {Object} The object to get the type of
+ * @return {String} The type of the object as a string
+ * @async
+ **/
+proto._getTypeStr = function _getTypeStr(obj) {
+	var typeofStr = typeof obj,
+		typeStr = (typeofStr == "object" && obj !== null) ? obj.constructor.name : typeofStr;
+	return typeStr;
 };
 
+/**
+ * Get the dependencies of the group
+ *
+ * @method getDependencies
+ * @param deps {Object} The
+ * @return {DocumentSource.getDepsReturn} An enum value specifying that these dependencies are exhaustive
+ * @async
+ **/
 proto.getDependencies = function getDependencies(deps) {
 	var self = this;
 	// add _id
@@ -205,67 +370,53 @@ proto.getDependencies = function getDependencies(deps) {
 	return DocumentSource.GetDepsReturn.EXHAUSTIVE;
 };
 
+/**
+ * Called internally only. Adds an accumulator for each matching group.
+ *
+ * @method addAccumulator
+ * @param fieldName {String} The name of the field where the accumulated value will be placed
+ * @param accumulatorFactory {Accumulator} The constructor for creating accumulators
+ * @param epxression {Expression} The expression to be evaluated on incoming documents before they are accumulated
+ **/
 proto.addAccumulator = function addAccumulator(fieldName, accumulatorFactory, expression) {
 	this.fieldNames.push(fieldName);
 	this.accumulatorFactories.push(accumulatorFactory);
 	this.expressions.push(expression);
 };
 
-proto.populate = function populate() {
-	for (var hasNext = !this.source.eof(); hasNext; hasNext = this.source.advance()) {
-		var group,
-			currentDocument = this.source.getCurrent(),
-			_id = this.idExpression.evaluate(currentDocument);
-
-		if (undefined === _id) _id = null;
+/**
+ * Makes a document with the given id and accumulators
+ *
+ * @method makeDocument
+ * @param fieldName {String} The name of the field where the accumulated value will be placed
+ * @param accums {Array} An array of accumulators
+ * @param epxression {Expression} The expression to be evaluated on incoming documents before they are accumulated
+ **/
+proto.makeDocument = function makeDocument(id, accums /*,mergeableOutput*/) {
+	var out = {};
 
-		var idHash = JSON.stringify(_id); //TODO: USE A REAL HASH.  I didn't have time to take collision into account.
+	/* add the _id field */
+	out._id = id;
 
-		if (idHash in this.groups) {
-			group = this.groups[idHash];
+	/* add the rest of the fields */
+	this.fieldNames.forEach(function(fieldName, i) {
+		var val = accums[i].getValue(/*mergeableOutput*/);
+		if(!val) {
+			out[fieldName] = null;
 		} else {
-			this.groups[idHash] = group = [];
-			this.groupsKeys[this.currentGroupsKeysIndex] = idHash;
-			this.originalGroupsKeys[this.currentGroupsKeysIndex] = (_id && typeof _id === 'object') ? Document.clone(_id) : _id;
-			++this.currentGroupsKeysIndex;
-			for (var ai = 0; ai < this.accumulatorFactories.length; ++ai) {
-				var accumulator = new this.accumulatorFactories[ai]();
-				accumulator.addOperand(this.expressions[ai]);
-				group.push(accumulator);
-			}
-		}
-
-
-		// tickle all the accumulators for the group we found
-		for (var gi = 0; gi < group.length; ++gi) {
-			group[gi].evaluate(currentDocument);
+			out[fieldName] = val;
 		}
+	});
 
-	}
-
-	this.currentGroupsKeysIndex = 0; // Start the group
-	if (this.groupsKeys.length > 0) {
-		this.currentDocument = this.makeDocument(this.currentGroupsKeysIndex);
-	}
-	this.populated = true;
-
+	return out;
 };
 
-proto.makeDocument = function makeDocument(groupKeyIndex) {
-	var groupKey = this.groupsKeys[groupKeyIndex],
-		originalGroupKey = this.originalGroupsKeys[groupKeyIndex],
-		group = this.groups[groupKey],
-		doc = {};
-
-	doc[Document.ID_PROPERTY_NAME] = originalGroupKey;
-
-	for (var i = 0; i < this.fieldNames.length; ++i) {
-		var fieldName = this.fieldNames[i],
-			item = group[i];
-		if (item !== "null" && item !== undefined) {
-			doc[fieldName] = item.getValue();
-		}
-	}
-
-	return doc;
+/**
+ * Sets the id expression for the group
+ *
+ * @method setIdExpression
+ * @param epxression {Expression} The expression to set
+ **/
+proto.setIdExpression = function setIdExpression(expression) {
+	this.idExpression = expression;
 };

lib/pipeline/documentSources/LimitDocumentSource.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var DocumentSource = require('./DocumentSource');
+
 /**
  * A document source limiter
  * @class LimitDocumentSource
@@ -35,60 +37,23 @@ proto.coalesce = function coalesce(nextSource) {
 
 	// if it's not another $limit, we can't coalesce
 	if (!nextLimit) return false;
-	
+
 	// we need to limit by the minimum of the two limits
 	if (nextLimit.limit < this.limit) this.limit = nextLimit.limit;
 
 	return true;
 };
 
-/**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	return this.source.eof() || this.count >= this.limit;
-};
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
 
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	return this.source.getCurrent();
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-	++this.count;
-	if (this.count >= this.limit) {
-		return false;
+	if (++this.count > this.limit) {
+		this.source.dispose();
+		callback(null, DocumentSource.EOF);
+		return DocumentSource.EOF;
 	}
-	this.current = this.source.getCurrent();
-	return this.source.advance();
-};
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	builder.$limit = this.limit;
+	return this.source.getNext(callback);
 };
 
 /**
@@ -106,3 +71,21 @@ klass.createFromJson = function createFromJson(jsonElement, ctx) {
 
 	return nextLimit;
 };
+
+proto.getLimit = function getLimit(newLimit) {
+	return this.limit;
+};
+
+proto.setLimit = function setLimit(newLimit) {
+	this.limit = newLimit;
+};
+
+proto.getDependencies = function(deps) {
+	return DocumentSource.GetDepsReturn.SEE_NEXT;
+};
+
+proto.serialize = function(explain) {
+	var out = {};
+	out[this.getSourceName()] = this.limit;
+	return out;
+};

lib/pipeline/documentSources/MatchDocumentSource.js

@@ -1,8 +1,10 @@
 "use strict";
-var sift = require("sift");	//TODO: DEVIATION FROM MONGO: this was a temporary hack to get this done quickly but it is too inconsistent to keep; need a real port of MatchDocumentSource
+var async = require("async"),
+	matcher = require("../matcher/Matcher2.js"),
+	DocumentSource = require("./DocumentSource");
 
 /**
- * A match document source built off of FilterBaseDocumentSource
+ * A match document source built off of DocumentSource
  *
  * NOTE: THIS IS A DEVIATION FROM THE MONGO IMPLEMENTATION.
  * TODO: internally uses `sift` to fake it, which has bugs, so we need to reimplement this by porting the MongoDB implementation
@@ -19,8 +21,8 @@ var MatchDocumentSource = module.exports = function MatchDocumentSource(query, c
 	if (!query) throw new Error("arg `query` is required");
 	base.call(this, ctx);
 	this.query = query; // save the query, so we can check it for deps later. THIS IS A DEVIATION FROM THE MONGO IMPLEMENTATION
-	this.matcher = sift(query);
-}, klass = MatchDocumentSource, base = require('./FilterBaseDocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	this.matcher = new matcher(query);
+}, klass = MatchDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 klass.matchName = "$match";
 
@@ -28,56 +30,56 @@ proto.getSourceName = function getSourceName(){
 	return klass.matchName;
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	builder[this.getSourceName()] = this.matcher.query;
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		next,
+		test = function test(doc) {
+			return self.matcher.matches(doc);
+		},
+		makeReturn = function makeReturn(doc) {
+			if(doc !== DocumentSource.EOF && test(doc)) { // Passes the match criteria
+				return doc;
+			} else if(doc === DocumentSource.EOF){ // Got EOF
+				return doc;
+			}
+			return undefined; // Didn't match, but not EOF
+		};
+	async.doUntil(
+		function(cb) {
+			self.source.getNext(function(err, doc) {
+				if(err) return callback(err);
+				if (makeReturn(doc)) {
+					next = doc;
+				}
+				return cb();
+			});
+		},
+		function() {
+			var foundDoc = (next === DocumentSource.EOF || next !== undefined);
+			return foundDoc; //keep going until doc is found
+		},
+		function(err) {
+			return callback(err, next);
+		}
+	);
+	return next;
 };
 
-/**
- *  Test the given document against the predicate and report if it should be accepted or not.
- * @param {object} document the document to test
- * @returns {bool} true if the document matches the filter, false otherwise
- **/
-proto.accept = function accept(document) {
-	/**
-	* The matcher only takes BSON documents, so we have to make one.
-	*
-	* LATER
-	* We could optimize this by making a document with only the
-	* fields referenced by the Matcher.  We could do this by looking inside
-	* the Matcher's BSON before it is created, and recording those.  The
-	* easiest implementation might be to hold onto an ExpressionDocument
-	* in here, and give that pDocument to create the created subset of
-	* fields, and then convert that instead.
-	**/
-	return this.matcher.test(document);
+proto.coalesce = function coalesce(nextSource) {
+	if (!(nextSource instanceof MatchDocumentSource))
+		return false;
+
+	this.matcher = new matcher({"$and": [this.getQuery(), nextSource.getQuery()]});
+
+	return true;
 };
 
-/**
- * Create a JSONObj suitable for Matcher construction.
- *
- * This is used after filter analysis has moved as many filters to
- * as early a point as possible in the document processing pipeline.
- * See db/Matcher.h and the associated wiki documentation for the
- * format.  This conversion is used to move back to the low-level
- * find() Cursor mechanism.
- *
- * @param builder the builder to write to
- **/
-proto.toMatcherJson = function toMatcherJson(builder) {
-	var q = this.matcher.query;
-	for(var k in q){
-		builder[k] = q[k];
-	}
+proto.serialize = function(explain) {
+	var out = {};
+	out[this.getSourceName()] = this.getQuery();
+	return out;
 };
 
 klass.uassertNoDisallowedClauses = function uassertNoDisallowedClauses(query) {
@@ -100,3 +102,188 @@ klass.createFromJson = function createFromJson(jsonElement, ctx) {
 	var matcher = new MatchDocumentSource(jsonElement, ctx);
 	return matcher;
 };
+
+proto.getQuery = function getQuery() {
+	return this.matcher._pattern;
+};
+
+/** Returns the portion of the match that can safely be promoted to before a $redact.
+ * If this returns an empty BSONObj, no part of this match may safely be promoted.
+ *
+ * To be safe to promote, removing a field from a document to be matched must not cause
+ * that document to be accepted when it would otherwise be rejected. As an example,
+ * {name: {$ne: "bob smith"}} accepts documents without a name field, which means that
+ * running this filter before a redact that would remove the name field would leak
+ * information. On the other hand, {age: {$gt:5}} is ok because it doesn't accept documents
+ * that have had their age field removed.
+ */
+proto.redactSafePortion = function redactSafePortion() {
+	var self = this;
+
+	// This block contains the functions that make up the implementation of
+	// DocumentSourceMatch::redactSafePortion(). They will only be called after
+	// the Match expression has been successfully parsed so they can assume that
+	// input is well formed.
+
+	var isAllDigits = function(n) {
+		return !isNaN(n);
+	};
+
+	var isFieldnameRedactSafe = function isFieldnameRedactSafe(field) {
+		var dotPos = field.indexOf('.');
+		if (dotPos === -1)
+			return !isAllDigits(field);
+
+		var part = field.slice(0, dotPos),
+			rest = field.slice(dotPos+1, field.length);
+
+		return !isAllDigits(part) && isFieldnameRedactSafe(rest);
+	};
+
+	// Returns the redact-safe portion of an "inner" match expression. This is the layer like
+	// {$gt: 5} which does not include the field name. Returns an empty document if none of the
+	// expression can safely be promoted in front of a $redact.
+	var redactSavePortionDollarOps = function redactSafePortionDollarOps(expr) {
+		var output = {},
+			elem,i,j,k;
+
+		var keys = Object.keys(expr);
+		for (i = 0; i < keys.length; i++) {
+			var field = keys[i],
+				value = expr[field];
+
+			if (field[0] !== '$')
+				continue;
+
+			// Ripped the case apart and did not implement this painful thing:
+			// https://github.com/mongodb/mongo/blob/r2.5.4/src/mongo/db/jsobj.cpp#L286
+			// Somebody should be taken to task for that work of art.
+			if (field === '$type' || field === '$regex' || field === '$options' || field === '$mod') {
+				output[field] = value;
+			} else if (field === '$lte' || field === '$gte' || field === '$lt' || field === '$gt') {
+				if (isTypeRedactSafeInComparison(field))
+					output[field] = value;
+			} else if (field === '$in') {
+				// TODO: value/elem/field/etc may be mixed up and wrong here
+				var allOk = true;
+				for (j = 0; j < Object.keys(value).length; j++) {
+					elem = Object.keys(value)[j];
+					if (!isTypeRedactSafeInComparison(value[elem])) {
+						allOk = false;
+						break;
+					}
+				}
+				if (allOk) {
+					output[field] = value;
+				}
+				break;
+			} else if (field === '$all') {
+				// TODO: value/elem/field/etc may be mixed up and wrong here
+				var matches = [];
+				for (j = 0; j < value.length; j++) {
+					elem = Object.keys(value)[j];
+					if (isTypeRedactSafeInComparison(value[elem]))
+						matches.push(value[elem]);
+				}
+				if (matches.length)
+					output[field] = matches;
+
+			} else if (field === '$elemMatch') {
+				var subIn = value,
+					subOut;
+
+				if (subIn[0] === '$')
+					subOut = redactSafePortionDollarOps(subIn);
+				else
+					subOut = redactSafePortionTopLevel(subIn);
+
+				if (subOut && Object.keys(subOut).length)
+					output[field] = subOut;
+
+				break;
+			} else {
+				// never allowed:
+				// equality, maxDist, near, ne, opSize, nin, exists, within, geoIntersects
+				continue;
+			}
+		}
+
+		return output;
+	};
+
+	var isTypeRedactSafeInComparison = function isTypeRedactSafeInComparison(type) {
+		if (type instanceof Array || (type instanceof Object && type.constructor === Object) || type === null || type === undefined)
+			return false;
+		return true;
+	};
+
+	// Returns the redact-safe portion of an "outer" match expression. This is the layer like
+	// {fieldName: {...}} which does include the field name. Returns an empty document if none of
+	// the expression can safely be promoted in front of a $redact.
+	var redactSafePortionTopLevel = function(topQuery) {
+		var output = {},
+			okClauses = [],
+			keys = topQuery ? Object.keys(topQuery) : [],
+			j, elm, clause;
+
+		for (var i = 0; i < keys.length; i++) {
+			var field = keys[i],
+				value = topQuery[field];
+
+			if (field.length && field[0] === '$') {
+				if (field === '$or') {
+					okClauses = [];
+					for (j = 0; j < Object.keys(value).length; j++) {
+						elm = value[Object.keys(value)[j]];
+						clause = redactSafePortionTopLevel(elm);
+
+						if (!clause || Object.keys(clause).length === 0) {
+							okClauses = [];
+							break;
+						}
+
+						okClauses.push(clause);
+					}
+
+					if (okClauses && okClauses.length) {
+						output.$or = okClauses;
+					}
+				} else if (field === '$and') {
+					okClauses = [];
+					for (j = 0; j < Object.keys(value).length; j++) {
+						elm = value[Object.keys(value)[j]];
+						clause = redactSafePortionTopLevel(elm);
+
+						if (clause && Object.keys(clause).length)
+							okClauses.push(clause);
+					}
+
+					if (okClauses.length)
+						output.$and = okClauses;
+				}
+
+				continue;
+			}
+
+			if (!isFieldnameRedactSafe(field))
+					continue;
+
+			if (value instanceof Array || !value) {
+				continue;
+			} else if (value instanceof Object && value.constructor === Object) {
+				// subobjects (not regex etc)
+				var sub = redactSavePortionDollarOps(value);
+				if (sub && Object.keys(sub).length)
+					output[field] = sub;
+
+				break;
+			} else {
+				output[field] = value;
+			}
+		}
+
+		return output;
+	};
+
+	return redactSafePortionTopLevel(this.getQuery());
+};

lib/pipeline/documentSources/OutDocumentSource.js

@@ -0,0 +1,53 @@
+"use strict";
+
+var DocumentSource = require('./DocumentSource');
+
+/**
+ * @class OutDocumentSource
+ * @namespace mungedb-aggregate.pipeline.documentSources
+ * @module mungedb-aggregate
+ * @constructor
+ * @param [ctx] {ExpressionContext}
+ **/
+var OutDocumentSource = module.exports = function OutDocumentSource(outputNs, ctx){
+	if (arguments.length > 2) throw new Error("up to two arg expected");
+	base.call(this, ctx);
+	// defaults
+	this._done = false;
+	this._outputNs = outputNs;
+	this._collectionName = "";
+}, klass = OutDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+klass.outName = "$out";
+
+proto.getSourceName = function() {
+	return klass.outName;
+};
+
+proto.getNext = DocumentSource.GET_NEXT_PASS_THROUGH;
+
+proto.serialize = function(explain) {
+	var doc = {},
+		srcNm = this.getSourceName();
+	doc[srcNm] = this._collectionName;
+	return doc;
+};
+
+proto.getOutputNs = function() {
+	return this._outputNs;
+};
+
+klass.createFromJson = function(jsonElement, ctx) {
+	if (typeof jsonElement !== 'string')
+		throw new Error('code 16990; $out only supports a string argument, not ' + typeof jsonElement);
+
+	var out = new OutDocumentSource(null, ctx); // TODO: outputNs
+	out._collectionName = jsonElement;
+
+	return out;
+};
+
+proto.getDependencies = function(deps) {
+	deps.needWholeDocument = true;
+	return DocumentSource.GetDepsReturn.EXHAUSTIVE;
+};

lib/pipeline/documentSources/ProjectDocumentSource.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var DocumentSource = require('./DocumentSource');
+
 /**
  * A base class for filter document sources
  * @class ProjectDocumentSource
@@ -16,9 +18,12 @@ var ProjectDocumentSource = module.exports = function ProjectDocumentSource(ctx)
 }, klass = ProjectDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
-var Expression = require('../expressions/Expression');
-var ObjectExpression = require('../expressions/ObjectExpression');
-var Value = require('../Value');
+var Expression = require('../expressions/Expression'),
+	ObjectExpression = require('../expressions/ObjectExpression'),
+	Value = require('../Value'),
+	Variables = require('../expressions/Variables'),
+	VariablesIdGenerator = require('../expressions/VariablesIdGenerator'),
+	VariablesParseState = require('../expressions/VariablesParseState');
 
 klass.projectName = "$project";
 
@@ -30,6 +35,39 @@ proto.getSourceName = function getSourceName() {
 	return klass.projectName;
 };
 
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		out;
+
+	this.source.getNext(function(err, input) {
+		if (err)
+			return callback(null, err);
+
+		if (input === DocumentSource.EOF) {
+			out = input;
+			return callback(null, DocumentSource.EOF);
+		}
+
+		/* create the result document */
+		out = {};
+
+		/**
+		 * Use the ExpressionObject to create the base result.
+		 *
+		 * If we're excluding fields at the top level, leave out the _id if
+		 * it is found, because we took care of it above.
+		 **/
+		self._variables.setRoot(input);
+		self.OE.addToDocument(out, input, self._variables);
+		self._variables.clearRoot();
+
+		return callback(null, out);
+	});
+	return out;
+};
+
 /**
  * Returns the object that was used to construct the ProjectDocumentSource
  * @return {object} the object that was used to construct the ProjectDocumentSource
@@ -38,33 +76,10 @@ proto.getRaw = function getRaw() {
 	return this._raw;
 };
 
-/**
- * Calls base document source eof()
- * @return {bool} The result of base.source.eof()
- **/
-proto.eof = function eof() {
-	return this.source.eof();
-};
-
-/**
- * Calls base document source advance()
- * @return {bool} The result of base.source.advance()
- **/
-proto.advance = function advance() {
-	return this.source.advance();
-};
-
-
-/**
- * Builds a new document(object) that represents this base document
- * @return {object} A document that represents this base document
- **/
-proto.getCurrent = function getCurrent() {
-	var inDocument = this.source.getCurrent();
-	if (!inDocument) throw new Error('inDocument must be an object');
-	var resultDocument = {};
-	this.OE.addToDocument(resultDocument, inDocument, /*root=*/inDocument);
-	return resultDocument;
+proto.serialize = function serialize(explain) {
+	var out = {};
+	out[this.getSourceName()] = this.OE.serialize(explain);
+	return out;
 };
 
 /**
@@ -99,18 +114,24 @@ proto.sourceToJson = function sourceToJson(builder, explain) {
  **/
 klass.createFromJson = function(jsonElement, expCtx) {
 	if (!(jsonElement instanceof Object) || jsonElement.constructor !== Object) throw new Error('Error 15969. Specification must be an object but was ' + typeof jsonElement);
+
 	var objectContext = new Expression.ObjectCtx({
 		isDocumentOk: true,
 		isTopLevel: true,
 		isInclusionOk: true
 	});
-	var project = new ProjectDocumentSource(expCtx);
+
+	var project = new ProjectDocumentSource(expCtx),
+		idGenerator = new VariablesIdGenerator(),
+		vps = new VariablesParseState(idGenerator);
+
 	project._raw = jsonElement;
-	var parsed = Expression.parseObject(jsonElement, objectContext);
+	var parsed = Expression.parseObject(jsonElement, objectContext, vps);
 	var exprObj = parsed;
 	if (!exprObj instanceof ObjectExpression) throw new Error("16402, parseObject() returned wrong type of Expression");
 	if (!exprObj.getFieldCount()) throw new Error("16403, $projection requires at least one output field");
 	project.OE = exprObj;
+	project._variables = new Variables(idGenerator.getIdCount());
 	return project;
 };
 

lib/pipeline/documentSources/RedactDocumentSource.js

@@ -0,0 +1,156 @@
+"use strict";
+
+var async = require("async"),
+	DocumentSource = require("./DocumentSource"),
+	Expression = require("../expressions/Expression"),
+	Variables = require("../expressions/Variables"),
+	VariablesIdGenerator = require("../expressions/VariablesIdGenerator"),
+	VariablesParseState = require("../expressions/VariablesParseState");
+
+/**
+ * A document source skipper
+ * @class RedactDocumentSource
+ * @namespace mungedb-aggregate.pipeline.documentSources
+ * @module mungedb-aggregate
+ * @constructor
+ * @param [ctx] {ExpressionContext}
+ **/
+var RedactDocumentSource = module.exports = function RedactDocumentSource(ctx, expression){
+	if (arguments.length > 2) throw new Error("up to two args expected");
+	base.call(this, ctx);
+	this._expression = expression;
+	this._variables = new Variables();
+	this._currentId = null;
+}, klass = RedactDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+klass.redactName = "$redact";
+proto.getSourceName = function getSourceName(){
+	return klass.redactName;
+};
+
+var DESCEND_VAL = 'descend',
+	PRUNE_VAL = 'prune',
+	KEEP_VAL = 'keep';
+
+proto.getNext = function getNext(callback) {
+	var self = this,
+		doc;
+	async.whilst(
+		function() {
+			return doc !== DocumentSource.EOF;
+		},
+		function(cb) {
+			self.source.getNext(function(err, input) {
+				doc = input;
+				if (input === DocumentSource.EOF)
+					return cb();
+				self._variables.setRoot(input);
+				self._variables.setValue(self._currentId, input);
+				var result = self.redactObject();
+				if (result !== DocumentSource.EOF)
+					return cb(result); //Using the err argument to pass the result document; this lets us break out without having EOF
+				return cb();
+			});
+		},
+		function(doc) {
+			if (doc)
+				return callback(null, doc);
+			return callback(null, DocumentSource.EOF);
+		}
+	);
+	return doc;
+};
+
+proto.redactValue = function redactValue(input) {
+	// reorder to make JS happy with types
+	if (input instanceof Array) {
+		var newArr,
+			arr = input;
+		for (var i = 0; i < arr.length; i++) {
+			if ((arr[i] instanceof Object && arr[i].constructor === Object) || arr[i] instanceof Array) {
+				var toAdd = this.redactValue(arr[i]);
+				if (toAdd)
+					newArr.push(arr[i]);
+			} else {
+				newArr.push(arr[i]);
+			}
+		}
+		return newArr;
+	} else if (input instanceof Object && input.constructor === Object) {
+		this._variables.setValue(this._currentId, input);
+		var result = this.redactObject();
+		if (result !== DocumentSource.EOF)
+			return result;
+		return null;
+	} else {
+		return input;
+	}
+};
+
+/**
+ * Redacts the current object
+ **/
+proto.redactObject = function redactObject() {
+	var expressionResult = this._expression.evaluate(this._variables);
+
+	if (expressionResult === KEEP_VAL) {
+		return this._variables.getDocument(this._currentId);
+	} else if (expressionResult === PRUNE_VAL) {
+		return DocumentSource.EOF;
+	} else if (expressionResult === DESCEND_VAL) {
+		var input = this._variables.getDocument(this._currentId);
+		var out;
+
+		var inputKeys = Object.keys(input);
+		for (var i = 0; i < inputKeys.length; i++) {
+			var field = inputKeys[i],
+				value = input[field];
+
+			var val = this.redactValue(value);
+			if (val)
+				out[field] = val;
+		}
+
+		return out;
+	} else {
+		throw new Error("17053 $redact's expression should not return anything aside from the variables $$KEEP, $$DESCEND, and $$PRUNE, but returned " + expressionResult);
+	}
+};
+
+proto.optimize = function optimize() {
+	this._expression = this._expression.optimize();
+};
+
+proto.serialize = function serialize(explain) {
+	var doc = {};
+	doc[this.getSourceName()] = this._expression.serialize(explain);
+	return doc;
+};
+
+/**
+ * Creates a new RedactDocumentSource with the input number as the skip
+ *
+ * @param {Number} JsonElement this thing is *called* Json, but it expects a number
+ **/
+klass.createFromJson = function createFromJson(jsonElement, ctx) {
+	if (!jsonElement)
+		throw new Error("#createFromJson requires at least one argument");
+
+	var idGenerator = new VariablesIdGenerator(),
+		vps = new VariablesParseState(idGenerator),
+		currentId = vps.defineVariable("CURRENT"),
+		descendId = vps.defineVariable("DESCEND"),
+		pruneId = vps.defineVariable("PRUNE"),
+		keepId = vps.defineVariable("KEEP");
+
+	var expression = new Expression.parseOperand(jsonElement, vps),
+		source = new RedactDocumentSource(ctx, expression);
+
+	source._currentId = currentId;
+	source._variables = new Variables(idGenerator.getIdCount());
+	source._variables.setValue(descendId, DESCEND_VAL);
+	source._variables.setValue(pruneId, PRUNE_VAL);
+	source._variables.setValue(keepId, KEEP_VAL);
+
+	return source;
+};

lib/pipeline/documentSources/SkipDocumentSource.js

@@ -1,5 +1,8 @@
 "use strict";
 
+var async = require('async'),
+	DocumentSource = require('./DocumentSource');
+
 /**
  * A document source skipper
  * @class SkipDocumentSource
@@ -36,72 +39,44 @@ proto.coalesce = function coalesce(nextSource) {
 	return true;
 };
 
-proto.skipper = function skipper() {
-	if (this.count === 0) {
-		while (!this.source.eof() && this.count++ < this.skip) {
-			this.source.advance();
-		}
-	}
-
-	if (this.source.eof()) {
-		this.current = null;
-		return;
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		next;
+
+	if (this.count < this.skip) {
+
+		async.doWhilst(
+			function(cb) {
+				self.source.getNext(function(err, val) {
+					if(err) return cb(err);
+					self.count++;
+					next = val;
+					return cb();
+				});
+			},
+			function() {
+				return self.count < self.skip || next === DocumentSource.EOF;
+			},
+			function(err) {
+				if (err)
+					return callback(err);
+			}
+		);
 	}
 
-	this.current = this.source.getCurrent();
-};
-
-
-/**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	this.skipper();
-	return this.source.eof();
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	this.skipper();
-	return this.source.getCurrent();
+	return this.source.getNext(callback);
 };
 
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-	if (this.eof()) {
-		this.current = null;
-		return false;
-	}
-
-	this.current = this.source.getCurrent();
-	return this.source.advance();
+proto.serialize = function serialize(explain) {
+	var out = {};
+	out[this.getSourceName()] = this.skip;
+	return out;
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	builder.$skip = this.skip;
+proto.getSkip = function getSkip() {
+	return this.skip;
 };
 
 /**

lib/pipeline/documentSources/SortDocumentSource.js

@@ -1,5 +1,9 @@
 "use strict";
 
+var async = require("async"),
+	DocumentSource = require("./DocumentSource"),
+	LimitDocumentSource = require("./LimitDocumentSource");
+
 /**
  * A document source sorter
  *
@@ -21,7 +25,6 @@ var SortDocumentSource = module.exports = function SortDocumentSource(ctx){
 	* boolean indicates that this has been done
 	**/
 	this.populated = false;
-	this.current = null;
 	this.docIterator = null; // a number tracking our position in the documents array
 	this.documents = []; // an array of documents
 
@@ -47,6 +50,16 @@ klass.GetDepsReturn = {
 	SEE_NEXT: "SEE_NEXT" // Add the next Source's deps to the set
 };
 
+proto.dispose = function dispose() {
+	this.docIterator = 0;
+	this.documents = [];
+	this.source.dispose();
+};
+
+proto.getLimit = function getLimit() {
+	return this.limitSrc ? this.limitSrc.getLimit() : -1;
+};
+
 proto.getDependencies = function getDependencies(deps) {
 	for(var i = 0; i < this.vSortKey.length; ++i) {
 		this.vSortKey[i].addDependencies(deps);
@@ -54,64 +67,78 @@ proto.getDependencies = function getDependencies(deps) {
 	return klass.GetDepsReturn.SEE_NEXT;
 };
 
-/**
- * Is the source at EOF?
- * @method	eof
- * @return {bool} return if we have hit the end of input
- **/
-proto.eof = function eof() {
-	if (!this.populated) this.populate();
-	return (this.docIterator == this.documents.length);
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	if (!this.populated) this.populate();
-	return this.current;
+proto.coalesce = function coalesce(nextSource) {
+	if (!this.limitSrc) {
+		if (nextSource instanceof LimitDocumentSource) {
+			this.limitSrc = nextSource;
+			return nextSource;
+		}
+		return false;
+	} else {
+		return this.limitSrc.coalesce(nextSource);
+	}
 };
 
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-
-	if (!this.populated) this.populate();
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		out;
+	async.series(
+		[
+			function(next) {
+				if (!self.populated)
+					self.populate(function(err) {
+						return next(err);
+					});
+				else
+					next();
+			},
+			function(next) {
+				if (self.docIterator >= self.documents.length) {
+					out = DocumentSource.EOF;
+					return next(null, DocumentSource.EOF);
+				}
+
+				var output = self.documents[self.docIterator++];
+				if (!output || output === DocumentSource.EOF) {
+					out = DocumentSource.EOF;
+					return next(null, DocumentSource.EOF);
+				}
+
+				out = output;
+				return next(null, output);
+			}
+		],
+		function(err, results) {
+			return callback(err, out);
+		}
+	);
 
-	if (this.docIterator == this.documents.length) throw new Error("This should never happen");
-	++this.docIterator;
+	return out;
+};
 
-	if (this.docIterator == this.documents.length) {
-		this.current = null;
-		return false;
+proto.serializeToArray = function serializeToArray(array, explain) {
+	var doc = {};
+	if (explain) {
+		doc.sortKey = this.serializeSortKey();
+		doc.mergePresorted = this._mergePresorted;
+		doc.limit = this.limitSrc ? this.limitSrc.getLimit() : undefined;
+		array.push(doc);
+	} else {
+		var inner = this.serializeSortKey();
+		if (this._mergePresorted)
+			inner.$mergePresorted = true;
+		doc[this.getSourceName()] = inner;
+		array.push(doc);
+
+		if (this.limitSrc)
+			this.limitSrc.serializeToArray(array);
 	}
-	this.current = this.documents[this.docIterator];
-	return true;
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
-**/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	var insides = {};
-	this.sortKeyToJson(insides, false);
-	builder[this.getSourceName()] = insides;
+proto.serialize = function serialize(explain) {
+	throw new Error("should call serializeToArray instead");
 };
 
 /**
@@ -129,34 +156,45 @@ proto.addKey = function addKey(fieldPath, ascending) {
 	if (ascending === true || ascending === false) {
 		this.vAscending.push(ascending);
 	} else {
+		// This doesn't appear to be an error in real mongo?
 		throw new Error("ascending must be true or false");
 	}
 };
 
-proto.populate = function populate() {
+proto.populate = function populate(callback) {
 	/* make sure we've got a sort key */
-	if (this.vSortKey.length === null) throw new Error("This should never happen");
+	if (!this.vSortKey.length) throw new Error("no sort key for " + this.getSourceName());
 
-	/* pull everything from the underlying source */
-	for(var hasNext = !this.source.eof(); hasNext; hasNext = this.source.advance()) {
-		var doc = this.source.getCurrent();
-		this.documents.push(doc);
-	}
-
-	this.vSortKeyFPEs = this.vSortKey.map(function(aSortKey){
-		return new FieldPathExpression(aSortKey.getFieldPath(false));
-	});
+	// Skipping stuff about mergeCursors and commandShards
 
+	/* pull everything from the underlying source */
+	var self = this,
+		next;
+	async.doWhilst(
+		function (cb) {
+			self.source.getNext(function(err, doc) {
+				next = doc;
+
+				// Don't add EOF; it doesn't sort well.
+				if (doc !== DocumentSource.EOF)
+					self.documents.push(doc);
+				return cb();
+			});
+		},
+		function() {
+			return next !== DocumentSource.EOF;
+		},
+		function(err) {
 	/* sort the list */
-	this.documents.sort(SortDocumentSource.prototype.compare.bind(this));
+			self.documents.sort(SortDocumentSource.prototype.compare.bind(self));
 
 	/* start the sort iterator */
-	this.docIterator = 0;
+			self.docIterator = 0;
 
-	if (this.docIterator < this.documents.length) {
-		this.current = this.documents[this.docIterator];
+			self.populated = true;
+			return callback();
 	}
-	this.populated = true;
+	);
 };
 
 /**
@@ -177,7 +215,7 @@ proto.compare = function compare(pL,pR) {
 	var n = this.vSortKey.length;
 	for(var i = 0; i < n; ++i) {
 		/* evaluate the sort keys */
-		var pathExpr = this.vSortKeyFPEs[i];
+		var pathExpr = new FieldPathExpression(this.vSortKey[i].getFieldPath(false));
 		var left = pathExpr.evaluate(pL), right = pathExpr.evaluate(pR);
 
 		/*
@@ -201,19 +239,16 @@ proto.compare = function compare(pL,pR) {
 
 /**
 * Write out an object whose contents are the sort key.
-*
-* @param {Object} builder initialized object builder.
-* @param {bool} fieldPrefix specify whether or not to include the field
 **/
-proto.sortKeyToJson = function sortKeyToJson(builder, usePrefix) {
-	// add the key fields
+proto.serializeSortKey = function sortKeyToJson() {
+	var keyObj = {};
+
 	var n = this.vSortKey.length;
-	for(var i = 0; i < n; ++i) {
-		// create the "field name"
-		var ss = this.vSortKey[i].getFieldPath(usePrefix); // renamed write to get
-		// push a named integer based on the sort order
-		builder[ss] = this.vAscending[i] > 0 ? 1 : -1;
+	for (var i = 0; i < n; i++) {
+		var fieldPath = this.vSortKey[i].getFieldPath();
+		keyObj[fieldPath] = this.vAscending[i] ? 1 : -1;
 	}
+	return keyObj;
 };
 
 /**

lib/pipeline/documentSources/UnwindDocumentSource.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var async = require("async");
+
 /**
  * A document source unwinder
  * @class UnwindDocumentSource
@@ -66,6 +68,21 @@ klass.Unwinder = (function(){
 		this._unwindArrayIteratorCurrent = this._unwindArrayIterator.splice(0,1)[0];
 	};
 
+	/**
+	 * getNext
+	 *
+	 * This is just wrapping the old functions because they are somewhat different
+	 * than the original mongo implementation, but should get updated to follow the current API.
+	 **/
+	proto.getNext = function getNext() {
+		if (this.eof())
+			return DocumentSource.EOF;
+
+		var output = this.getCurrent();
+		this.advance();
+		return output;
+	};
+
 	/**
 	 * eof
 	 * @returns	{Boolean}	true if done unwinding the last document passed to resetDocument().
@@ -166,40 +183,6 @@ klass.Unwinder = (function(){
 	return klass;
 })();
 
-/**
- * Lazily construct the _unwinder and initialize the iterator state of this DocumentSource.
- * To be called by all members that depend on the iterator state.
- **/
-proto.lazyInit = function lazyInit(){
-	if (!this._unwinder) {
-		if (!this._unwindPath){
-			throw new Error("unwind path does not exist!");
-		}
-		this._unwinder = new klass.Unwinder(this._unwindPath);
-		if (!this.source.eof()) {
-			// Set up the first source document for unwinding.
-			this._unwinder.resetDocument(this.source.getCurrent());
-		}
-		this.mayAdvanceSource();
-	}
-};
-
-/**
- * If the _unwinder is exhausted and the source may be advanced, advance the source and
- * reset the _unwinder's source document.
- **/
-proto.mayAdvanceSource = function mayAdvanceSource(){
-	while(this._unwinder.eof()) {
-		// The _unwinder is exhausted.
-
-		if (this.source.eof()) return; // The source is exhausted.
-		if (!this.source.advance()) return; // The source is exhausted.
-
-		// Reset the _unwinder with source's next document.
-		this._unwinder.resetDocument(this.source.getCurrent());
-	}
-};
-
 /**
  * Specify the field to unwind.
 **/
@@ -209,6 +192,7 @@ proto.unwindPath = function unwindPath(fieldPath){
 
 	// Record the unwind path.
 	this._unwindPath = new FieldPath(fieldPath);
+	this._unwinder = new klass.Unwinder(this._unwindPath);
 };
 
 klass.unwindName = "$unwind";
@@ -231,55 +215,47 @@ proto.getDependencies = function getDependencies(deps) {
 	return DocumentSource.GetDepsReturn.SEE_NEXT;
 };
 
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		out = this._unwinder.getNext(),
+		exhausted = false;
+
+	async.until(
+		function() {
+			if(out === DocumentSource.EOF && exhausted) return true;	// Really is EOF, not just an empty unwinder
+			else if(out !== DocumentSource.EOF) return true; // Return whatever we got that wasn't EOF
+			return false;
+		},
+		function(cb) {
+			self.source.getNext(function(err, doc) {
+				if(err) return cb(err);
+				out = doc;
+				if(out === DocumentSource.EOF) { // Our source is out of documents, we're done
+					exhausted = true;
+					return cb();
+				} else {
+					self._unwinder.resetDocument(doc);
+					out = self._unwinder.getNext();
+					return cb();
+				}
+			});
+		},
+		function(err) {
+			if(err) return callback(err);
+			return callback(null, out);
+		}
+	);
 
-/**
- * Is the source at EOF?
- * @method	eof
-**/
-proto.eof = function eof() {
-	this.lazyInit();
-	return this._unwinder.eof();
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
-**/
-proto.getCurrent = function getCurrent() {
-	this.lazyInit();
-	return this._unwinder.getCurrent();
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
-**/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-	this.lazyInit();
-	this._unwinder.advance();
-	this.mayAdvanceSource();
-	return !this._unwinder.eof();
+	return out; //For sync mode
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
-**/
-proto.sourceToJson = function sourceToJson(builder, explain) {
+proto.serialize = function serialize(explain) {
 	if (!this._unwindPath) throw new Error("unwind path does not exist!");
-	builder[this.getSourceName()] = this._unwindPath.getPath(true);
+	var doc = {};
+	doc[this.getSourceName()] = this._unwindPath.getPath(true);
+	return doc;
 };
 
 /**

lib/pipeline/documentSources/index.js

@@ -2,12 +2,14 @@
 module.exports = {
 	CursorDocumentSource: require("./CursorDocumentSource.js"),
 	DocumentSource: require("./DocumentSource.js"),
-	FilterBaseDocumentSource: require("./FilterBaseDocumentSource.js"),
 	GroupDocumentSource: require("./GroupDocumentSource.js"),
 	LimitDocumentSource: require("./LimitDocumentSource.js"),
 	MatchDocumentSource: require("./MatchDocumentSource.js"),
 	ProjectDocumentSource: require("./ProjectDocumentSource.js"),
 	SkipDocumentSource: require("./SkipDocumentSource.js"),
 	SortDocumentSource: require("./SortDocumentSource.js"),
-	UnwindDocumentSource: require("./UnwindDocumentSource.js")
+	UnwindDocumentSource: require("./UnwindDocumentSource.js"),
+	RedactDocumentSource: require("./RedactDocumentSource.js"),
+	GeoNearDocumentSource: require("./GeoNearDocumentSource.js"),
+	OutDocumentSource: require("./OutDocumentSource.js")
 };

lib/pipeline/expressions/AddExpression.js

@@ -1,7 +1,7 @@
 "use strict";
 
-/** 
- * Create an expression that finds the sum of n operands. 
+/**
+ * Create an expression that finds the sum of n operands.
  * @class AddExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
@@ -13,25 +13,22 @@ var AddExpression = module.exports = function AddExpression(){
 }, klass = AddExpression, NaryExpression = require("./NaryExpression"), base = NaryExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$add";
 };
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
-};
-
 /**
  * Takes an array of one or more numbers and adds them together, returning the sum.
  * @method @evaluate
  **/
-proto.evaluate = function evaluate(doc) {
+proto.evaluateInternal = function evaluateInternal(vars) {
 	var total = 0;
 	for (var i = 0, n = this.operands.length; i < n; ++i) {
-		var value = this.operands[i].evaluate(doc);
+		var value = this.operands[i].evaluateInternal(vars);
 		if (value instanceof Date) throw new Error("$add does not support dates; code 16415");
 		if (typeof value == "string") throw new Error("$add does not support strings; code 16416");
 		total += Value.coerceToDouble(value);
@@ -39,3 +36,7 @@ proto.evaluate = function evaluate(doc) {
 	if (typeof total != "number") throw new Error("$add resulted in a non-numeric type; code 16417");
 	return total;
 };
+
+
+/** Register Expression */
+Expression.registerExpression("$add",base.parse(AddExpression));

lib/pipeline/expressions/AllElementsTrueExpression.js

@@ -0,0 +1,49 @@
+"use strict";
+
+/**
+ * Create an expression that returns true exists in all elements.
+ * @class AllElementsTrueExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var AllElementsTrueExpression = module.exports = function AllElementsTrueExpression() {
+	this.nargs = 1;
+	base.call(this);
+},
+	klass = AllElementsTrueExpression,
+	NaryExpression = require("./NaryExpression"),
+	base = NaryExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	CoerceToBoolExpression = require("./CoerceToBoolExpression"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$allElementsTrue";
+};
+
+/**
+ * Takes an array of one or more numbers and returns true if all.
+ * @method @evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var value = evaluateInternal(vars);
+	if (!vars instanceof Array) throw new Error("$allElementsTrue requires an array");
+
+	for (var i = 0, n = this.operands.length; i < n; ++i) {
+		var checkValue = this.operands[i].evaluateInternal(vars);
+		if (!checkValue.coerceToBool()) return false;
+	}
+	return true;
+};
+
+/** Register Expression */
+Expression.registerExpression("$allElementsTrue", base.parse(AllElementsTrueExpression));

lib/pipeline/expressions/AndExpression.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * Create an expression that finds the conjunction of n operands. The
  * conjunction uses short-circuit logic; the expressions are evaluated in the
  * order they were added to the conjunction, and the evaluation stops and
@@ -11,33 +11,36 @@
  * @module mungedb-aggregate
  * @constructor
  **/
-var AndExpression = module.exports = function AndExpression(){
+var AndExpression = module.exports = function AndExpression() {
 	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = AndExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = AndExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
 var Value = require("../Value"),
 	ConstantExpression = require("./ConstantExpression"),
-	CoerceToBoolExpression = require("./CoerceToBoolExpression");
+	CoerceToBoolExpression = require("./CoerceToBoolExpression"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$and";
 };
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
-};
-
 /**
  * Takes an array one or more values and returns true if all of the values in the array are true. Otherwise $and returns false.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc) {
+proto.evaluateInternal = function evaluateInternal(vars) {
 	for (var i = 0, n = this.operands.length; i < n; ++i) {
-		var value = this.operands[i].evaluate(doc);
-		if (!Value.coerceToBool(value)) return false;
+		var value = this.operands[i].evaluateInternal(vars);
+		if (!Value.coerceToBool()) return false;
 	}
 	return true;
 };
@@ -71,4 +74,7 @@ proto.optimize = function optimize() {
 	return expr;
 };
 
-//TODO:	proto.toMatcherBson
+/** Register Expression */
+Expression.registerExpression("$and", base.parse(AndExpression));
+
+//TODO: proto.toMatcherBson

lib/pipeline/expressions/AnyElementTrueExpression.js

@@ -0,0 +1,41 @@
+"use strict";
+
+/**
+ * Create an expression that returns true exists in any element.
+ * @class AnyElementTrueExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var AnyElementTrueExpression = module.exports = function AnyElementTrueExpression(){
+	this.nargs = (1);
+	base.call(this);
+}, klass = AnyElementTrueExpression, NaryExpression = require("./NaryExpression"), base = NaryExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName(){
+	return "$anyElementTrue";
+};
+
+/**
+ * Takes an array of one or more numbers and returns true if any.
+ * @method @evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	if (!vars instanceof Array) throw new Error("$anyElementTrue requires an array");
+
+	var total = 0;
+	for (var i = 0, n = vars.length; i < n; ++i) {
+		var value = vars[i].evaluateInternal([i]);
+		if ( value.coerceToBool() )
+			return true;
+	}
+	return false;
+};
+
+/** Register Expression */
+Expression.registerExpression("$anyElementTrue",base.parse(AnyElementTrueExpression));

lib/pipeline/expressions/CoerceToBoolExpression.js

@@ -1,7 +1,7 @@
 "use strict";
 
-/** 
- * internal expression for coercing things to booleans 
+/**
+ * internal expression for coercing things to booleans
  * @class CoerceToBoolExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
@@ -17,16 +17,11 @@ var CoerceToBoolExpression = module.exports = function CoerceToBoolExpression(ex
 var Value = require("../Value"),
 	AndExpression = require("./AndExpression"),
 	OrExpression = require("./OrExpression"),
-	NotExpression = require("./NotExpression");
-
-// PROTOTYPE MEMBERS
-proto.evaluate = function evaluate(doc){
-	var result = this.expression.evaluate(doc);
-	return Value.coerceToBool(result);
-};
+	NotExpression = require("./NotExpression"),
+	Expression = require("./Expression");
 
 proto.optimize = function optimize() {
-	this.expression = this.expression.optimize();	// optimize the operand
+	this.expression = this.expression.optimize();   // optimize the operand
 
 	// if the operand already produces a boolean, then we don't need this
 	// LATER - Expression to support a "typeof" query?
@@ -43,10 +38,25 @@ proto.addDependencies = function addDependencies(deps, path) {
 	return this.expression.addDependencies(deps);
 };
 
+// PROTOTYPE MEMBERS
+proto.evaluateInternal = function evaluateInternal(vars){
+	var result = this.expression.evaluateInternal(vars);
+	return Value.coerceToBool(result);
+};
+
+proto.serialize = function serialize(explain) {
+	if ( explain ) {
+		return {$coerceToBool:[this.expression.toJSON()]};
+	}
+	else {
+		return {$and:[this.expression.toJSON()]};
+	}
+};
+
 proto.toJSON = function toJSON() {
 	// Serializing as an $and expression which will become a CoerceToBool
 	return {$and:[this.expression.toJSON()]};
 };
 
-//TODO:	proto.addToBsonObj   --- may be required for $project to work
-//TODO:	proto.addToBsonArray
+//TODO: proto.addToBsonObj   --- may be required for $project to work
+//TODO: proto.addToBsonArray

lib/pipeline/expressions/CompareExpression.js

@@ -1,108 +1,113 @@
 "use strict";
 
 /**
- * Generic comparison expression that gets used for $eq, $ne, $lt, $lte, $gt, $gte, and $cmp. 
+ * Generic comparison expression that gets used for $eq, $ne, $lt, $lte, $gt, $gte, and $cmp.
  * @class CompareExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var CompareExpression = module.exports = function CompareExpression(cmpOp) {
-	if (arguments.length !== 1) throw new Error("args expected: cmpOp");
-	this.cmpOp = cmpOp;
-	base.call(this);
-}, klass = CompareExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+    this.nargs = 2;
+    this.cmpOp = cmpOp;
+    base.call(this);
+}, klass = CompareExpression,
+    base = require("./NaryExpression"),
+    proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+    });
 
 // DEPENDENCIES
-var Value = require("../Value"),
-	Expression = require("./Expression"),
-	ConstantExpression = require("./ConstantExpression"),
-	FieldPathExpression = require("./FieldPathExpression"),
-	FieldRangeExpression = require("./FieldRangeExpression");
+var Value = require("../Value");
+var Expression = require("./Expression");
+var ConstantExpression = require("./ConstantExpression");
+var FieldPathExpression = require("./FieldPathExpression");
+var FieldRangeExpression = require("./FieldRangeExpression");
+var NaryExpression = require("./NaryExpression");
 
 // NESTED CLASSES
 /**
  * Lookup table for truth value returns
  *
- * @param truthValues	truth value for -1, 0, 1
- * @param reverse		reverse comparison operator
- * @param name			string name
+ * @param truthValues   truth value for -1, 0, 1
+ * @param reverse               reverse comparison operator
+ * @param name                  string name
  **/
-var CmpLookup = (function(){	// emulating a struct
+var CmpLookup = (function() { // emulating a struct
 	// CONSTRUCTOR
 	var klass = function CmpLookup(truthValues, reverse, name) {
-		if(arguments.length !== 3) throw new Error("args expected: truthValues, reverse, name");
+		if (arguments.length !== 3) throw new Error("args expected: truthValues, reverse, name");
 		this.truthValues = truthValues;
 		this.reverse = reverse;
 		this.name = name;
-	}, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	}, base = Object,
+		proto = klass.prototype = Object.create(base.prototype, {
+			constructor: {
+				value: klass
+			}
+		});
 	return klass;
 })();
 
+// verify we need this below
 // PRIVATE STATIC MEMBERS
 /**
  * a table of cmp type lookups to truth values
  * @private
  **/
-var cmpLookupMap = [	//NOTE: converted from this Array to a Dict/Object below using CmpLookup#name as the key
-	//              -1      0      1      reverse             name     (taking advantage of the fact that our 'enums' are strings below)
-	new CmpLookup([false, true, false], Expression.CmpOp.EQ, Expression.CmpOp.EQ),
-	new CmpLookup([true, false, true], Expression.CmpOp.NE, Expression.CmpOp.NE),
-	new CmpLookup([false, false, true], Expression.CmpOp.LT, Expression.CmpOp.GT),
-	new CmpLookup([false, true, true], Expression.CmpOp.LTE, Expression.CmpOp.GTE),
-	new CmpLookup([true, false, false], Expression.CmpOp.GT, Expression.CmpOp.LT),
-	new CmpLookup([true, true, false], Expression.CmpOp.GTE, Expression.CmpOp.LTE),
-	new CmpLookup([false, false, false], Expression.CmpOp.CMP, Expression.CmpOp.CMP)
-].reduce(function(r,o){r[o.name]=o;return r;},{});
+var cmpLookupMap = [ //NOTE: converted from this Array to a Dict/Object below using CmpLookup#name as the key
+    //              -1      0      1      reverse             name     (taking advantage of the fact that our 'enums' are strings below)
+    new CmpLookup([false, true, false], CompareExpression.EQ, CompareExpression.EQ),
+    new CmpLookup([true, false, true], CompareExpression.NE, CompareExpression.NE),
+    new CmpLookup([false, false, true], CompareExpression.LT, CompareExpression.GT),
+    new CmpLookup([false, true, true], CompareExpression.LTE, CompareExpression.GTE),
+    new CmpLookup([true, false, false], CompareExpression.GT, CompareExpression.LT),
+    new CmpLookup([true, true, false], CompareExpression.GTE, CompareExpression.LTE),
+    new CmpLookup([false, false, false], CompareExpression.CMP, CompareExpression.CMP)
+].reduce(function(r, o) {
+	r[o.name] = o;
+	return r;
+}, {});
 
 
-// PROTOTYPE MEMBERS
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
+klass.parse = function parse(bsonExpr, vps, op) {
+    var expr = new CompareExpression(op);
+    var args = NaryExpression.parseArguments(bsonExpr, vps);
+    expr.validateArguments(args);
+    expr.vpOperand = args;
+    return expr;
 
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc),
-		right = this.operands[1].evaluate(doc),
-		cmp = Expression.signum(Value.compare(left, right));
-	if (this.cmpOp == Expression.CmpOp.CMP) return cmp;
-	return cmpLookupMap[this.cmpOp].truthValues[cmp + 1] || false;
 };
 
-proto.optimize = function optimize(){
-	var expr = base.prototype.optimize.call(this); // first optimize the comparison operands
-	if (!(expr instanceof CompareExpression)) return expr; // if no longer a comparison, there's nothing more we can do.
-
-	// check to see if optimizing comparison operator is supported	// CMP and NE cannot use ExpressionFieldRange which is what this optimization uses
-	var newOp = this.cmpOp;
-	if (newOp == Expression.CmpOp.CMP || newOp == Expression.CmpOp.NE) return expr;
-
-	// There's one localized optimization we recognize:  a comparison between a field and a constant.  If we recognize that pattern, replace it with an ExpressionFieldRange.
-	// When looking for this pattern, note that the operands could appear in any order.  If we need to reverse the sense of the comparison to put it into the required canonical form, do so.
-	var leftExpr = this.operands[0],
-		rightExpr = this.operands[1];
-	var fieldPathExpr, constantExpr;
-	if (leftExpr instanceof FieldPathExpression) {
-		fieldPathExpr = leftExpr;
-		if (!(rightExpr instanceof ConstantExpression)) return expr; // there's nothing more we can do
-		constantExpr = rightExpr;
-	} else {
-		// if the first operand wasn't a path, see if it's a constant
-		if (!(leftExpr instanceof ConstantExpression)) return expr; // there's nothing more we can do
-		constantExpr = leftExpr;
-
-		// the left operand was a constant; see if the right is a path
-		if (!(rightExpr instanceof FieldPathExpression)) return expr; // there's nothing more we can do
-		fieldPathExpr = rightExpr;
-
-		// these were not in canonical order, so reverse the sense
-		newOp = cmpLookupMap[newOp].reverse;
-	}
-	return new FieldRangeExpression(fieldPathExpr, newOp, constantExpr.getValue());
+// PROTOTYPE MEMBERS
+proto.evaluateInternal = function evaluateInternal(vars) {
+	//debugger;
+    var left = this.operands[0].evaluateInternal(vars),
+        right = this.operands[1].evaluateInternal(vars),
+        cmp = Expression.signum(Value.compare(left, right));
+    if (this.cmpOp == Expression.CmpOp.CMP) return cmp;
+    return cmpLookupMap[this.cmpOp].truthValues[cmp + 1] || false;
 };
 
-proto.getOpName = function getOpName(){
+klass.EQ = "$eq";
+klass.NE = "$ne";
+klass.GT = "$gt";
+klass.GTE = "$gte";
+klass.LT = "$lt";
+klass.LTE = "$lte";
+klass.CMP = "$cmp";
+
+proto.getOpName = function getOpName() {
 	return this.cmpOp;
 };
+
+/** Register Expression */
+Expression.registerExpression("$eq", klass.parse);
+Expression.registerExpression("$ne", klass.parse);
+Expression.registerExpression("$gt", klass.parse);
+Expression.registerExpression("$gte", klass.parse);
+Expression.registerExpression("$lt", klass.parse);
+Expression.registerExpression("$lte", klass.parse);
+Expression.registerExpression("$cmp", klass.parse);

lib/pipeline/expressions/ConcatExpression.js

@@ -1,6 +1,8 @@
 "use strict";
 
-/** 
+var Expression = require("./Expression");
+
+/**
  * Creates an expression that concatenates a set of string operands.
  * @class ConcatExpression
  * @namespace mungedb-aggregate.pipeline.expressions
@@ -14,27 +16,28 @@ var ConcatExpression = module.exports = function ConcatExpression(){
 
 // DEPENDENCIES
 var Value = require("../Value");
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$concat";
 };
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
-};
-
 /**
  * Concats a string of values together.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc) {
-	var result = "";
-	for (var i = 0, n = this.operands.length; i < n; ++i) {
-		var val = this.operands[i].evaluate(doc);
-		if (val === null) return null; // if any operand is null, return null for all
-		if (typeof(val) != "string") throw new Error("$concat only supports strings, not " + typeof(val) + "; code 16702");
-		result = result + Value.coerceToString(val);
+proto.evaluateInternal = function evaluateInternal(vars) {
+    var n = this.operands.length;
+
+    return this.operands.map(function(x) {
+	var y = x.evaluateInternal(vars);
+	if(typeof(y) !== "string") {
+	    throw new Error("$concat only supports strings - 16702");
 	}
-	return result;
+	return y;
+    }).join("");
 };
+
+
+Expression.registerExpression("$concat", base.parse(ConcatExpression));

lib/pipeline/expressions/CondExpression.js

@@ -1,37 +1,78 @@
 "use strict";
 
 /**
- * $cond expression;  @see evaluate 
- * @class AndExpression
+ * $cond expression;  @see evaluate
+ * @class CondExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var CondExpression = module.exports = function CondExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
-	base.call(this);
-}, klass = CondExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+var CondExpression = module.exports = function CondExpression(vars) {
+    this.nargs = 3;
+    this.pCond = this.evaluateInternal(vars);
+    this.idx = this.pCond.coerceToBool() ? 1 : 2;
+
+    if (arguments.length !== 3) throw new Error("three args expected");
+    base.call(this);
+}, klass = CondExpression,
+    base = require("./NaryExpression"),
+    proto = klass.prototype = Object.create(base.prototype, {
+	constructor: {
+	    value: klass
+	}
+    });
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+    Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
-	return "$cond";
+proto.getOpName = function getOpName() {
+    return "$cond";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(3);
-	base.prototype.addOperand.call(this, expr);
+klass.parse = function parse(expr, vps) {
+    this.checkArgLimit(3);
+
+    // if not an object, return;
+    if (typeof(expr) !== Object)
+		return Expression.parse(expr, vps);
+
+    // verify
+    if (Expression.parseOperand(expr) !== "$cond")
+		throw new Error("Invalid expression");
+
+    var ret = new CondExpression();
+
+    var ex = Expression.parseObject(expr);
+    var args = Expression.parseOperand(expr, vps);
+    if (args[0] !== "if")
+		throw new Error("Missing 'if' parameter to $cond");
+    if (args[1] !== "then")
+		throw new Error("Missing 'then' parameter to $cond");
+    if (args[2] !== "else")
+		throw new Error("Missing 'else' parameter to $cond");
+
+
+    return ret;
 };
 
-/** 
- * Use the $cond operator with the following syntax:  { $cond: [ <boolean-expression>, <true-case>, <false-case> ] } 
+/**
+ * Use the $cond operator with the following syntax:  { $cond: [ <boolean-expression>, <true-case>, <false-case> ] }
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(3);
-	var pCond = this.operands[0].evaluate(doc),
-		idx = Value.coerceToBool(pCond) ? 1 : 2;
-	return this.operands[idx].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+		var pCond1 = this.operands[0].evaluateInternal(vars);
+
+		this.idx = 0;
+		if (pCond1.coerceToBool()) {
+			this.idx = 1;
+		} else {
+			this.idx = 2;
+		}
+
+		return this.operands[this.idx].evaluateInternal(vars);
 };
+
+/** Register Expression */
+Expression.registerExpression("$cond", klass.parse);

lib/pipeline/expressions/ConstantExpression.js

@@ -1,18 +1,23 @@
 "use strict";
 
-/** 
- * Internal expression for constant values 
+/**
+ * Internal expression for constant values
  * @class ConstantExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var ConstantExpression = module.exports = function ConstantExpression(value){
-	if (arguments.length !== 1) throw new Error("args expected: value");
-	this.value = value;	//TODO: actually make read-only in terms of JS?
-	base.call(this);
+    if (arguments.length !== 1) throw new Error("args expected: value");
+    this.value = value; //TODO: actually make read-only in terms of JS?
+    base.call(this);
 }, klass = ConstantExpression, base = require("./Expression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
+
+// DEPENDENCIES
+var Value = require("../Value"),
+    Expression = require("./Expression");
+
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$const";
@@ -23,19 +28,23 @@ proto.getOpName = function getOpName(){
  * @method getValue
  * @returns the value
  **/
-proto.getValue = function getValue(){	//TODO: convert this to an instance field rather than a property
-	return this.value;
+proto.getValue = function getValue(){   //TODO: convert this to an instance field rather than a property
+    return this.value;
 };
 
 proto.addDependencies = function addDependencies(deps, path) {
 	// nothing to do
 };
 
+klass.parse = function parse(expr, vps){
+    return new ConstantExpression(expr);
+};
+
 /**
  * Get the constant value represented by this Expression.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars){
 	return this.value;
 };
 
@@ -43,9 +52,13 @@ proto.optimize = function optimize() {
 	return this; // nothing to do
 };
 
-proto.toJSON = function(isExpressionRequired){
-	return isExpressionRequired ? {$const: this.value} : this.value;
+proto.serialize = function(rawValue){
+	return rawValue ? {$const: this.value} : this.value;
 };
 
-//TODO:	proto.addToBsonObj   --- may be required for $project to work -- my hope is that we can implement toJSON methods all around and use that instead
-//TODO:	proto.addToBsonArray
+//TODO: proto.addToBsonObj   --- may be required for $project to work -- my hope is that we can implement toJSON methods all around and use that instead
+//TODO: proto.addToBsonArray
+
+/** Register Expression */
+Expression.registerExpression("$const",klass.parse(ConstantExpression));
+Expression.registerExpression("$literal", klass.parse(ConstantExpression)); // alias

lib/pipeline/expressions/DayOfMonthExpression.js

@@ -7,27 +7,34 @@
  * @module mungedb-aggregate
  * @constructor
  **/
-var DayOfMonthExpression = module.exports = function DayOfMonthExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
-	base.call(this);
-}, klass = DayOfMonthExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+var DayOfMonthExpression = module.exports = function DayOfMonthExpression() {
+    this.nargs = 1;
+    base.call(this);
+}, klass = DayOfMonthExpression,
+    base = require("./NaryExpression"),
+    proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+    });
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
-// PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
-	return "$dayOfMonth";
-};
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+    return "$dayOfMonth";
 };
 
 /**
  * Takes a date and returns the day of the month as a number between 1 and 31.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
-	return date.getUTCDate();
+proto.evaluateInternal = function evaluateInternal(vars) {
+    var date = this.operands[0].evaluateInternal(vars);
+    return date.getUTCDate();
 };
+
+/** Register Expression */
+Expression.registerExpression("$dayOfMonth", base.parse(DayOfMonthExpression));

lib/pipeline/expressions/DayOfWeekExpression.js

@@ -8,26 +8,26 @@
  * @constructor
  **/
 var DayOfWeekExpression = module.exports = function DayOfWeekExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+	this.nargs = 1;
 	base.call(this);
 }, klass = DayOfWeekExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
+// DEPENDENCIES
+var Expression = require("./Expression");
+
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$dayOfWeek";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the day of the week as a number between 1 (Sunday) and 7 (Saturday.)
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars){
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCDay()+1;
 };
+
+/** Register Expression */
+Expression.registerExpression("$dayOfWeek",base.parse(DayOfWeekExpression));

lib/pipeline/expressions/DayOfYearExpression.js

@@ -8,34 +8,34 @@
  * @constructor
  **/
 var DayOfYearExpression = module.exports = function DayOfYearExpression(){
-	if(arguments.length !== 0) throw new Error("zero args expected");
+	this.nargs = 1;
 	base.call(this);
 }, klass = DayOfYearExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
+// DEPENDENCIES
+var Expression = require("./Expression");
+
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
-	return "$dayOfYear";
-};
-
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
+    return "$dayOfYear";
 };
 
 /**
  * Takes a date and returns the day of the year as a number between 1 and 366.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars){
 	//NOTE: the below silliness is to deal with the leap year scenario when we should be returning 366
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
-	return klass.getDateDayOfYear(date);
+    var date = this.operands[0].evaluateInternal(vars);
+    return klass.getDateDayOfYear(date);
 };
 
 // STATIC METHODS
 klass.getDateDayOfYear = function getDateDayOfYear(d){
-	var y11 = new Date(d.getUTCFullYear(), 0, 1),	// same year, first month, first day; time omitted
-		ymd = new Date(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()+1);	// same y,m,d; time omitted, add 1 because days start at 1
-	return Math.ceil((ymd - y11) / 86400000);	//NOTE: 86400000 ms is 1 day
+    var y11 = new Date(d.getUTCFullYear(), 0, 1),       // same year, first month, first day; time omitted
+	ymd = new Date(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()+1);  // same y,m,d; time omitted, add 1 because days start at 1
+    return Math.ceil((ymd - y11) / 86400000);   //NOTE: 86400000 ms is 1 day
 };
+
+/** Register Expression */
+Expression.registerExpression("$dayOfYear",base.parse(DayOfYearExpression));

lib/pipeline/expressions/DivideExpression.js

@@ -1,42 +1,40 @@
 "use strict";
 
-/** 
- * A $divide pipeline expression. 
- * @see evaluate 
+/**
+ * A $divide pipeline expression.
+ * @see evaluateInternal
  * @class DivideExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var DivideExpression = module.exports = function DivideExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
-	base.call(this);
+    this.nargs = 2;
+    base.call(this);
 }, klass = DivideExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){	//TODO: try to move this to a static and/or instance field instead of a getter function
+proto.getOpName = function getOpName(){ //TODO: try to move this to a static and/or instance field instead of a getter function
 	return "$divide";
 };
 
-proto.addOperand = function addOperand(expr){
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes an array that contains a pair of numbers and returns the value of the first number divided by the second number.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc),
-		right = this.operands[1].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var left = this.operands[0].evaluateInternal(vars),
+		right = this.operands[1].evaluateInternal(vars);
 	if (!(left instanceof Date) && (!right instanceof Date)) throw new Error("$divide does not support dates; code 16373");
 	right = Value.coerceToDouble(right);
 	if (right === 0) return undefined;
 	left = Value.coerceToDouble(left);
 	return left / right;
 };
+
+/** Register Expression */
+Expression.registerExpression("$divide",base.parse(DivideExpression));

lib/pipeline/expressions/Expression.js

@@ -5,20 +5,32 @@
  *
  * NOTE: An object expression can take any of the following forms:
  *
- *	f0: {f1: ..., f2: ..., f3: ...}
- *	f0: {$operator:[operand1, operand2, ...]}
+ *      f0: {f1: ..., f2: ..., f3: ...}
+ *      f0: {$operator:[operand1, operand2, ...]}
  *
  * @class Expression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var Expression = module.exports = function Expression(){
+
+
+var Expression = module.exports = function Expression() {
 	if (arguments.length !== 0) throw new Error("zero args expected");
-}, klass = Expression, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = Expression,
+    base = Object,
+    proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+    });
+
+
+
+function fn(){
+	return;
+}
 
-// DEPENDENCIES
-var Document = require("../Document");
 
 // NESTED CLASSES
 /**
@@ -26,7 +38,7 @@ var Document = require("../Document");
  * @static
  * @property ObjectCtx
  **/
-var ObjectCtx = Expression.ObjectCtx = (function(){
+var ObjectCtx = Expression.ObjectCtx = (function() {
 	// CONSTRUCTOR
 	/**
 	 * Utility class for parseObject() below. isDocumentOk indicates that it is OK to use a Document in the current context.
@@ -38,237 +50,134 @@ var ObjectCtx = Expression.ObjectCtx = (function(){
 	 * @module mungedb-aggregate
 	 * @constructor
 	 * @param opts
-	 *	@param [opts.isDocumentOk]	{Boolean}
-	 *	@param [opts.isTopLevel]	{Boolean}
-	 *	@param [opts.isInclusionOk]	{Boolean}
+	 *      @param [opts.isDocumentOk]      {Boolean}
+	 *      @param [opts.isTopLevel]        {Boolean}
+	 *      @param [opts.isInclusionOk]     {Boolean}
 	 **/
-	var klass = function ObjectCtx(opts /*= {isDocumentOk:..., isTopLevel:..., isInclusionOk:...}*/){
-		if(!(opts instanceof Object && opts.constructor == Object)) throw new Error("opts is required and must be an Object containing named args");
+	var klass = function ObjectCtx(opts /*= {isDocumentOk:..., isTopLevel:..., isInclusionOk:...}*/ ) {
+		if (!(opts instanceof Object && opts.constructor == Object)) throw new Error("opts is required and must be an Object containing named args");
 		for (var k in opts) { // assign all given opts to self so long as they were part of klass.prototype as undefined properties
 			if (opts.hasOwnProperty(k) && proto.hasOwnProperty(k) && proto[k] === undefined) this[k] = opts[k];
 		}
-	}, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
-
-	// PROTOTYPE MEMBERS
-	proto.isDocumentOk =
-	proto.isTopLevel =
-	proto.isInclusionOk = undefined;
-
-	return klass;
-})();
-
-/**
- * Reference to the `mungedb-aggregate.pipeline.expressions.Expression.OpDesc` class
- * @static
- * @property OpDesc
- **/
-var OpDesc = Expression.OpDesc = (function(){
-	// CONSTRUCTOR
-	/**
-	 * Decribes how and when to create an Op instance
-	 *
-	 * @class OpDesc
-	 * @namespace mungedb-aggregate.pipeline.expressions.Expression
-	 * @module mungedb-aggregate
-	 * @constructor
-	 * @param name
-	 * @param factory
-	 * @param flags
-	 * @param argCount
-	 **/
-	var klass = function OpDesc(name, factory, flags, argCount){
-		var firstArg = arguments[0];
-		if (firstArg instanceof Object && firstArg.constructor == Object) { //TODO: using this?
-			var opts = firstArg;
-			for (var k in opts) { // assign all given opts to self so long as they were part of klass.prototype as undefined properties
-				if (opts.hasOwnProperty(k) && proto.hasOwnProperty(k) && proto[k] === undefined) this[k] = opts[k];
+	}, base = Object,
+		proto = klass.prototype = Object.create(base.prototype, {
+			constructor: {
+				value: klass
 			}
-		} else {
-			this.name = name;
-			this.factory = factory;
-			this.flags = flags || 0;
-			this.argCount = argCount || 0;
-		}
-	}, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
-
-	// STATIC MEMBERS
-	klass.FIXED_COUNT = 1;
-	klass.OBJECT_ARG = 2;
+		});
 
 	// PROTOTYPE MEMBERS
-	proto.name =
-	proto.factory =
-	proto.flags =
-	proto.argCount = undefined;
-
-	/**
-	 * internal `OpDesc#name` comparer
-	 * @method cmp
-	 * @param that the other `OpDesc` instance
-	 **/
-	proto.cmp = function cmp(that) {
-		return this.name < that.name ? -1 : this.name > that.name ? 1 : 0;
-	};
+	proto.isDocumentOk =
+		proto.isTopLevel =
+		proto.isInclusionOk = undefined;
 
 	return klass;
 })();
-// END OF NESTED CLASSES
-/**
- * @class Expression
- * @namespace mungedb-aggregate.pipeline.expressions
- * @module mungedb-aggregate
- **/
-
-var kinds = {
-	UNKNOWN: "UNKNOWN",
-	OPERATOR: "OPERATOR",
-	NOT_OPERATOR: "NOT_OPERATOR"
-};
 
-
-// STATIC MEMBERS
-/**
- * Enumeration of comparison operators.  These are shared between a few expression implementations, so they are factored out here.
- *
- * @static
- * @property CmpOp
- **/
-klass.CmpOp = {
-	EQ: "$eq",		// return true for a == b, false otherwise
-	NE: "$ne",		// return true for a != b, false otherwise
-	GT: "$gt",		// return true for a > b, false otherwise
-	GTE: "$gte",	// return true for a >= b, false otherwise
-	LT: "$lt",		// return true for a < b, false otherwise
-	LTE: "$lte",	// return true for a <= b, false otherwise
-	CMP: "$cmp"		// return -1, 0, 1 for a < b, a == b, a > b
-};
-
-// DEPENDENCIES (later in this file as compared to others to ensure that the required statics are setup first)
-var FieldPathExpression = require("./FieldPathExpression"),
-	ObjectExpression = require("./ObjectExpression"),
-	ConstantExpression = require("./ConstantExpression"),
-	CompareExpression = require("./CompareExpression");
-
-// DEFERRED DEPENDENCIES
-/**
- * Expressions, as exposed to users
- *
- * @static
- * @property opMap
- **/
-setTimeout(function(){ // Even though `opMap` is deferred, force it to load early rather than later to prevent even *more* potential silliness
-	Object.defineProperty(klass, "opMap", {value:klass.opMap});
-}, 0);
-Object.defineProperty(klass, "opMap", {	//NOTE: deferred requires using a getter to allow circular requires (to maintain the ported API)
-	configurable: true,
-	get: function getOpMapOnce() {
-		return Object.defineProperty(klass, "opMap", {
-			value: [	//NOTE: rather than OpTable because it gets converted to a dict via OpDesc#name in the Array#reduce() below
-				new OpDesc("$add", require("./AddExpression"), 0),
-				new OpDesc("$and", require("./AndExpression"), 0),
-				new OpDesc("$cmp", CompareExpression.bind(null, Expression.CmpOp.CMP), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$concat", require("./ConcatExpression"), 0),
-				new OpDesc("$cond", require("./CondExpression"), OpDesc.FIXED_COUNT, 3),
-		//		$const handled specially in parseExpression
-				new OpDesc("$dayOfMonth", require("./DayOfMonthExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$dayOfWeek", require("./DayOfWeekExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$dayOfYear", require("./DayOfYearExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$divide", require("./DivideExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$eq", CompareExpression.bind(null, Expression.CmpOp.EQ), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$gt", CompareExpression.bind(null, Expression.CmpOp.GT), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$gte", CompareExpression.bind(null, Expression.CmpOp.GTE), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$hour", require("./HourExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$ifNull", require("./IfNullExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$lt", CompareExpression.bind(null, Expression.CmpOp.LT), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$lte", CompareExpression.bind(null, Expression.CmpOp.LTE), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$minute", require("./MinuteExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$mod", require("./ModExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$month", require("./MonthExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$multiply", require("./MultiplyExpression"), 0),
-				new OpDesc("$ne", CompareExpression.bind(null, Expression.CmpOp.NE), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$not", require("./NotExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$or", require("./OrExpression"), 0),
-				new OpDesc("$second", require("./SecondExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$strcasecmp", require("./StrcasecmpExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$substr", require("./SubstrExpression"), OpDesc.FIXED_COUNT, 3),
-				new OpDesc("$subtract", require("./SubtractExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$toLower", require("./ToLowerExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$toUpper", require("./ToUpperExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$week", require("./WeekExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$year", require("./YearExpression"), OpDesc.FIXED_COUNT, 1)
-			].reduce(function(r,o){r[o.name]=o; return r;}, {})
-		}).opMap;
+proto.removeFieldPrefix = function removeFieldPrefix(prefixedField) {
+	if (prefixedField.indexOf("\0") !== -1) {
+		// field path must not contain embedded null characters - 16419
 	}
-});
-
+	if (prefixedField[0] !== '$') {
+		// "field path references must be prefixed with a '$'"
+	}
+	return prefixedField.slice(1);
+};
+var KIND_UNKNOWN = 0,
+	KIND_NOTOPERATOR = 1,
+	KIND_OPERATOR = 2;
 /**
  * Parse an Object.  The object could represent a functional expression or a Document expression.
  *
  * An object expression can take any of the following forms:
  *
- *	f0: {f1: ..., f2: ..., f3: ...}
- *	f0: {$operator:[operand1, operand2, ...]}
+ *      f0: {f1: ..., f2: ..., f3: ...}
+ *      f0: {$operator:[operand1, operand2, ...]}
  *
  * @static
  * @method parseObject
- * @param obj	the element representing the object
- * @param ctx	a MiniCtx representing the options above
+ * @param obj   the element representing the object
+ * @param ctx   a MiniCtx representing the options above
  * @returns the parsed Expression
  **/
-klass.parseObject = function parseObject(obj, ctx){
-	if(!(ctx instanceof ObjectCtx)) throw new Error("ctx must be ObjectCtx");
-	var kind = kinds.UNKNOWN,
-		expr, // the result
-		exprObj; // the alt result
-	if (obj === undefined) return new ObjectExpression();
+klass.parseObject = function parseObject(obj, ctx, vps) {
+	if (!(ctx instanceof ObjectCtx)) throw new Error("ctx must be ObjectCtx");
+	var kind = KIND_UNKNOWN,
+		pExpression, // the result
+		pExpressionObject; // the alt result
+	if (obj === undefined || obj == {}) return new ObjectExpression();
 	var fieldNames = Object.keys(obj);
-	for (var fc = 0, n = fieldNames.length; fc < n; ++fc) {
-		var fn = fieldNames[fc];
-		if (fn[0] === "$") {
-			if (fc !== 0) throw new Error("the operator must be the only field in a pipeline object (at '" + fn + "'.; code 16410");
-			if(ctx.isTopLevel) throw new Error("$expressions are not allowed at the top-level of $project; code 16404");
-			kind = kinds.OPERATOR;	//we've determined this "object" is an operator expression
-			expr = Expression.parseExpression(fn, obj[fn]);
+	if (fieldNames.length === 0) { //NOTE: Added this for mongo 2.5 port of document sources. Should reconsider when porting the expressions themselves
+		return new ObjectExpression();
+	}
+	for (var fieldCount = 0, n = fieldNames.length; fieldCount < n; ++fieldCount) {
+		var pFieldName = fieldNames[fieldCount];
+
+		if (pFieldName[0] === "$") {
+			if (fieldCount !== 0)
+				throw new Error("the operator must be the only field in a pipeline object (at '" + pFieldName + "'.; code 16410");
+
+			if (ctx.isTopLevel)
+				throw new Error("$expressions are not allowed at the top-level of $project; code 16404");
+			kind = KIND_OPERATOR; //we've determined this "object" is an operator expression
+			pExpression = Expression.parseExpression(pFieldName, obj[pFieldName], vps);
 		} else {
-			if (kind === kinds.OPERATOR) throw new Error("this object is already an operator expression, and can't be used as a document expression (at '" + fn + "'.; code 15990");
-			if (!ctx.isTopLevel && fn.indexOf(".") != -1) throw new Error("dotted field names are only allowed at the top level; code 16405");
-			if (expr === undefined) { // if it's our first time, create the document expression
-				if (!ctx.isDocumentOk) throw new Error("document not allowed in this context"); // CW TODO error: document not allowed in this context
-				expr = exprObj = new ObjectExpression();
-				kind = kinds.NOT_OPERATOR;	//this "object" is not an operator expression
+			if (kind === KIND_OPERATOR)
+				throw new Error("this object is already an operator expression, and can't be used as a document expression (at '" + pFieldName + "'.; code 15990");
+
+			if (!ctx.isTopLevel && pFieldName.indexOf(".") != -1)
+				throw new Error("dotted field names are only allowed at the top level; code 16405");
+			if (pExpression === undefined) { // if it's our first time, create the document expression
+				if (!ctx.isDocumentOk)
+					throw new Error("document not allowed in this context"); // CW TODO error: document not allowed in this context
+				pExpression = pExpressionObject = new ObjectExpression(); //check for top level?
+				kind = KIND_NOTOPERATOR; //this "object" is not an operator expression
 			}
-			var fv = obj[fn];
-			switch (typeof(fv)) {
-			case "object":
-				// it's a nested document
-				var subCtx = new ObjectCtx({
-					isDocumentOk: ctx.isDocumentOk,
-					isInclusionOk: ctx.isInclusionOk
-				});
-				exprObj.addField(fn, Expression.parseObject(fv, subCtx));
-				break;
-			case "string":
-				// it's a renamed field		// CW TODO could also be a constant
-				var pathExpr = new FieldPathExpression(Expression.removeFieldPrefix(fv));
-				exprObj.addField(fn, pathExpr);
-				break;
-			case "boolean":
-			case "number":
-				// it's an inclusion specification
-				if (fv) {
-					if (!ctx.isInclusionOk) throw new Error("field inclusion is not allowed inside of $expressions; code 16420");
-					exprObj.includePath(fn);
-				} else {
-					if (!(ctx.isTopLevel && fn == Document.ID_PROPERTY_NAME)) throw new Error("The top-level " + Document.ID_PROPERTY_NAME + " field is the only field currently supported for exclusion; code 16406");
-					exprObj.excludeId = true;
-				}
-				break;
-			default:
-				throw new Error("disallowed field type " + (fv ? fv.constructor.name + ":" : "") + typeof(fv) + " in object expression (at '" + fn + "')");
+			var fieldValue = obj[pFieldName];
+			switch (typeof(fieldValue)) {
+				case "object":
+					// it's a nested document
+					var subCtx = new ObjectCtx({
+						isDocumentOk: ctx.isDocumentOk,
+						isInclusionOk: ctx.isInclusionOk
+					});
+					pExpressionObject.addField(pFieldName, Expression.parseObject(fieldValue, subCtx, vps));
+					break;
+				case "string":
+					// it's a renamed field         // CW TODO could also be a constant
+					var pathExpr = new FieldPathExpression.parse(fieldValue);
+					pExpressionObject.addField(pFieldName, pathExpr);
+					break;
+				case "boolean":
+				case "number":
+					// it's an inclusion specification
+					if (fieldValue) {
+						if (!ctx.isInclusionOk)
+							throw new Error("field inclusion is not allowed inside of $expressions; code 16420");
+						pExpressionObject.includePath(pFieldName);
+					} else {
+						if (!(ctx.isTopLevel && fn == Document.ID_PROPERTY_NAME))
+							throw new Error("The top-level " + Document.ID_PROPERTY_NAME + " field is the only field currently supported for exclusion; code 16406");
+						pExpressionObject.excludeId = true;
+					}
+					break;
+				default:
+					throw new Error("disallowed field type " + (fieldValue ? fieldValue.constructor.name + ":" : "") + typeof(fieldValue) + " in object expression (at '" + pFieldName + "')");
 			}
 		}
 	}
-	return expr;
+	return pExpression;
+};
+
+
+klass.expressionParserMap = {};
+
+klass.registerExpression = function registerExpression(key, parserFunc) {
+	if (key in klass.expressionParserMap) {
+		throw new Error("Duplicate expression registrarion for " + key);
+	}
+	klass.expressionParserMap[key] = parserFunc;
+	return 0; // Should
 };
 
 /**
@@ -276,40 +185,15 @@ klass.parseObject = function parseObject(obj, ctx){
  *
  * @static
  * @method parseExpression
- * @param opName	the name of the (prefix) operator
- * @param obj	the BSONElement to parse
+ * @param opName        the name of the (prefix) operator
+ * @param obj   the BSONElement to parse
  * @returns the parsed Expression
  **/
-klass.parseExpression = function parseExpression(opName, obj) {
-	// look for the specified operator
-	if (opName === "$const") return new ConstantExpression(obj); //TODO: createFromBsonElement was here, not needed since this isn't BSON?
-	var op = klass.opMap[opName];
-	if (!(op instanceof OpDesc)) throw new Error("invalid operator " + opName + "; code 15999");
-
-	// make the expression node
-	var IExpression = op.factory,	//TODO: should this get renamed from `factory` to `ctor` or something?
-		expr = new IExpression();
-
-	// add the operands to the expression node
-	if (op.flags & OpDesc.FIXED_COUNT && op.argCount > 1 && !(obj instanceof Array)) throw new Error("the " + op.name + " operator requires an array of " + op.argCount + " operands; code 16019");
-	var operand; // used below
-	if (obj.constructor === Object) { // the operator must be unary and accept an object argument
-		if (!(op.flags & OpDesc.OBJECT_ARG)) throw new Error("the " + op.name + " operator does not accept an object as an operand");
-		operand = Expression.parseObject(obj, new ObjectCtx({isDocumentOk: 1}));
-		expr.addOperand(operand);
-	} else if (obj instanceof Array) { // multiple operands - an n-ary operator
-		if (op.flags & OpDesc.FIXED_COUNT && op.argCount !== obj.length) throw new Error("the " + op.name + " operator requires " + op.argCount + " operand(s); code 16020");
-		for (var i = 0, n = obj.length; i < n; ++i) {
-			operand = Expression.parseOperand(obj[i]);
-			expr.addOperand(operand);
-		}
-	} else { //assume it's an atomic operand
-		if (op.flags & OpDesc.FIXED_COUNT && op.argCount != 1) throw new Error("the " + op.name + " operator requires an array of " + op.argCount + " operands; code 16022");
-		operand = Expression.parseOperand(obj);
-		expr.addOperand(operand);
+klass.parseExpression = function parseExpression(exprKey, exprValue, vps) {
+	if (!(exprKey in Expression.expressionParserMap)) {
+		throw new Error("Invalid operator : " + exprKey);
 	}
-
-	return expr;
+	return Expression.expressionParserMap[exprKey](exprValue, vps);
 };
 
 /**
@@ -319,14 +203,16 @@ klass.parseExpression = function parseExpression(opName, obj) {
  * @param pBsonElement the expected operand's BSONElement
  * @returns the parsed operand, as an Expression
  **/
-klass.parseOperand = function parseOperand(obj){
-	var t = typeof(obj);
-	if (t === "string" && obj[0] == "$") { //if we got here, this is a field path expression
-		var path = Expression.removeFieldPrefix(obj);
-		return new FieldPathExpression(path);
-	}
-	else if (t === "object" && obj && obj.constructor === Object) return Expression.parseObject(obj, new ObjectCtx({isDocumentOk: true}));
-	else return new ConstantExpression(obj);
+klass.parseOperand = function parseOperand(exprElement, vps) {
+	var t = typeof(exprElement);
+	if (t === "string" && exprElement[0] == "$") { //if we got here, this is a field path expression
+	    return new FieldPathExpression.parse(exprElement, vps);
+	} else
+	if (t === "object" && exprElement && exprElement.constructor === Object)
+		return Expression.parseObject(exprElement, new ObjectCtx({
+			isDocumentOk: true
+		}), vps);
+	else return ConstantExpression.parse(exprElement, vps);
 };
 
 /**
@@ -343,19 +229,6 @@ klass.removeFieldPrefix = function removeFieldPrefix(prefixedField) {
 	return prefixedField.substr(1);
 };
 
-/**
- * returns the signe of a number
- *
- * @static
- * @method signum
- * @returns the sign of a number; -1, 1, or 0
- **/
-klass.signum = function signum(i) {
-	if (i < 0) return -1;
-	if (i > 0) return 1;
-	return 0;
-};
-
 
 // PROTOTYPE MEMBERS
 /**
@@ -364,7 +237,7 @@ klass.signum = function signum(i) {
  * @method evaluate
  * @returns the computed value
  **/
-proto.evaluate = function evaluate(obj) {
+proto.evaluateInternal = function evaluateInternal(obj) {
 	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");
 };
 
@@ -392,8 +265,8 @@ proto.optimize = function optimize() {
  * If any other Expression is an ancestor, or in other cases where {a:1} inclusion objects aren't allowed, they get NULL.
  *
  * @method addDependencies
- * @param deps	output parameter
- * @param path	path to self if all ancestors are ExpressionObjects.
+ * @param deps  output parameter
+ * @param path  path to self if all ancestors are ExpressionObjects.
  **/
 proto.addDependencies = function addDependencies(deps, path) {
 	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");
@@ -407,6 +280,13 @@ proto.getIsSimple = function getIsSimple() {
 	return false;
 };
 
-proto.toMatcherBson = function toMatcherBson(){
-	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");	//verify(false && "Expression::toMatcherBson()");
+proto.toMatcherBson = function toMatcherBson() {
+	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!"); //verify(false && "Expression::toMatcherBson()");
 };
+
+
+// DEPENDENCIES
+var Document = require("../Document");
+var ObjectExpression = require("./ObjectExpression");
+var FieldPathExpression = require("./FieldPathExpression");
+var ConstantExpression = require("./ConstantExpression");

lib/pipeline/expressions/FieldPathExpression.js

@@ -9,19 +9,88 @@
  * @constructor
  * @param {String} fieldPath the field path string, without any leading document indicator
  **/
-var FieldPathExpression = module.exports = function FieldPathExpression(path){
-	if (arguments.length !== 1) throw new Error("args expected: path");
-	this.path = new FieldPath(path);
+
+var Expression = require("./Expression"),
+    Variables = require("./Variables"),
+    Value = require("../Value"),
+    FieldPath = require("../FieldPath");
+
+
+var FieldPathExpression = module.exports = function FieldPathExpression(path, variableId){
+    if (arguments.length > 2) throw new Error("args expected: path[, vps]");
+    this.path = new FieldPath(path);
+    if(arguments.length == 2) {
+        this.variable = variableId;
+    } else {
+        this.variable = Variables.ROOT_ID;
+    }
 }, klass = FieldPathExpression, base = require("./Expression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// DEPENDENCIES
-var FieldPath = require("../FieldPath");
+klass.create = function create(path) {
+    return new FieldPathExpression("CURRENT."+path, Variables.ROOT_ID);
+};
+
 
 // PROTOTYPE MEMBERS
-proto.evaluate = function evaluate(obj){
-	return this._evaluatePath(obj, 0, this.path.fields.length);
+proto.evaluateInternal = function evaluateInternal(vars){
+
+    if(this.path.fields.length === 1) {
+        return vars.getValue(this.variable);
+    }
+
+    if(this.variable === Variables.ROOT_ID) {
+        return this.evaluatePath(1, vars.getRoot());
+    }
+
+    var vari = vars.getValue(this.variable);
+    if(vari instanceof Array) {
+        return this.evaluatePathArray(1,vari);
+    } else if (vari instanceof Object) {
+        return this.evaluatePath(1, vari);
+    } else {
+        return undefined;
+    }
+};
+
+
+/**
+ * Parses a fieldpath using the mongo 2.5 spec with optional variables
+ *
+ * @param raw raw string fieldpath
+ * @param vps variablesParseState
+ * @returns a new FieldPathExpression
+ **/
+klass.parse = function parse(raw, vps) {
+    if(raw[0] !== "$") {
+        throw new Error("FieldPath: '" + raw + "' doesn't start with a $");
+    }
+    if(raw.length === 1) {
+        throw new Error("'$' by itself is not a valid FieldPath");
+    }
+
+    if(raw[1] === "$") {
+        var firstPeriod = raw.indexOf('.');
+        var varname = (firstPeriod === -1 ? raw.slice(2) : raw.slice(2,firstPeriod));
+        Variables.uassertValidNameForUserRead(varname);
+        return new FieldPathExpression(raw.slice(2), vps.getVariableName(varname));
+    } else {
+        return new FieldPathExpression("CURRENT." + raw.slice(1), vps.getVariable("CURRENT"));
+    }
+};
+
+
+/**
+ * Parses a fieldpath using the mongo 2.5 spec with optional variables
+ *
+ * @param raw raw string fieldpath
+ * @param vps variablesParseState
+ * @returns a new FieldPathExpression
+ **/
+proto.optimize = function optimize() {
+    return this;
 };
 
+
 /**
  * Internal implementation of evaluate(), used recursively.
  *
@@ -69,22 +138,67 @@ proto._evaluatePath = function _evaluatePath(obj, i, len){
 	return undefined;
 };
 
+proto.evaluatePathArray = function evaluatePathArray(index, input) {
+
+    if(!(input instanceof Array)) {
+        throw new Error("evaluatePathArray called on non-array");
+    }
+    var result = [];
+
+    for(var ii = 0; ii < input.length; ii++) {
+        if(input[ii] instanceof Object) {
+            var nested = this.evaluatePath(index, input[ii]);
+            if(nested) {
+				result.push(nested);
+            }
+        }
+    }
+    return result;
+};
+
+
+proto.evaluatePath = function(index, input) {
+    if(index === this.path.fields.length -1) {
+        return input[this.path.fields[index]];
+    }
+    var val = input[this.path.fields[index]];
+    if(val instanceof Array) {
+        return this.evaluatePathArray(index+1, val);
+    } else if (val instanceof Object) {
+        return this.evaluatePath(index+1, val);
+    } else {
+        return undefined;
+    }
+
+};
+
+
+
 proto.optimize = function(){
-	return this;
+        return this;
 };
 
 proto.addDependencies = function addDependencies(deps){
-	deps[this.path.getPath()] = 1;
-	return deps;
+	if(this.path.fields[0] === "CURRENT" || this.path.fields[0] === "ROOT") {
+		if(this.path.fields.length === 1) {
+			deps[""] = 1;
+		} else {
+			deps[this.path.tail().getPath(false)] = 1;
+		}
+	}
 };
 
 // renamed write to get because there are no streams
 proto.getFieldPath = function getFieldPath(usePrefix){
-	return this.path.getPath(usePrefix);
+        return this.path.getPath(usePrefix);
 };
 
-proto.toJSON = function toJSON(){
-	return this.path.getPath(true);
+proto.serialize = function toJSON(){
+    if(this.path.fields[0] === "CURRENT" && this.path.fields.length > 1) {
+        return "$" + this.path.tail().getPath(false);
+    } else {
+        return "$$" + this.path.getPath(false);
+    }
 };
 
 //TODO: proto.addToBsonObj = ...?

lib/pipeline/expressions/FieldRangeExpression.js

@@ -6,9 +6,9 @@
  * Field ranges are meant to match up with classic Matcher semantics, and therefore are conjunctions.
  *
  * For example, these appear in mongo shell predicates in one of these forms:
- *	{ a : C } -> (a == C) // degenerate "point" range
- *	{ a : { $lt : C } } -> (a < C) // open range
- *	{ a : { $gt : C1, $lte : C2 } } -> ((a > C1) && (a <= C2)) // closed
+ *      { a : C } -> (a == C) // degenerate "point" range
+ *      { a : { $lt : C } } -> (a < C) // open range
+ *      { a : { $gt : C1, $lte : C2 } } -> ((a > C1) && (a <= C2)) // closed
  *
  * When initially created, a field range only includes one end of the range.  Additional points may be added via intersect().
  *
@@ -141,6 +141,8 @@ var Range = (function(){
 proto.evaluate = function evaluate(obj){
 	if(this.range === undefined) return false;
 	var value = this.pathExpr.evaluate(obj);
+	if(value instanceof Array)
+		throw new Error('FieldRangeExpression cannot evaluate an array.');
 	return this.range.contains(value);
 };
 

lib/pipeline/expressions/HourExpression.js

@@ -1,15 +1,15 @@
 "use strict";
 
-/** 
- * An $hour pipeline expression. 
- * @see evaluate 
+/**
+ * An $hour pipeline expression.
+ * @see evaluateInternal
  * @class HourExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var HourExpression = module.exports = function HourExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+	this.nargs = 1;
 	base.call(this);
 }, klass = HourExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
@@ -18,17 +18,18 @@ proto.getOpName = function getOpName(){
 	return "$hour";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
+// DEPENDENCIES
+var Expression = require("./Expression");
 
-/** 
- * Takes a date and returns the hour between 0 and 23. 
- * @method evaluate
+/**
+ * Takes a date and returns the hour between 0 and 23.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars){
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCHours();
 };
+
+
+/** Register Expression */
+Expression.registerExpression("$hour",base.parse(HourExpression));

lib/pipeline/expressions/IfNullExpression.js

@@ -8,29 +8,38 @@
  * @module mungedb-aggregate
  * @constructor
  **/
-var IfNullExpression = module.exports = function IfNullExpression(){
+var IfNullExpression = module.exports = function IfNullExpression() {
+	this.nargs = 2;
 	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = IfNullExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = IfNullExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$ifNull";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
+// virtuals from ExpressionNary
 
 /**
  * Use the $ifNull operator with the following syntax: { $ifNull: [ <expression>, <replacement-if-null> ] }
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc);
-	if(left !== undefined && left !== null) return left;
-	var right = this.operands[1].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var left = this.operands[0].evaluateInternal(vars);
+	if (left !== undefined && left !== null) return left;
+	var right = this.operands[1].evaluateInternal(vars);
 	return right;
 };
+
+/** Register Expression */
+Expression.registerExpression("$ifNull", base.parse(IfNullExpression));

lib/pipeline/expressions/LetExpression.js

@@ -0,0 +1,105 @@
+"use strict";
+
+var LetExpression = module.exports = function LetExpression(vars, subExpression){
+	if (arguments.length !== 2) throw new Error("Two args expected");
+	this._variables = vars;
+	this._subExpression = subExpression;
+}, klass = LetExpression, Expression = require("./Expression"), base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var Variables = require("./Variables"),
+	VariablesParseState = require("./VariablesParseState");
+
+// PROTOTYPE MEMBERS
+
+
+proto.parse = function parse(expr, vpsIn){
+	if(!("$let" in expr)) {
+		throw new Error("Tried to create a $let with something other than let. Looks like your parse map went all funny.");
+	}
+
+	if(typeof(expr.$let) !== 'object' || (expr.$let instanceof Array)) {
+		throw new Error("$let only supports an object as it's argument:16874");
+	}
+
+	var args = expr.$let,
+		varsElem = args.vars,
+		inElem = args['in'];
+
+	if(!varsElem) {
+		throw new Error("Missing 'vars' parameter to $let: 16876");
+	}
+	if(!inElem) {
+		throw new Error("Missing 'in' parameter to $let: 16877");
+	}
+
+	if(Object.keys(args).length > 2) {
+		var bogus = Object.keys(args).filter(function(x) {return !(x === 'in' || x === 'vars');});
+		throw new Error("Unrecognized parameter to $let: " + bogus.join(",") + "- 16875");
+	}
+
+	var vpsSub = new VariablesParseState(vpsIn),
+		vars = {};
+
+	for(var varName in varsElem) {
+		Variables.uassertValidNameForUserWrite(varName);
+		var id = vpsSub.defineVariable(varName);
+
+		vars[id] = {};
+		vars[id][varName] = Expression.parseOperand(varsElem, vpsIn);
+	}
+
+	var subExpression = Expression.parseOperand(inElem, vpsSub);
+	return new LetExpression(vars, subExpression);
+};
+
+proto.optimize = function optimize() {
+	if(this._variables.empty()) {
+		return this._subExpression.optimize();
+	}
+
+	for(var id in this._variables){
+		for(var name in this._variables[id]) {
+			this._variables[id][name] = this._variables[id][name].optimize();
+		}
+	}
+
+	this._subExpression = this._subExpression.optimize();
+
+	return this;
+};
+
+proto.addDependencies = function addDependencies(deps, path){
+	for(var id in this._variables) {
+		for(var name in this._variables[id]) {
+			this._variables[id][name].addDependencies(deps);
+		}
+	}
+	this._subExpression.addDependencies(deps, path);
+	return deps;
+
+};
+
+proto.evaluateInternal = function evaluateInternal(vars) {
+	for(var id in this._variables) {
+		for(var name in this._variables[id]) {
+			vars.setValue(id, this._variables[id][name]);
+		}
+	}
+
+	return this._subExpression.evaluateInternal(vars);
+};
+
+
+proto.serialize = function serialize(explain) {
+	var vars = {};
+	for(var id in this._variables) {
+		for(var name in this._variables[id]) {
+			vars[name] = this._variables[id][name];
+		}
+	}
+
+	return {$let: {vars:vars, 'in':this._subExpression.serialize(explain)}};
+};
+
+Expression.registerExpression("$let", LetExpression.parse);

lib/pipeline/expressions/MapExpression.js

@@ -0,0 +1,107 @@
+"use strict";
+
+var MapExpression = module.exports = function MapExpression(varName, varId, input, each){
+	if (arguments.length !== 4) throw new Error("Four args expected");
+	this._varName = varName;
+	this._varId = varId;
+	this._input = input;
+	this._each = each;
+}, klass = MapExpression, Expression = require("./Expression"), base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var Variables = require("./Variables"),
+	VariablesParseState = require("./VariablesParseState");
+
+// PROTOTYPE MEMBERS
+
+
+klass.parse = function parse(expr, vpsIn){
+	if(!("$map" in expr)) {
+		throw new Error("Tried to create a $let with something other than let. Looks like your parse map went all funny.");
+	}
+
+	if(typeof(expr.$map) !== 'object' || (expr.$map instanceof Array)) {
+		throw new Error("$map only supports an object as it's argument:16878");
+	}
+
+	var args = expr.$map,
+		inputElem = args.input,
+		inElem = args['in'],
+		asElem = args.as;
+
+	if(!inputElem) {
+		throw new Error("Missing 'input' parameter to $map: 16880");
+	}
+	if(!asElem) {
+		throw new Error("Missing 'as' parameter to $map: 16881");
+	}
+	if(!inElem) {
+		throw new Error("Missing 'in' parameter to $let: 16882");
+	}
+
+
+	if(Object.keys(args).length > 3) {
+		var bogus = Object.keys(args).filter(function(x) {return !(x === 'in' || x === 'as' || x === 'input');});
+		throw new Error("Unrecognized parameter to $map: " + bogus.join(",") + "- 16879");
+	}
+
+	var input = Expression.parseOperand(inputElem, vpsIn);
+
+	var vpsSub = new VariablesParseState(vpsIn),
+		varName = asElem;
+
+	Variables.uassertValidNameForUserWrite(varName);
+	var varId = vpsSub.defineVariable(varName);
+
+	var invert = Expression.parseOperand(inElem, vpsSub);
+
+	return new MapExpression(varName, varId, input, invert);
+};
+
+
+proto.optimize = function optimize() {
+	this._input = this._input.optimize();
+	this._each = this._each.optimize();
+	return this;
+};
+
+proto.serialize = function serialize(explain) {
+	return {$map: {input:this._input.serialize(explain),
+				   as: this._varName,
+				   'in': this._each.serialize(explain)}};
+};
+
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var inputVal = this._input.evaluateInternal(vars);
+	if( inputVal === null) {
+		return null;
+	}
+
+	if(!(inputVal instanceof Array)) {
+		throw new Error("Input to $map must be an Array, not a ____ 16883");
+	}
+
+	if(inputVal.length === 0) {
+		return [];
+	}
+
+	// Diverge from Mongo source here, as Javascript has a builtin map operator.
+	return inputVal.map(function(x) {
+	   vars.setValue(this._varId, x);
+	   var toInsert = this._each.evaluateInternal(vars);
+	   if(toInsert === undefined) {
+		   toInsert = null;
+	   }
+
+	   return toInsert;
+   });
+};
+
+proto.addDependencies = function addDependencies(deps, path){
+	this._input.addDependencies(deps, path);
+	this._each.addDependencies(deps, path);
+	return deps;
+};
+
+
+Expression.registerExpression("$map", klass.parse);

lib/pipeline/expressions/MillisecondExpression.js

@@ -0,0 +1,42 @@
+"use strict";
+
+/**
+ * An $millisecond pipeline expression.
+ * @see evaluateInternal
+ * @class MillisecondExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var MillisecondExpression = module.exports = function MillisecondExpression() {
+	this.nargs = 1;
+	base.call(this);
+}, klass = MillisecondExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$millisecond";
+};
+
+/**
+ * Takes a date and returns the millisecond between 0 and 999, but can be 1000 to account for leap milliseconds.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
+	return date.getUTCMilliseconds(); //TODO: incorrect for last millisecond of leap year, need to fix...
+	// currently leap milliseconds are unsupported in v8
+	// http://code.google.com/p/v8/issues/detail?id=1944
+};
+
+/** Register Expression */
+Expression.registerExpression("$millisecond", base.parse(MillisecondExpression));

lib/pipeline/expressions/MinuteExpression.js

@@ -1,34 +1,40 @@
 "use strict";
 
-/** 
- * An $minute pipeline expression. 
- * @see evaluate 
+/**
+ * An $minute pipeline expression.
+ * @see evaluateInternal
  * @class MinuteExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var MinuteExpression = module.exports = function MinuteExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var MinuteExpression = module.exports = function MinuteExpression() {
+	this.nargs = 1;
 	base.call(this);
-}, klass = MinuteExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = MinuteExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$minute";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Takes a date and returns the minute between 0 and 59. 
- * @method evaluate
+/**
+ * Takes a date and returns the minute between 0 and 59.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCMinutes();
 };
+
+/** Register Expression */
+Expression.registerExpression("$minute", base.parse(MinuteExpression));

lib/pipeline/expressions/ModExpression.js

@@ -1,49 +1,54 @@
 "use strict";
 
-/** 
- * An $mod pipeline expression. 
- * @see evaluate 
+/**
+ * An $mod pipeline expression.
+ * @see evaluate
  * @class ModExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var ModExpression = module.exports = function ModExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var ModExpression = module.exports = function ModExpression() {
+	this.nargs = 2;
 	base.call(this);
-}, klass = ModExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = ModExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$mod";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Takes an array that contains a pair of numbers and returns the remainder of the first number divided by the second number. 
+/**
+ * Takes an array that contains a pair of numbers and returns the remainder of the first number divided by the second number.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars) {
 	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc),
-		right = this.operands[1].evaluate(doc);
-	if(left instanceof Date || right instanceof Date) throw new Error("$mod does not support dates; code 16374");
+	var left = this.operands[0].evaluateInternal(vars),
+		right = this.operands[1].evaluateInternal(vars);
+	if (left instanceof Date || right instanceof Date) throw new Error("$mod does not support dates; code 16374");
 
 	// pass along jstNULLs and Undefineds
-	if(left === undefined || left === null) return left;
-	if(right === undefined || right === null) return right;
+	if (left === undefined || left === null) return left;
+	if (right === undefined || right === null) return right;
 
 	// ensure we aren't modding by 0
 	right = Value.coerceToDouble(right);
-	if(right === 0) return undefined;
+	if (right === 0) return undefined;
 
 	left = Value.coerceToDouble(left);
 	return left % right;
 };
+
+/** Register Expression */
+Expression.registerExpression("$mod", base.parse(ModExpression));

lib/pipeline/expressions/MonthExpression.js

@@ -1,34 +1,40 @@
 "use strict";
 
-/** 
- * A $month pipeline expression. 
- * @see evaluate 
+/**
+ * A $month pipeline expression.
+ * @see evaluate
  * @class MonthExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var MonthExpression = module.exports = function MonthExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var MonthExpression = module.exports = function MonthExpression() {
+	this.nargs = 1;
 	base.call(this);
-}, klass = MonthExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
-	
+}, klass = MonthExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
+
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$month";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the month as a number between 1 and 12.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
-	return date.getUTCMonth() + 1;
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
+	return date.getUTCMonth();
 };
+
+/** Register Expression */
+Expression.registerExpression("$month", base.parse(MonthExpression));

lib/pipeline/expressions/MultiplyExpression.js

@@ -1,34 +1,35 @@
 "use strict";
 
-/** 
- * A $multiply pipeline expression. 
- * @see evaluate 
+/**
+ * A $multiply pipeline expression.
+ * @see evaluateInternal
  * @class MultiplyExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var MultiplyExpression = module.exports = function MultiplyExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+	if (arguments.length !== 0) throw new Error("Zero args expected");
 	base.call(this);
 }, klass = MultiplyExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+ Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$multiply";
 };
 
-/** 
- * Takes an array of one or more numbers and multiples them, returning the resulting product. 
- * @method evaluate
+/**
+ * Takes an array of one or more numbers and multiples them, returning the resulting product.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars){
 	var product = 1;
 	for(var i = 0, n = this.operands.length; i < n; ++i){
-		var value = this.operands[i].evaluate(doc);
+		var value = this.operands[i].evaluateInternal(vars);
 		if(value instanceof Date) throw new Error("$multiply does not support dates; code 16375");
 		product *= Value.coerceToDouble(value);
 	}
@@ -36,6 +37,5 @@ proto.evaluate = function evaluate(doc){
 	return product;
 };
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
-};
+/** Register Expression */
+Expression.registerExpression("$multiply", base.parse(MultiplyExpression));

lib/pipeline/expressions/NaryExpression.js

@@ -8,12 +8,49 @@
  * @extends mungedb-aggregate.pipeline.expressions.Expression
  * @constructor
  **/
+var Expression = require("./Expression");
+
 var NaryExpression = module.exports = function NaryExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+	if (arguments.length !== 0) throw new Error("Zero args expected");
 	this.operands = [];
 	base.call(this);
-}, klass = NaryExpression, base = require("./Expression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = NaryExpression, base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+klass.parse = function(SubClass) {
+	return function parse(expr, vps) {
+		var outExpr = new SubClass(),
+			args = NaryExpression.parseArguments(expr, vps);
+		outExpr.validateArguments(args);
+		outExpr.operands = args;
+		return outExpr;
+	};
+};
 
+klass.parseArguments = function(exprElement, vps) {
+	var out = [];
+	if(exprElement instanceof Array) {
+		for(var ii = 0; ii < exprElement.length; ii++) {
+			out.push(Expression.parseOperand(exprElement[ii], vps));
+		}
+	} else {
+		out.push(Expression.parseOperand(exprElement, vps));
+	}
+	return out;
+};
+
+
+function partitionBy(fn, coll) {
+	var ret = {pass:[],
+			   fail:[]};
+	coll.forEach(function(x) {
+		if(fn(x)) {
+			ret.pass.push(x);
+		} else {
+			ret.fail.push(x);
+		}
+	});
+	return ret;
+}
 // DEPENDENCIES
 var ConstantExpression = require("./ConstantExpression");
 
@@ -25,69 +62,47 @@ proto.getOpName = function getOpName(doc){
 };
 
 proto.optimize = function optimize(){
-	var constsFound = 0,
-		stringsFound = 0;
-	for (var i = 0, l = this.operands.length; i < l; i++) {
-		var optimizedExpr = this.operands[i].optimize();
-		if (optimizedExpr instanceof ConstantExpression) {
-			constsFound++;
-			if (typeof(optimizedExpr.value) == "string") stringsFound++;
-		}
-		this.operands[i] = optimizedExpr;
-	}
-	// If all the operands are constant, we can replace this expression with a constant.  We can find the value by evaluating this expression over a NULL Document because evaluating the ExpressionConstant never refers to the argument Document.
-	if (constsFound === l) return new ConstantExpression(this.evaluate());
-	// If there are any strings, we can't re-arrange anything, so stop now.     LATER:  we could concatenate adjacent strings as a special case.
-	if (stringsFound) return this;
-	// If there's no more than one constant, then we can't do any constant folding, so don't bother going any further.
-	if (constsFound <= 1) return this;
-	// If the operator isn't commutative or associative, there's nothing more we can do.  We test that by seeing if we can get a factory; if we can, we can use it to construct a temporary expression which we'll evaluate to collapse as many constants as we can down to a single one.
-	var IExpression = this.getFactory();
-	if (!(IExpression instanceof Function)) return this;
-	// Create a new Expression that will be the replacement for this one.  We actually create two:  one to hold constant expressions, and one to hold non-constants.
-	// Once we've got these, we evaluate the constant expression to produce a single value, as above.  We then add this operand to the end of the non-constant expression, and return that.
-	var expr = new IExpression(),
-		constExpr = new IExpression();
-	for (i = 0; i < l; ++i) {
-		var operandExpr = this.operands[i];
-		if (operandExpr instanceof ConstantExpression) {
-			constExpr.addOperand(operandExpr);
+	var n = this.operands.length,
+		constantCount = 0;
+
+	for(var ii = 0; ii < n; ii++) {
+		if(this.operands[ii] instanceof ConstantExpression) {
+			constantCount++;
 		} else {
-			// If the child operand is the same type as this, then we can extract its operands and inline them here because we already know this is commutative and associative because it has a factory.  We can detect sameness of the child operator by checking for equality of the factory
-			// Note we don't have to do this recursively, because we called optimize() on all the children first thing in this call to optimize().
-			if (!(operandExpr instanceof NaryExpression)) {
-				expr.addOperand(operandExpr);
-			} else {
-				if (operandExpr.getFactory() !== IExpression) {
-					expr.addOperand(operandExpr);
-				} else { // same factory, so flatten
-					for (var i2 = 0, n2 = operandExpr.operands.length; i2 < n2; ++i2) {
-						var childOperandExpr = operandExpr.operands[i2];
-						if (childOperandExpr instanceof ConstantExpression) {
-							constExpr.addOperand(childOperandExpr);
-						} else {
-							expr.addOperand(childOperandExpr);
+			this.operands[ii] = this.operands[ii].optimize();
 						}
 					}
+
+	if(constantCount === n) {
+		return new ConstantExpression(this.evaluateInternal({}));
 				}
-			}
-		}
-	}
 
-	if (constExpr.operands.length === 1) { // If there was only one constant, add it to the end of the expression operand vector.
-		expr.addOperand(constExpr.operands[0]);
-	} else if (constExpr.operands.length > 1) { // If there was more than one constant, collapse all the constants together before adding the result to the end of the expression operand vector.
-		var pResult = constExpr.evaluate();
-		expr.addOperand(new ConstantExpression(pResult));
+	if(!this.isAssociativeAndCommutative) {
+		return this;
 	}
 
-	return expr;
+	// Flatten and inline nested operations of the same type
+
+	var similar = partitionBy(function(x){ return x.getOpName() === this.getOpName();}, this.operands);
+
+	this.operands = similar.fail;
+	similar.pass.forEach(function(x){
+		this.operands.concat(x.operands);
+	});
+
+	// Partial constant folding
+
+	var constantOperands = partitionBy(function(x) {return x instanceof ConstantExpression;}, this.operands);
+
+	this.operands = constantOperands.pass;
+	this.operands = [new ConstantExpression(this.evaluateInternal({}))].concat(constantOperands.fail);
+
+	return this;
 };
 
 proto.addDependencies = function addDependencies(deps){
 	for(var i = 0, l = this.operands.length; i < l; ++i)
 		this.operands[i].addDependencies(deps);
-	return deps;
 };
 
 /**
@@ -99,40 +114,21 @@ proto.addOperand = function addOperand(expr) {
 	this.operands.push(expr);
 };
 
-proto.getFactory = function getFactory() {
-	return undefined;
+proto.serialize = function serialize() {
+	var ret = {}, subret = [];
+	for(var ii = 0; ii < this.operands.length; ii++) {
+		subret.push(this.operands[ii].serialize());
+	}
+	ret[this.getOpName()] = subret;
+	return ret;
 };
 
-proto.toJSON = function toJSON() {
-	var o = {};
-	o[this.getOpName()] = this.operands.map(function(operand){
-		return operand.toJSON();
-	});
-	return o;
+proto.fixedArity = function(nargs) {
+	this.nargs = nargs;
 };
 
-//TODO:	proto.toBson  ?   DONE NOW???
-//TODO:	proto.addToBsonObj  ?
-//TODO: proto.addToBsonArray  ?
-
-/**
- * Checks the current size of vpOperand; if the size equal to or greater than maxArgs, fires a user assertion indicating that this operator cannot have this many arguments.
- * The equal is there because this is intended to be used in addOperand() to check for the limit *before* adding the requested argument.
- *
- * @method checkArgLimit
- * @param maxArgs the maximum number of arguments the operator accepts
- **/
-proto.checkArgLimit = function checkArgLimit(maxArgs) {
-	if (this.operands.length >= maxArgs) throw new Error(this.getOpName() + " only takes " + maxArgs + " operand" + (maxArgs == 1 ? "" : "s") + "; code 15993");
+proto.validateArguments = function(args) {
+	if(this.nargs !== args.length) {
+		throw new Error("Expression " + this.getOpName() + " takes exactly " + this.nargs + " arguments. " + args.length + " were passed in.");
+	}
 };
-
-/**
- * Checks the current size of vpOperand; if the size is not equal to reqArgs, fires a user assertion indicating that this must have exactly reqArgs arguments.
- * This is meant to be used in evaluate(), *before* the evaluation takes place.
- *
- * @method checkArgCount
- * @param reqArgs the number of arguments this operator requires
- **/
-proto.checkArgCount = function checkArgCount(reqArgs) {
-	if (this.operands.length !== reqArgs) throw new Error(this.getOpName() + ":  insufficient operands; " + reqArgs + " required, only got " + this.operands.length + "; code 15997");
-};

lib/pipeline/expressions/NotExpression.js

@@ -1,37 +1,41 @@
 "use strict";
 
-/** 
- * A $not pipeline expression. 
- * @see evaluate 
+/**
+ * A $not pipeline expression.
+ * @see evaluateInternal
  * @class NotExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var NotExpression = module.exports = function NotExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var NotExpression = module.exports = function NotExpression() {
+	this.nargs = 1;
 	base.call(this);
-}, klass = NotExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = NotExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$not";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Returns the boolean opposite value passed to it. When passed a true value, $not returns false; when passed a false value, $not returns true. 
- * @method evaluate
+/**
+ * Returns the boolean opposite value passed to it. When passed a true value, $not returns false; when passed a false value, $not returns true.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var op = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var op = this.operands[0].evaluateInternal(vars);
 	return !Value.coerceToBool(op);
 };
+
+/** Register Expression */
+Expression.registerExpression("$not", base.parse(NotExpression));

lib/pipeline/expressions/ObjectExpression.js

@@ -1,20 +1,29 @@
 "use strict";
 
 /**
- * Create an empty expression.  Until fields are added, this will evaluate to an empty document (object).
+ * Create an empty expression.  Until fields are added, this will evaluateInternal to an empty document (object).
  * @class ObjectExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @extends mungedb-aggregate.pipeline.expressions.Expression
  * @constructor
  **/
-var ObjectExpression = module.exports = function ObjectExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var ObjectExpression = module.exports = function ObjectExpression(atRoot){
+	if (arguments.length !== 1) throw new Error("one arg expected");
 	this.excludeId = false;	/// <Boolean> for if _id is to be excluded
+	this.atRoot = atRoot;
 	this._expressions = {};	/// <Object<Expression>> mapping from fieldname to Expression to generate the value NULL expression means include from source document
 	this._order = []; /// <Array<String>> this is used to maintain order for generated fields not in the source document
 }, klass = ObjectExpression, Expression = require("./Expression"), base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
+klass.create = function create() {
+	return new ObjectExpression(false);
+};
+
+klass.createRoot = function createRoot() {
+	return new ObjectExpression(true);
+};
+
 // DEPENDENCIES
 var Document = require("../Document"),
 	FieldPath = require("../FieldPath");
@@ -41,20 +50,20 @@ proto._order = [];
 // PROTOTYPE MEMBERS
 
 /**
- * evaluate(), but return a Document instead of a Value-wrapped Document.
+ * evaluateInternal(), but return a Document instead of a Value-wrapped Document.
  * @method evaluateDocument
- * @param pDocument the input Document
+ * @param currentDoc the input Document
  * @returns the result document
  **/
-proto.evaluateDocument = function evaluateDocument(doc) {
+proto.evaluateDocument = function evaluateDocument(vars) {
 	// create and populate the result
-	var pResult = {};
-	this.addToDocument(pResult, pResult, doc); // No inclusion field matching.
+	var pResult = {_id:0};
+	this.addToDocument(pResult, pResult, vars); // No inclusion field matching.
 	return pResult;
 };
 
-proto.evaluate = function evaluate(doc) { //TODO: collapse with #evaluateDocument()?
-	return this.evaluateDocument(doc);
+proto.evaluateInternal = function evaluateInternal(vars) { //TODO: collapse with #evaluateDocument()?
+	return this.evaluateDocument(vars);
 };
 
 proto.optimize = function optimize(){
@@ -65,21 +74,22 @@ proto.optimize = function optimize(){
 	return this;
 };
 
-proto.getIsSimple = function getIsSimple(){
+proto.isSimple = function isSimple(){
 	for (var key in this._expressions) {
 		var expr = this._expressions[key];
-		if (expr !== undefined && expr !== null && !expr.getIsSimple()) return false;
+		if (expr !== undefined && expr !== null && !expr.isSimple()) return false;
 	}
 	return true;
 };
 
 proto.addDependencies = function addDependencies(deps, path){
-	var depsSet = {};
 	var pathStr = "";
 	if (path instanceof Array) {
 		if (path.length === 0) {
 			// we are in the top level of a projection so _id is implicit
-			if (!this.excludeId) depsSet[Document.ID_PROPERTY_NAME] = 1;
+			if (!this.excludeId) {
+							deps[Document.ID_PROPERTY_NAME] = 1;
+						}
 		} else {
 			pathStr = new FieldPath(path).getPath() + ".";
 		}
@@ -94,38 +104,35 @@ proto.addDependencies = function addDependencies(deps, path){
 			if (path instanceof Array) path.pop();
 		} else { // inclusion
 			if (path === undefined || path === null) throw new Error("inclusion not supported in objects nested in $expressions; uassert code 16407");
-			depsSet[pathStr + key] = 1;
+			deps[pathStr + key] = 1;
 		}
 	}
-	//Array.prototype.push.apply(deps, Object.getOwnPropertyNames(depsSet));
-	for(key in depsSet) {
-		deps[key] = 1;
-	}
+
 	return deps;	// NOTE: added to munge as a convenience
 };
 
 /**
- * evaluate(), but add the evaluated fields to a given document instead of creating a new one.
+ * evaluateInternal(), but add the evaluated fields to a given document instead of creating a new one.
  * @method addToDocument
  * @param pResult the Document to add the evaluated expressions to
- * @param pDocument the input Document for this level
- * @param rootDoc the root of the whole input document
+ * @param currentDoc the input Document for this level
+ * @param vars the root of the whole input document
  **/
-proto.addToDocument = function addToDocument(pResult, pDocument, rootDoc){
-	var atRoot = (pDocument === rootDoc);
+proto.addToDocument = function addToDocument(out, currentDoc, vars){
+
 
 	var doneFields = {};	// This is used to mark fields we've done so that we can add the ones we haven't
 
-	for(var fieldName in pDocument){
-		if (!pDocument.hasOwnProperty(fieldName)) continue;
-		var fieldValue = pDocument[fieldName];
+	for(var fieldName in currentDoc){
+		if (!currentDoc.hasOwnProperty(fieldName)) continue;
+		var fieldValue = currentDoc[fieldName];
 
 		// This field is not supposed to be in the output (unless it is _id)
 		if (!this._expressions.hasOwnProperty(fieldName)) {
-			if (!this.excludeId && atRoot && fieldName == Document.ID_PROPERTY_NAME) {
+			if (!this.excludeId && this.atRoot && fieldName == Document.ID_PROPERTY_NAME) {
 				// _id from the root doc is always included (until exclusion is supported)
 				// not updating doneFields since "_id" isn't in _expressions
-				pResult[fieldName] = fieldValue;
+				out[fieldName] = fieldValue;
 			}
 			continue;
 		}
@@ -136,41 +143,41 @@ proto.addToDocument = function addToDocument(pResult, pDocument, rootDoc){
 		// This means pull the matching field from the input document
 		var expr = this._expressions[fieldName];
 		if (!(expr instanceof Expression)) {
-			pResult[fieldName] = fieldValue;
+			out[fieldName] = fieldValue;
 			continue;
 		}
 
 		// Check if this expression replaces the whole field
 		if (!(fieldValue instanceof Object) || (fieldValue.constructor !== Object && fieldValue.constructor !== Array) || !(expr instanceof ObjectExpression)) {
-			var pValue = expr.evaluate(rootDoc);
+			var pValue = expr.evaluateInternal(vars);
 
 			// don't add field if nothing was found in the subobject
 			if (expr instanceof ObjectExpression && pValue instanceof Object && Object.getOwnPropertyNames(pValue).length === 0) continue;
 
 			// Don't add non-existent values (note:  different from NULL); this is consistent with existing selection syntax which doesn't force the appearnance of non-existent fields.
 			// TODO make missing distinct from Undefined
-			if (pValue !== undefined) pResult[fieldName] = pValue;
+			if (pValue !== undefined) out[fieldName] = pValue;
 			continue;
 		}
 
 		// Check on the type of the input value.  If it's an object, just walk down into that recursively, and add it to the result.
 		if (fieldValue instanceof Object && fieldValue.constructor === Object) {
-			pResult[fieldName] = expr.addToDocument({}, fieldValue, rootDoc);	//TODO: pretty sure this is broken;
+			out[fieldName] = expr.addToDocument({}, fieldValue, vars);	//TODO: pretty sure this is broken;
 		} else if (fieldValue instanceof Object && fieldValue.constructor === Array) {
 			// If it's an array, we have to do the same thing, but to each array element.  Then, add the array of results to the current document.
 			var result = [];
 			for(var fvi = 0, fvl = fieldValue.length; fvi < fvl; fvi++){
 				var subValue = fieldValue[fvi];
 				if (subValue.constructor !== Object) continue;	// can't look for a subfield in a non-object value.
-				result.push(expr.addToDocument({}, subValue, rootDoc));
+				result.push(expr.addToDocument({}, subValue, vars));
 			}
-			pResult[fieldName] = result;
+			out[fieldName] = result;
 		} else {
 			throw new Error("should never happen");	//verify( false );
 		}
 	}
 
-	if (Object.getOwnPropertyNames(doneFields).length == Object.getOwnPropertyNames(this._expressions).length) return pResult;	//NOTE: munge returns result as a convenience
+	if (Object.getOwnPropertyNames(doneFields).length == Object.getOwnPropertyNames(this._expressions).length) return out;	//NOTE: munge returns result as a convenience
 
 	// add any remaining fields we haven't already taken care of
 	for(var i = 0, l = this._order.length; i < l; i++){
@@ -183,18 +190,18 @@ proto.addToDocument = function addToDocument(pResult, pDocument, rootDoc){
 		// this is a missing inclusion field
 		if (!expr2) continue;
 
-		var value = expr2.evaluate(rootDoc);
+		var value = expr2.evaluateInternal(vars);
 
 		// Don't add non-existent values (note:  different from NULL); this is consistent with existing selection syntax which doesn't force the appearnance of non-existent fields.
-		if (value === undefined) continue;
+		if (value === undefined || (typeof(value) == 'object' && value !== null && Object.keys(value).length === 0)) continue;
 
 		// don't add field if nothing was found in the subobject
-		if (expr2 instanceof ObjectExpression && value && value instanceof Object && Object.getOwnPropertyNames(value).length === 0) continue;
+		if (expr2 instanceof ObjectExpression && value && value instanceof Object && Object.getOwnPropertyNames(value) == {} ) continue;
 
-		pResult[fieldName2] = value;
+		out[fieldName2] = value;
 	}
 
-	return pResult;	//NOTE: munge returns result as a convenience
+	return out;	//NOTE: munge returns result as a convenience
 };
 
 /**
@@ -206,15 +213,27 @@ proto.getSizeHint = function getSizeHint(){
 	return Object.getOwnPropertyNames(this._expressions).length + (this.excludeId ? 0 : 1);
 };
 
+
+proto.evaluateDocument = function evaluateDocument(vars) {
+	var out = {};
+	this.addToDocument(out, {}, vars);
+	return out;
+};
+
+proto.evaluateInternal = function evaluateInternal(vars) {
+	return this.evaluateDocument(vars);
+};
+
+
 /**
  * Add a field to the document expression.
  * @method addField
  * @param fieldPath the path the evaluated expression will have in the result Document
- * @param pExpression the expression to evaluate obtain this field's Value in the result Document
+ * @param pExpression the expression to evaluateInternal obtain this field's Value in the result Document
  **/
 proto.addField = function addField(fieldPath, pExpression){
 	if(!(fieldPath instanceof FieldPath)) fieldPath = new FieldPath(fieldPath);
-	var fieldPart = fieldPath.fields[0],
+	var fieldPart = fieldPath.getFieldName(0),
 		haveExpr = this._expressions.hasOwnProperty(fieldPart),
 		subObj = this._expressions[fieldPart];	// inserts if !haveExpr //NOTE: not in munge & JS it doesn't, handled manually below
 
@@ -245,7 +264,7 @@ proto.addField = function addField(fieldPath, pExpression){
 		return;
 	}
 
-	if (!haveExpr) subObj = this._expressions[fieldPart] = new ObjectExpression();
+	if (!haveExpr) subObj = this._expressions[fieldPart] = new ObjectExpression(false);
 
 	subObj.addField(fieldPath.tail(), pExpression);
 };
@@ -259,9 +278,33 @@ proto.addField = function addField(fieldPath, pExpression){
  * @param fieldPath the name of the field to be included
  **/
 proto.includePath = function includePath(path){
-	this.addField(path, undefined);
+	this.addField(path, null);
 };
 
+
+proto.serialize = function serialize(explain) {
+	var valBuilder = {};
+
+	if(this._excludeId) {
+		valBuilder._id = false;
+	}
+
+	for(var ii = 0; ii < this._order.length; ii ++) {
+		var fieldName = this._order[ii],
+			expr = this._expressions[fieldName];
+
+		if(expr === undefined || expr === null) {
+			valBuilder[fieldName] = {$const:expr};
+		} else {
+			valBuilder[fieldName] = expr.serialize(explain);
+		}
+
+	}
+	return valBuilder;
+};
+
+
+
 /**
  * Get a count of the added fields.
  * @method getFieldCount

lib/pipeline/expressions/OrExpression.js

@@ -1,8 +1,8 @@
 "use strict";
 
-/** 
- * An $or pipeline expression. 
- * @see evaluate 
+/**
+ * An $or pipeline expression.
+ * @see evaluateInternal
  * @class OrExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
@@ -16,20 +16,21 @@ var OrExpression = module.exports = function OrExpression(){
 // DEPENDENCIES
 var Value = require("../Value"),
 	ConstantExpression = require("./ConstantExpression"),
-	CoerceToBoolExpression = require("./CoerceToBoolExpression");
+	CoerceToBoolExpression = require("./CoerceToBoolExpression"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$or";
 };
 
-/** 
- * Takes an array of one or more values and returns true if any of the values in the array are true. Otherwise $or returns false. 
- * @method evaluate
+/**
+ * Takes an array of one or more values and returns true if any of the values in the array are true. Otherwise $or returns false.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars){
 	for(var i = 0, n = this.operands.length; i < n; ++i){
-		var value = this.operands[i].evaluate(doc);
+		var value = this.operands[i].evaluateInternal(vars);
 		if (Value.coerceToBool(value)) return true;
 	}
 	return false;
@@ -50,7 +51,7 @@ proto.optimize = function optimize() {
 	if (!(pLast instanceof ConstantExpression)) return pE;
 
 	// Evaluate and coerce the last argument to a boolean.  If it's true, then we can replace this entire expression.
-	var last = Value.coerceToBool(pLast.evaluate());
+	var last = Value.coerceToBool();
 	if (last) return new ConstantExpression(true);
 
 	// If we got here, the final operand was false, so we don't need it anymore.
@@ -62,6 +63,5 @@ proto.optimize = function optimize() {
 	return pE;
 };
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
-};
+/** Register Expression */
+Expression.registerExpression("$or", base.parse(OrExpression));

lib/pipeline/expressions/SecondExpression.js

@@ -1,36 +1,42 @@
 "use strict";
 
-/** 
- * An $second pipeline expression. 
- * @see evaluate 
+/**
+ * An $second pipeline expression.
+ * @see evaluateInternal
  * @class SecondExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var SecondExpression = module.exports = function SecondExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var SecondExpression = module.exports = function SecondExpression() {
+	this.nargs = 1;
 	base.call(this);
-}, klass = SecondExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = SecondExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$second";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the second between 0 and 59, but can be 60 to account for leap seconds.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
-	return date.getUTCSeconds();	//TODO: incorrect for last second of leap year, need to fix...
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
+	return date.getUTCSeconds(); //TODO: incorrect for last second of leap year, need to fix...
 	// currently leap seconds are unsupported in v8
 	// http://code.google.com/p/v8/issues/detail?id=1944
 };
+
+/** Register Expression */
+Expression.registerExpression("$second", base.parse(SecondExpression));

lib/pipeline/expressions/SetDifferenceExpression.js

@@ -0,0 +1,52 @@
+"use strict";
+
+/**
+ * A $setdifference pipeline expression.
+ * @see evaluateInternal
+ * @class SetDifferenceExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetDifferenceExpression = module.exports = function SetDifferenceExpression() {
+	this.nargs = 2;
+	base.call(this);
+}, klass = SetDifferenceExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setdifference";
+};
+
+/**
+ * Takes 2 arrays. Assigns the second array to the first array.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var array1 = this.operands[0].evaluateInternal(vars),
+		array2 = this.operands[1].evaluateInternal(vars);
+	if (array1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an array");
+	if (array2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an array");
+
+	var returnVec = [];
+
+	array1.forEach(function(key) {
+		if (-1 === array2.indexOf(key)) {
+			returnVec.push(key);
+		}
+	}, this);
+	return returnVec;
+};
+
+/** Register Expression */
+Expression.registerExpression("$setdifference", base.parse(SetDifferenceExpression));

lib/pipeline/expressions/SetEqualsExpression.js

@@ -0,0 +1,45 @@
+"use strict";
+
+/**
+ * A $setequals pipeline expression.
+ * @see evaluateInternal
+ * @class SetEqualsExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetEqualsExpression = module.exports = function SetEqualsExpression() {
+	this.nargs = 2;
+	base.call(this);
+}, klass = SetEqualsExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setequals";
+};
+
+/**
+ * Takes 2 arrays. Assigns the second array to the first array.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var array1 = this.operands[0].evaluateInternal(vars),
+		array2 = this.operands[1].evaluateInternal(vars);
+	if (array1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an array");
+	if (array2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an array");
+	array1 = array2;
+	return array1;
+};
+
+/** Register Expression */
+Expression.registerExpression("$setequals", base.parse(SetEqualsExpression));

lib/pipeline/expressions/SetIntersectionExpression.js

@@ -0,0 +1,47 @@
+"use strict";
+
+/**
+ * A $setintersection pipeline expression.
+ * @see evaluateInternal
+ * @class SetIntersectionExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetIntersectionExpression = module.exports = function SetIntersectionExpression() {
+	this.nargs = 2;
+	base.call(this);
+}, klass = SetIntersectionExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setIntersection";
+};
+
+/**
+ * Takes 2 objects. Returns the intersects of the objects.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var object1 = this.operands[0].evaluateInternal(vars),
+		object2 = this.operands[1].evaluateInternal(vars);
+	if (object1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an object");
+	if (object2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an object");
+
+	var result = object1.filter(function(n) {
+		return object2.indexOf(n) > -1;
+	});
+};
+
+/** Register Expression */
+Expression.registerExpression("$setIntersection", base.parse(SetIntersectionExpression));

lib/pipeline/expressions/SetIsSubsetExpression.js

@@ -0,0 +1,88 @@
+"use strict";
+
+/**
+ * A $setissubset pipeline expression.
+ * @see evaluateInternal
+ * @class SetIsSubsetExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetIsSubsetExpression = module.exports = function SetIsSubsetExpression() {
+	this.nargs = 2;
+	if (arguments.length !== 2) throw new Error("two args expected");
+	base.call(this);
+}, klass = SetIsSubsetExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setissubset";
+};
+
+proto.optimize = function optimize(cachedRhsSet, operands) {
+
+// This optimize needs to be done, eventually
+
+// // perfore basic optimizations
+//     intrusive_ptr<Expression> optimized = ExpressionNary::optimize();
+
+//     // if ExpressionNary::optimize() created a new value, return it directly
+//     if (optimized.get() != this)
+//         return optimized;
+
+//     if (ExpressionConstant* ec = dynamic_cast<ExpressionConstant*>(vpOperand[1].get())) {
+//         const Value rhs = ec->getValue();
+//         uassert(17311, str::stream() << "both operands of $setIsSubset must be arrays. Second "
+//                                      << "argument is of type: " << typeName(rhs.getType()),
+//                 rhs.getType() == Array);
+
+//         return new Optimized(arrayToSet(rhs), vpOperand);
+//     }
+
+//     return optimized;
+
+};
+
+/**
+ * Takes 2 arrays. Assigns the second array to the first array.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var array1 = this.operands[0].evaluateInternal(vars),
+		array2 = this.operands[1].evaluateInternal(vars);
+	if (array1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an array");
+	if (array2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an array");
+
+	var sizeOfArray1 = array1.length;
+	var sizeOfArray2 = array2.length;
+	var outerLoop = 0;
+	var innerLoop = 0;
+	for (outerLoop = 0; outerLoop < sizeOfArray1; outerLoop++) {
+		for (innerLoop = 0; innerLoop < sizeOfArray2; innerLoop++) {
+			if (array2[outerLoop] == array1[innerLoop])
+				break;
+		}
+
+		/* If the above inner loop was not broken at all then
+		 array2[i] is not present in array1[] */
+		if (innerLoop == sizeOfArray2)
+			return false;
+	}
+
+	/* If we reach here then all elements of array2[]
+	 are present in array1[] */
+	return true;
+};
+
+/** Register Expression */
+Expression.registerExpression("$setissubset", base.parse(SetIsSubsetExpression));

lib/pipeline/expressions/SetUnionExpression.js

@@ -0,0 +1,53 @@
+"use strict";
+
+/**
+ * A $setunion pipeline expression.
+ * @see evaluateInternal
+ * @class SetUnionExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetUnionExpression = module.exports = function SetUnionExpression() {
+	this.nargs = 2;
+	base.call(this);
+}, klass = SetUnionExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setUnion";
+};
+
+/**
+ * Takes 2 objects. Unions the objects
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var object1 = this.operands[0].evaluateInternal(vars),
+		object2 = this.operands[1].evaluateInternal(vars);
+	if (object1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an object");
+	if (object2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an object");
+
+	var object3 = {};
+	for (var attrname1 in object1) {
+		object3[attrname1] = object1[attrname1];
+	}
+	for (var attrname2 in object2) {
+		object3[attrname2] = object2[attrname2];
+	}
+
+	return object3;
+};
+
+/** Register Expression */
+Expression.registerExpression("$setUnion", base.parse(SetUnionExpression));

lib/pipeline/expressions/SizeExpression.js

@@ -0,0 +1,41 @@
+"use strict";
+
+/**
+ * A $size pipeline expression.
+ * @see evaluateInternal
+ * @class SizeExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SizeExpression = module.exports = function SizeExpression() {
+	this.nargs = 1;
+	base.call(this);
+}, klass = SizeExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$size";
+};
+
+/**
+ * Takes an array and return the size.
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var array = this.operands[0].evaluateInternal(vars);
+	if (array instanceof Date) throw new Error("$size does not support dates; code 16376");
+	return array.length;
+};
+
+/** Register Expression */
+Expression.registerExpression("$size", base.parse(SizeExpression));

lib/pipeline/expressions/StrcasecmpExpression.js

@@ -1,42 +1,46 @@
 "use strict";
 
-/** 
+/**
  * A $strcasecmp pipeline expression.
- * @see evaluate 
+ * @see evaluate
  * @class StrcasecmpExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var StrcasecmpExpression = module.exports = function StrcasecmpExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var StrcasecmpExpression = module.exports = function StrcasecmpExpression() {
+	this.nargs = 2;
 	base.call(this);
-}, klass = StrcasecmpExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = StrcasecmpExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
 var Value = require("../Value"),
-	NaryExpression = require("./NaryExpression");
+	NaryExpression = require("./NaryExpression"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$strcasecmp";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Takes in two strings. Returns a number. $strcasecmp is positive if the first string is “greater than” the second and negative if the first string is “less than” the second. $strcasecmp returns 0 if the strings are identical. 
+/**
+ * Takes in two strings. Returns a number. $strcasecmp is positive if the first string is “greater than” the second and negative if the first string is “less than” the second. $strcasecmp returns 0 if the strings are identical.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(2);
-	var val1 = this.operands[0].evaluate(doc),
-		val2 = this.operands[1].evaluate(doc),
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var val1 = this.operands[0].evaluateInternal(vars),
+		val2 = this.operands[1].evaluateInternal(vars),
 		str1 = Value.coerceToString(val1).toUpperCase(),
 		str2 = Value.coerceToString(val2).toUpperCase(),
 		cmp = Value.compare(str1, str2);
 	return cmp;
 };
+
+/** Register Expression */
+Expression.registerExpression("$strcasecmp", base.parse(StrcasecmpExpression));

lib/pipeline/expressions/SubstrExpression.js

@@ -2,39 +2,40 @@
 
 /**
  * A $substr pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class SubstrExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var SubstrExpression = module.exports = function SubstrExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var SubstrExpression = module.exports = function SubstrExpression() {
+	this.nargs = 3;
 	base.call(this);
-}, klass = SubstrExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = SubstrExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$substr";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(3);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a string and two numbers. The first number represents the number of bytes in the string to skip, and the second number specifies the number of bytes to return from the string.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(3);
-	var val = this.operands[0].evaluate(doc),
-		idx = this.operands[1].evaluate(doc),
-		len = this.operands[2].evaluate(doc),
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var val = this.operands[0].evaluateInternal(vars),
+		idx = this.operands[1].evaluateInternal(vars),
+		len = this.operands[2].evaluateInternal(vars),
 		str = Value.coerceToString(val);
 	if (typeof(idx) != "number") throw new Error(this.getOpName() + ": starting index must be a numeric type; code 16034");
 	if (typeof(len) != "number") throw new Error(this.getOpName() + ": length must be a numeric type; code 16035");
@@ -43,3 +44,6 @@ proto.evaluate = function evaluate(doc) {
 	len = (len === -1 ? undefined : len);
 	return str.substr(idx, len);
 };
+
+/** Register Expression */
+Expression.registerExpression("$substr", base.parse(SubstrExpression));

lib/pipeline/expressions/SubtractExpression.js

@@ -1,37 +1,42 @@
 "use strict";
 
-/** 
+/**
  * A $subtract pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class SubtractExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var SubtractExpression = module.exports = function SubtractExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+	this.nargs = 2;
 	base.call(this);
-}, klass = SubtractExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = SubtractExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$subtract";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
-* Takes an array that contains a pair of numbers and subtracts the second from the first, returning their difference. 
+/**
+* Takes an array that contains a pair of numbers and subtracts the second from the first, returning their difference.
 **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc),
-		right = this.operands[1].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var left = this.operands[0].evaluateInternal(vars),
+		right = this.operands[1].evaluateInternal(vars);
+	if (left instanceof Date || right instanceof Date) throw new Error("$subtract does not support dates; code 16376");
 	return left - right;
 };
+
+/** Register Expression */
+Expression.registerExpression("$subtract", base.parse(SubtractExpression));

lib/pipeline/expressions/ToLowerExpression.js

@@ -1,37 +1,41 @@
 "use strict";
-	
-/** 
+
+/**
  * A $toLower pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class ToLowerExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var ToLowerExpression = module.exports = function ToLowerExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var ToLowerExpression = module.exports = function ToLowerExpression() {
+	this.nargs = 1;
 	base.call(this);
-}, klass = ToLowerExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = ToLowerExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$toLower";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
-* Takes a single string and converts that string to lowercase, returning the result. All uppercase letters become lowercase. 
-**/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(1);
-	var val = this.operands[0].evaluate(doc),
+/**
+ * Takes a single string and converts that string to lowercase, returning the result. All uppercase letters become lowercase.
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var val = this.operands[0].evaluateInternal(vars),
 		str = Value.coerceToString(val);
 	return str.toLowerCase();
 };
+
+/** Register Expression */
+Expression.registerExpression("$toLower", base.parse(ToLowerExpression));

lib/pipeline/expressions/ToUpperExpression.js

@@ -1,37 +1,41 @@
 "use strict";
 
-/** 
+/**
  * A $toUpper pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class ToUpperExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var ToUpperExpression = module.exports = function ToUpperExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var ToUpperExpression = module.exports = function ToUpperExpression() {
+	this.nargs = 1;
 	base.call(this);
-}, klass = ToUpperExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = ToUpperExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$toUpper";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
-* Takes a single string and converts that string to lowercase, returning the result. All uppercase letters become lowercase. 
-**/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(1);
-	var val = this.operands[0].evaluate(doc),
+/**
+ * Takes a single string and converts that string to lowercase, returning the result. All uppercase letters become lowercase.
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var val = this.operands[0].evaluateInternal(vars),
 		str = Value.coerceToString(val);
 	return str.toUpperCase();
 };
+
+/** Register Expression */
+Expression.registerExpression("$toUpper", base.parse(ToUpperExpression));

lib/pipeline/expressions/Variables.js

@@ -0,0 +1,126 @@
+"use strict";
+
+/**
+ * Class that stores/tracks variables
+ * @class Variables
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var Variables = module.exports = function Variables(numVars, root){
+	if(numVars) {
+		if(typeof numVars !== 'number') {
+			throw new Error('numVars must be a number');
+		}
+	}
+	this._root = root || {};
+	this._rest = numVars ? [] : undefined; //An array of `Value`s
+	this._numVars = numVars;
+}, klass = Variables,
+	base = Object,
+	proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+klass.ROOT_ID = -1;
+
+// PROTOTYPE MEMBERS
+
+/**
+ * Sets the root variable
+ * @method setRoot
+ * @parameter root {Document} The root variable
+ **/
+proto.setRoot = function setRoot(root){
+	if(!(root instanceof Object && root.constructor.name === 'Object')) { //NOTE: Type checking cause c++ does this for you
+		throw new Error('root must be an Object');
+	}
+	this._root = root;
+};
+
+/**
+ * Clears the root variable
+ * @method clearRoot
+ **/
+proto.clearRoot = function clearRoot(){
+	this._root = {};
+};
+
+/**
+ * Gets the root variable
+ * @method getRoot
+ * @return {Document} the root variable
+ **/
+proto.getRoot = function getRoot(){
+	return this._root;
+};
+
+/**
+ * Inserts a value with the given id
+ * @method setValue
+ * @param id {Number} The index where the value is stored in the _rest Array
+ * @param value {Value} The value to store
+ **/
+proto.setValue = function setValue(id, value) {
+	//NOTE: Some added type enforcement cause c++ does this for you
+	if(typeof id !== 'number') {
+		throw new Error('id must be a Number');
+	}
+
+	if(id === klass.ROOT_ID) {
+		throw new Error("mError 17199: can't use Variables#setValue to set ROOT");
+	}
+	if(id >= this._numVars) { // a > comparator would be off-by-one; i.e. if we have 5 vars, the max id would be 4
+		throw new Error("You have more variables than _numVars");
+	}
+
+	this._rest[id] = value;
+};
+
+/**
+ * Get the value at the given id
+ * @method getValue
+ * @param id {Number} The index where the value was stored
+ * @return {Value} The value
+ **/
+proto.getValue = function getValue(id) {
+	//NOTE: Some added type enforcement cause c++ does this for you
+	if(typeof id !== 'number') {
+		throw new Error('id must be a Number');
+	}
+
+	if(id === klass.ROOT_ID) {
+		return this._root;
+	}
+	if(id >= this._numVars) { // a > comparator would be off-by-one; i.e. if we have 5 vars, the max id would be 4
+		throw new Error("Cannot get value; id was greater than _numVars");
+	}
+
+	return this._rest[id];
+};
+
+
+/**
+ * Get the value for id if it's a document
+ * @method getDocument
+ * @param id {Number} The index where the document was stored
+ * @return {Object} The document
+ **/
+proto.getDocument = function getDocument(id) {
+	//NOTE: Some added type enforcement cause c++ does this for you
+	if(typeof id !== 'number') {
+		throw new Error('id must be a Number');
+	}
+
+	if(id === klass.ROOT_ID) {
+		return this._root;
+	}
+	if(id >= this._numVars) { // a > comparator would be off-by-one; i.e. if we have 5 vars, the max id would be 4
+		throw new Error("Cannot get value; id was greater than _numVars");
+	}
+
+	var value = this._rest[id];
+	if(typeof value === 'object' && value.constructor.name === 'Object') {
+		return value;
+	}
+	return {};
+};

lib/pipeline/expressions/VariablesIdGenerator.js

@@ -0,0 +1,31 @@
+"use strict";
+
+/** 
+ * Class generates unused ids
+ * @class VariablesIdGenerator
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var VariablesIdGenerator = module.exports = function VariablesIdGenerator(){
+	this._nextId = 0;
+}, klass = VariablesIdGenerator, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ * Gets the next unused id
+ * @method generateId
+ * @return {Number} The unused id
+ **/
+proto.generateId = function generateId() {
+	return this._nextId++;
+};
+
+/**
+ * Gets the number of used ids
+ * @method getIdCount
+ * @return {Number} The number of used ids
+ **/
+proto.getIdCount = function getIdCount() {
+	return this._nextId;
+};
+

lib/pipeline/expressions/VariablesParseState.js

@@ -0,0 +1,55 @@
+"use strict";
+
+/** 
+ * Class generates unused ids
+ * @class VariablesParseState
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var Variables = require('./Variables'),
+	VariablesIdGenerator = require('./VariablesIdGenerator');
+
+var VariablesParseState = module.exports = function VariablesParseState(idGenerator){
+	if(!idGenerator || idGenerator.constructor !== VariablesIdGenerator) {
+		throw new Error("idGenerator is required and must be of type VariablesIdGenerator");
+	}
+	this._idGenerator = idGenerator;
+	this._variables = {}; //Note: The c++ type was StringMap<Variables::Id>
+}, klass = VariablesParseState, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ * Assigns a named variable a unique Id. This differs from all other variables, even
+ * others with the same name.
+ *
+ * The special variables ROOT and CURRENT are always implicitly defined with CURRENT
+ * equivalent to ROOT. If CURRENT is explicitly defined by a call to this function, it
+ * breaks that equivalence.
+ *
+ * NOTE: Name validation is responsibility of caller.
+ **/
+proto.defineVariable = function generateId(name) {
+	// caller should have validated before hand by using Variables::uassertValidNameForUserWrite
+	if(name === 'ROOT') {
+		throw new Error("mError 17275: Can't redefine ROOT");
+	}
+	var id = this._idGenerator.generateId();
+	this._variables[name] = id;
+	return id;
+};
+
+/**
+ * Returns the current Id for a variable. uasserts if the variable isn't defined.
+ * @method getVariable
+ * @param name {String} The name of the variable
+ **/
+proto.getVariable = function getIdCount(name) {
+	var found = this._variables[name];
+	if(typeof found === 'number') return found;
+	if(name !== "ROOT" && name !== "CURRENT") {
+		throw new Error("uError 17276: Use of undefined variable " + name);
+	}
+
+	return Variables.ROOT_ID;
+};
+

lib/pipeline/expressions/WeekExpression.js

@@ -1,46 +1,50 @@
 "use strict";
 
-/** 
+/**
  * A $week pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class WeekExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var WeekExpression = module.exports = function WeekExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var WeekExpression = module.exports = function WeekExpression() {
+	this.nargs = 1;
 	base.call(this);
-}, klass = WeekExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = WeekExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
 var Value = require("../Value"),
-	DayOfYearExpression = require("./DayOfYearExpression");
+	DayOfYearExpression = require("./DayOfYearExpression"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$week";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Takes a date and returns the week of the year as a number between 0 and 53. 
- * Weeks begin on Sundays, and week 1 begins with the first Sunday of the year. 
- * Days preceding the first Sunday of the year are in week 0. 
+/**
+ * Takes a date and returns the week of the year as a number between 0 and 53.
+ * Weeks begin on Sundays, and week 1 begins with the first Sunday of the year.
+ * Days preceding the first Sunday of the year are in week 0.
  * This behavior is the same as the “%U” operator to the strftime standard library function.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc),
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars),
 		dayOfWeek = date.getUTCDay(),
 		dayOfYear = DayOfYearExpression.getDateDayOfYear(date),
-		prevSundayDayOfYear = dayOfYear - dayOfWeek,	// may be negative
-		nextSundayDayOfYear = prevSundayDayOfYear + 7;	// must be positive
+		prevSundayDayOfYear = dayOfYear - dayOfWeek, // may be negative
+		nextSundayDayOfYear = prevSundayDayOfYear + 7; // must be positive
 	// Return the zero based index of the week of the next sunday, equal to the one based index of the week of the previous sunday, which is to be returned.
 	return (nextSundayDayOfYear / 7) | 0; // also, the `| 0` here truncates this so that we return an integer
 };
+
+/** Register Expression */
+Expression.registerExpression("$week", base.parse(WeekExpression));

lib/pipeline/expressions/YearExpression.js

@@ -1,38 +1,43 @@
 "use strict";
 
-/** 
+/**
  * A $year pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class YearExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var YearExpression = module.exports = function YearExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var YearExpression = module.exports = function YearExpression() {
+	this.nargs = 1;
 	base.call(this);
-}, klass = YearExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = YearExpression,
+	base = require("./NaryExpression"),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
 var Value = require("../Value"),
-	DayOfYearExpression = require("./DayOfYearExpression");
+	DayOfYearExpression = require("./DayOfYearExpression"),
+	Expression = require("./Expression");
+
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$year";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the full year.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCFullYear();
 };
+
+/** Register Expression */
+Expression.registerExpression("$year", base.parse(YearExpression));

lib/pipeline/matcher/AllElemMatchOp.js

@@ -0,0 +1,229 @@
+"use strict";
+
+var MatchExpression = require('./MatchExpression');
+
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var AllElemMatchOp = module.exports = function AllElemMatchOp(){
+	base.call(this);
+	this._matchType = 'ALL';
+	this._elementPath = new ElementPath();
+	this._list = [];
+}, klass = AllElemMatchOp, base =  MatchExpression , proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes,
+	ElementPath = require('./ElementPath.js');
+
+// File: expression_array.h lines: 175-175
+//         ElementPath _elementPath;
+
+proto._elementPath = undefined;
+
+
+// File: expression_array.h lines: 176-176
+//         std::vector< const ArrayMatchingMatchExpression* > _list;
+
+proto._list = undefined;
+
+
+// File: expression_array.h lines: 174-174
+//         StringData _path;
+
+proto._path = undefined;
+
+/**
+ *
+ * This method checks the input array and determines if each item inside matches
+ * @method _allMatch
+ * @param anArray
+ *
+ */
+proto._allMatch = function _allMatch( anArray ){ //  const BSONObj& anArray
+	// File: expression_array.cpp lines: 208-215
+	if(this._list.length === 0) { return false; }
+
+	for (var i = 0; i < this._list.length; i++) {
+		if( ! this._list[i].matchesArray( anArray, null ) ) { return false; }
+	}
+
+	return true;
+};
+
+
+/**
+ *
+ * This method adds a new expression to the internal array of expression
+ * @method add
+ * @param expr
+ *
+ */
+proto.add = function add( expr ){//  const ArrayMatchingMatchExpression* expr
+	// File: expression_array.cpp lines: 184-186
+	if (!expr) throw new Error("AllElemMatchOp:add#68 failed to verify expr");
+	this._list.push(expr);
+};
+
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString( level ){ //   StringBuilder& debug, int level
+	// File: expression_array.cpp lines: 219-224
+	console.debug(this._debugAddSpace(level) + this._path + " AllElemMatchOp: " + this._path + '\n');
+	for (var i = 0; i < this._list.length; i++) {
+		this._list[i].debugString(level +1);
+	}
+};
+
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent( other ){//  const MatchExpression* other
+// File: expression_array.cpp lines: 227-242
+	if (this.matchType() != other.matchType()) {
+		return false;
+	}
+
+	if( this._path != other._path ) {
+		return false;
+	}
+
+	if( this._list.length != other._list.length ) {
+		return false;
+	}
+	for (var i = 0; i < this._list.length; i++) {
+		if ( !this._list[i].equivalent( other._list[i] ) ) {
+			return false;
+		}
+	}
+	return true;
+};
+
+
+
+/**
+ *
+ * gets the specified item from the list
+ * @method getChild
+ * @param i
+ *
+ */
+proto.getChild = function getChild( i ){ //  size_t i
+// File: expression_array.h lines: 167-166
+	return this._list[i];
+};
+
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ *
+ */
+proto.init = function init( path ){ //  const StringData& path
+// File: expression_array.cpp lines: 177-181
+	this._path = path;
+	var s = this._elementPath.init( this._path );
+	this._elementPath.setTraverseLeafArray( false );
+	return s;
+};
+
+/**
+ *
+ * matches checks the input doc against the internal path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc, details){
+	// File: expression_array.cpp lines: 189-198
+	var self = this,
+		checker = function(element) {
+			if (!(element instanceof Array))
+				return false;
+
+			//var amIRoot = (element.length === 0);
+
+			if (self._allMatch(element))
+				return true;
+
+			/*
+			if (!amIRoot && details && details.needRecord() {
+				details.setElemMatchKey(element);
+			}
+			*/
+			return false;
+		};
+	return this._elementPath._matches(doc, details, checker);
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement( e ){ //  const BSONElement& e
+	// File: expression_array.cpp lines: 201-205
+	if (!(e instanceof Array)) {
+		return false;
+	}
+	return this._allMatch(e);
+};
+
+
+/**
+ *
+ * return the length of the internal array
+ * @method numChildren
+ * @param
+ *
+ */
+proto.numChildren = function numChildren( /*  */ ){
+// File: expression_array.h lines: 166-165
+	return this._list.length;
+};
+
+
+/**
+ *
+ * return the internal path
+ * @method path
+ * @param
+ *
+ */
+proto.path = function path( /*  */ ){
+// File: expression_array.h lines: 169-168
+	return this._path;
+};
+
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /*  */ ){
+// File: expression_array.h lines: 145-152
+	var e = new AllElemMatchOp();
+	e.init( this._path );
+	e._list = this._list.slice(0);
+	return e;
+};
+

lib/pipeline/matcher/AndMatchExpression.js

@@ -0,0 +1,82 @@
+"use strict";
+
+var ListOfMatchExpression = require('./ListOfMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var AndMatchExpression = module.exports = function AndMatchExpression(){
+	base.call(this);
+	this._expressions = [];
+	this._matchType = 'AND';
+}, klass = AndMatchExpression, base =  ListOfMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString( level ) { //  StringBuilder& debug, int level
+// File: expression_tree.cpp lines: 85-88
+	return this._debugAddSpace(level) + "$and\n" + this._debugList(level);
+};
+
+/**
+ *
+ * matches checks the input doc against the internal path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc, details) { //  const MatchableDocument* doc, MatchDetails* details
+	// File: expression_tree.cpp lines: 64-72
+	var tChild;
+	for (var i = 0; i < this.numChildren(); i++) {
+		tChild = this.getChild(i);
+		if (!tChild.matches(doc, details)) {
+			if (details) {
+				details.resetOutput();
+			}
+			return false;
+		}
+	}
+	return true;
+};
+
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement( e ){ //  const BSONElement& e
+	// File: expression_tree.cpp lines: 75-81
+	for (var i = 0; i < this.numChildren(); i++) {
+		if (!this.getChild(i).matchesSingleElement(e)) {
+			return false;
+		}
+	}
+	return true;
+};
+
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /*  */ ){
+// File: expression_tree.h lines: 67-72
+	var e = new AndMatchExpression();
+	for (var i = 0; i < this.numChildren(); i++) {
+		e.add(this.getChild(i).shallowClone());
+	}
+};
+

lib/pipeline/matcher/ArrayFilterEntries.js

@@ -0,0 +1,203 @@
+"use strict";
+var Value = require('../Value');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ArrayFilterEntries = module.exports = function ArrayFilterEntries(){
+	this._hasNull = false;
+	this._hasEmptyArray = false;
+	this._equalities = [];
+	this._regexes = [];
+}, klass = ArrayFilterEntries, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+// File: expression_leaf.h lines: 266-266
+proto._equalities = undefined;
+
+
+// File: expression_leaf.h lines: 265-265
+proto._hasEmptyArray = undefined;
+
+
+// File: expression_leaf.h lines: 264-264
+proto._hasNull = undefined;
+
+
+// File: expression_leaf.h lines: 267-267
+proto._regexes = undefined;
+
+
+/**
+ *
+ * Push the input expression onto the _equalities array
+ * @method addEquality
+ * @param e
+ *
+ */
+proto.addEquality = function addEquality(e) {
+	//File expression_leaf.cpp lines 369-387
+
+	if(e instanceof Object && Object.keys(e)[0][0] === '$'){
+		return {'code':'BAD_VALUE', 'desc':'cannot next $ under $in'};
+	}
+
+	if( e instanceof RegExp ) {
+		return {'code': 'BAD_VALUE', 'desc':'ArrayFilterEntries equality cannot be a regex'};
+	}
+
+	if( e === null ) {
+		this._hasNull = true;
+	}
+
+	if (e instanceof Array && e.length === 0) {
+		this._hasEmptyArray = true;
+	}
+
+	this._equalities.push( e );
+	return {'code':'OK'};
+};
+
+/**
+ *
+ * Push the input regex onto the _regexes array
+ * @method addRegex
+ * @param expr
+ *
+ */
+proto.addRegex = function addRegex(expr) {
+	// File: expression_leaf.cpp lines: 389-391
+	this._regexes.push( expr );
+	return {'code':'OK'};
+};
+
+/**
+ *
+ * Check if the input element is contained inside of _equalities
+ * @method contains
+ * @param elem
+ *
+ */
+proto.contains = function contains(elem) {
+	// File: expression_leaf.h lines: 249-248
+	for (var i = 0; i < this._equalities.length; i++) {
+		if(typeof(elem) == typeof(this._equalities[i])){
+			if(Value.compare(elem, this._equalities[i]) === 0) {
+				return true;
+			}
+		}
+	}
+	return false;
+};
+
+/**
+ *
+ * Copy our internal fields to the input
+ * @method copyTo
+ * @param toFillIn
+ *
+ */
+proto.copyTo = function copyTo(toFillIn) {
+	// File: expression_leaf.cpp lines: 407-412
+	toFillIn._hasNull = this._hasNull;
+	toFillIn._hasEmptyArray = this._hasEmptyArray;
+	toFillIn._equalities = this._equalities.slice(0); // Copy array
+	toFillIn._regexes = this._regexes.slice(0); // Copy array
+};
+
+/**
+ *
+ * Return the _equalities property
+ * @method equalities
+ *
+ */
+proto.equalities = function equalities(){
+	// File: expression_leaf.h lines: 248-247
+	return this._equalities;
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression_leaf.cpp lines: 394-404
+	if (this._hasNull != other._hasNull) {return false;}
+	if (this._regexes.length != other._regexes.length) {return false;}
+
+	for (var i = 0; i < this._regexes.length; i++) {
+		if ( !this._regexes[i].equivalent( other._regexes[i] ) ) {
+			return false;
+		}
+	}
+	return Value.compare(this._equalities, other._equalities);
+};
+
+/**
+ *
+ * Return the _hasEmptyArray property
+ * @method hasEmptyArray
+ *
+ */
+proto.hasEmptyArray = function hasEmptyArray(){
+	// File: expression_leaf.h lines: 256-255
+	return this._hasEmptyArray;
+};
+
+/**
+ *
+ * Return the _hasNull property
+ * @method hasNull
+ *
+ */
+proto.hasNull = function hasNull(){
+	// File: expression_leaf.h lines: 254-253
+	return this._hasNull;
+};
+
+/**
+ *
+ * Return the length of the _regexes property
+ * @method numRegexes
+ *
+ */
+proto.numRegexes = function numRegexes(){
+	// File: expression_leaf.h lines: 251-250
+	return this._regexes.length;
+};
+
+/**
+ *
+ * Return the regex at the given index
+ * @method regex
+ * @param idx
+ *
+ */
+proto.regex = function regex(idx) {
+	// File: expression_leaf.h lines: 252-251
+	return this._regexes[idx];
+};
+
+/**
+ *
+ * Return whether we have a single item and it is null
+ * @method singleNull
+ *
+ */
+proto.singleNull = function singleNull(){
+	// File: expression_leaf.h lines: 255-254
+	return this.size() == 1 && this._hasNull;
+};
+
+/**
+ *
+ * Return the length of both _regexes and _equalities
+ * @method size
+ *
+ */
+proto.size = function size(){
+	// File: expression_leaf.h lines: 257-256
+	return this._equalities.length + this._regexes.length;
+};
+

lib/pipeline/matcher/ArrayMatchingMatchExpression.js

@@ -0,0 +1,130 @@
+"use strict";
+
+var MatchExpression = require('./MatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ArrayMatchingMatchExpression = module.exports = function ArrayMatchingMatchExpression(matchType){
+	base.call(this);
+	this._matchType = matchType;
+	// File: expression_array.h lines: 55-55
+	this._elementPath = new ElementPath();
+}, klass = ArrayMatchingMatchExpression, base = MatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes,
+	ElementPath = require('./ElementPath.js');
+
+// File: expression_array.h lines: 54-54
+proto._path = undefined;
+
+/**
+ *
+ * Check if the input element is equivalent to us
+ * @method equivalent
+ * @param
+ *
+ */
+proto.equivalent = function equivalent(other){
+	// File: expression_array.cpp lines: 63-79
+	if ( this._matchType != other._matchType)
+		return false;
+
+	var realOther = new ArrayMatchingMatchExpression(other);
+
+	if (this._path != realOther._path)
+		return false;
+
+	if (this.numChildren() != realOther.numChildren())
+		return false;
+
+	for (var i = 0; i < this.numChildren(); i++)
+		if (!this.getChild(i).equivalent(realOther.getChild(i)))
+			return false;
+	return true;
+};
+
+/**
+ *
+ * Initialize the input path as our element path
+ * @method initPath
+ * @param path
+ *
+ */
+proto.initPath = function initPath(path){
+	// File: expression_array.cpp lines: 27-31
+	this._path = path;
+	var status = this._elementPath.init(this._path);
+	this._elementPath.setTraverseLeafArray(false);
+	return status;
+};
+
+
+
+/**
+ *
+ * matches checks the input doc against the internal path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc, details){
+	var self = this,
+		checker = function(element) {
+			// we got the whole path, now check it
+			if (!(element instanceof Array))
+				return false;
+
+			//var amIRoot = (element.length === 0);
+
+			if (!self.matchesArray(element, details))
+				return false;
+
+			/*
+			if (!amIRoot && details && details.needRecord() {
+				details.setElemMatchKey(element);
+			}
+			*/
+			return true;
+		};
+	return this._elementPath._matches(doc, details, checker);
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(element){
+	// File: expression_array.cpp lines: 56-59
+	if (!(element instanceof Array))
+		return false;
+	return this.matchesArray(element, null);
+};
+
+/**
+ *
+ * return the internal path
+ * @method path
+ * @param
+ *
+ */
+proto.path = function path(){
+	// File: expression_array.h lines: 52-51
+	return this._path;
+};
+
+/**
+ *
+ * Check if the input array matches
+ * @method path
+ * @param anArray
+ * @param details
+ *
+ */
+proto.matchesArray = function matchesArray(anArray, details){
+	throw new Error("not implemented");
+};

lib/pipeline/matcher/AtomicMatchExpression.js

@@ -0,0 +1,67 @@
+"use strict";
+var MatchExpression = require('./MatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var AtomicMatchExpression = module.exports = function AtomicMatchExpression(){
+	base.call(this);
+	this._matchType = 'ATOMIC';
+}, klass = AtomicMatchExpression, base =  MatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression.cpp lines: 48-50
+	return this._debugAddSpace( level ) + "$atomic\n";
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression.h lines: 198-199
+	return other._matchType == 'ATOMIC';
+};
+
+/**
+ *
+ * matches checks the input doc against the internal element path to see if it is a match
+ * @method matches
+ * @param doc
+ *
+ */
+proto.matches = function matches(doc) {
+	// File: expression.h lines: 184-185
+	return true;
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e) {
+	// File: expression.h lines: 188-189
+	return true;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression.h lines: 192-193
+	return new AtomicMatchExpression();
+};

lib/pipeline/matcher/ComparisonMatchExpression.js

@@ -0,0 +1,161 @@
+"use strict";
+var LeafMatchExpression = require('./LeafMatchExpression.js');
+var Value = require('../Value');
+
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ComparisonMatchExpression = module.exports = function ComparisonMatchExpression( type ){
+	base.call(this);
+	this._matchType = type;
+}, klass = ComparisonMatchExpression, base =  LeafMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+// File: expression_leaf.h lines: 88-88
+proto._rhs = undefined;
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression_leaf.cpp lines: 135-154
+	var retStr = this._debugAddSpace( level ) + this.path() + " ";
+	switch (this._matchType){
+		case 'LT':
+			retStr += '$lt';
+			break;
+		case 'LTE':
+			retStr += '$lte';
+			break;
+		case 'EQ':
+			retStr += '==';
+			break;
+		case 'GT':
+			retStr += '$gt';
+			break;
+		case 'GTE':
+			retStr += '$gte';
+			break;
+		default:
+			retStr += "Unknown comparison!";
+			break;
+	}
+
+	retStr += (this._rhs ? this._rhs.toString() : '?');
+	if ( this.getTag() ) {
+		retStr += this.getTag().debugString();
+	}
+	return retStr + '\n';
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression_leaf.cpp lines: 53-61
+	if (other._matchType != this._matchType)  return false;
+	return this.path() === other.path() && Value.compare(this._rhs,other._rhs) === 0;
+};
+
+/**
+ *
+ * Return the _rhs property
+ * @method getData
+ *
+ */
+proto.getData = function getData(){
+	// File: expression_leaf.h lines: 85-84
+	return this._rhs;
+};
+
+/**
+ *
+ * Return the _rhs property
+ * @method getRHS
+ *
+ */
+proto.getRHS = function getRHS(){
+	// File: expression_leaf.h lines: 79-78
+	return this._rhs;
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ * @param type
+ *
+ */
+proto.init = function init(path,rhs) {
+	// File: expression_leaf.cpp lines: 65-87
+	this._rhs = rhs;
+	if ( (rhs instanceof Object && Object.keys(rhs).length === 0)) { return {'code':'BAD_VALUE', 'description':'Need a real operand'};}
+
+	if ( rhs === undefined ) { return {'code':'BAD_VALUE', 'desc':'Cannot compare to undefined'};}
+	if (!(this._matchType in {"LT":1, "LTE":1, "EQ":1, "GT":1, "GTE":1})) {
+		return {'code':'BAD_VALUE', 'description':'Bad match type for ComparisonMatchExpression'};
+	}
+	return this.initPath( path );
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e){
+	// File: expression_leaf.cpp lines: 91-132
+	if( typeof(e) != typeof(this._rhs) ){
+		if( this._rhs === null) {
+			if(this._matchType in {'EQ':1, 'LTE':1, 'GTE':1}){
+				if(e === undefined || e === null || (e instanceof Object && Object.keys(e).length === 0)) {
+					return true;
+				}
+			}
+			return false;
+		}
+		if((e === null || e === undefined) && (this._rhs ===null || this._rhs === undefined)) {
+			return ["EQ","LTE","GTE"].indexOf(this._matchType) != -1;
+		}
+
+		if (this._rhs.constructor.name in {'MaxKey':1,'MinKey':1} ) {
+			return this._matchType != "EQ";
+		}
+		return false;
+	}
+
+	if( this._rhs instanceof Array) {
+		if( this._matchType != 'EQ') {
+			return false;
+		}
+	}
+
+	var x = Value.compare( e, this._rhs );
+
+	switch( this._matchType ) {
+		case "LT":
+			return x == -1;
+		case "LTE":
+			return x <= 0;
+		case "EQ":
+			return x === 0;
+		case "GT":
+			return x === 1;
+		case "GTE":
+			return x >= 0;
+		default:
+			throw new Error("Invalid comparison type evaluated.");
+	}
+	return false;
+};
+

lib/pipeline/matcher/Context.js

@@ -0,0 +1,73 @@
+"use strict";
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var Context = module.exports = function (){
+
+}, klass = Context, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// File: path.h lines: 62-62
+proto._arrayOffset = undefined;
+
+// File: path.h lines: 61-61
+proto._element = undefined;
+
+// File: path.h lines: 63-63
+proto._outerArray = undefined;
+
+/**
+ *
+ * Return the _arrayOffset property
+ * @method arrayOffset
+ *
+ */
+proto.arrayOffset = function arrayOffset( ){
+	// File: path.h lines: 57-56
+	return this._arrayOffset;
+};
+
+/**
+ *
+ * Return the _element property
+ * @method element
+ *
+ */
+proto.element = function element( ){
+	// File: path.h lines: 56-55
+	return this._element;
+};
+
+/**
+ *
+ * Return the _outerArray property
+ * @method outerArray
+ *
+ */
+proto.outerArray = function outerArray( ){
+	// File: path.h lines: 58-57
+	return this._outerArray;
+};
+
+/**
+ *
+ * Set _element to a new empty object
+ * @method reset
+ *
+ */
+proto.reset = function reset(){
+	// File: path.cpp lines: 37-38
+	this._element = {};
+};
+
+/**
+ *
+ * Set _arrayOffset property to the input element
+ * @method setArrayOffset
+ * @param e
+ *
+ */
+proto.setArrayOffset = function setArrayOffset(e){
+	// File: path.h lines: 54-53
+	this._arrayOffset = e;
+};
+
+

lib/pipeline/matcher/ElemMatchObjectMatchExpression.js

@@ -0,0 +1,97 @@
+"use strict";
+var ArrayMatchingMatchExpression = require('./ArrayMatchingMatchExpression.js');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ElemMatchObjectMatchExpression = module.exports = function ElemMatchObjectMatchExpression(){
+	base.call(this);
+}, klass = ElemMatchObjectMatchExpression, base = ArrayMatchingMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// File: expression_array.h lines: 77-77
+proto._sub = undefined;
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level){
+	// File: expression_array.cpp lines: 106-109
+	var debug = this._debugAddSpace(level);
+	debug = debug + this.path() + " $elemMatch\n";
+	return debug + this._sub.debugString(level + 1);
+};
+
+/**
+ *
+ * Return the _sub property
+ * @method getChild
+ *
+ */
+proto.getChild = function getChild(){
+	// File: expression_array.h lines: 74-73
+	return this._sub;
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ * @param type
+ *
+ */
+proto.init = function init(path, sub){
+	// File: expression_array.cpp lines: 85-87
+	this._sub = sub;
+	return this.initPath(path);
+};
+
+/**
+ *
+ * Check if one of the items in the input array matches _sub
+ * @method matchesArray
+ * @param anArray
+ * @param details
+ *
+ */
+proto.matchesArray = function matchesArray(anArray, details){
+	// File: expression_array.cpp lines: 90-103
+	for (var i in anArray) {
+		var inner = anArray[i];
+		if (!(inner instanceof Object))
+			continue;
+		if (this._sub.matchesBSON(inner, null)) {
+			if (details && details.needRecord()) {
+				details.setElemMatchKey(i);
+			}
+			return true;
+		}
+	}
+	return false;
+};
+
+/**
+ *
+ * Return 1 since we have a single "sub"
+ * @method numChildren
+ *
+ */
+proto.numChildren = function numChildren(){
+	// File: expression_array.h lines: 73-72
+	return 1;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression_array.h lines: 65-68
+	var element = new ElemMatchObjectMatchExpression();
+	element.init(this.path(), this._sub.shallowClone());
+	return element;
+};

lib/pipeline/matcher/ElemMatchValueMatchExpression.js

@@ -0,0 +1,140 @@
+"use strict";
+var ArrayMatchingMatchExpression = require('./ArrayMatchingMatchExpression.js');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ElemMatchValueMatchExpression = module.exports = function ElemMatchValueMatchExpression(){
+	base.call(this);
+	this._matchType = 'ELEM_MATCH_VALUE';
+	this._subs = [];
+}, klass = ElemMatchValueMatchExpression, base =  ArrayMatchingMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes;
+
+// File: expression_array.h lines: 108-108
+proto._subs = undefined;
+
+/**
+ *
+ * Check if the input element matches all items in the array
+ * @method _arrayElementMatchesAll
+ * @param element
+ *
+ */
+proto._arrayElementMatchesAll = function _arrayElementMatchesAll(element){
+	// File: expression_array.cpp lines: 152-157
+	for (var i = 0; i < this._subs.length; i++ ) {
+		if (!this._subs[i].matchesSingleElement(element))
+			return false;
+	}
+	return true;
+};
+
+/**
+ *
+ * push an item onto the internal array
+ * @method add
+ * @param sub
+ *
+ */
+proto.add = function add(sub){
+	// File: expression_array.cpp lines: 132-134
+	if (!sub) throw new Error(sub + " ElemMatchValueMatchExpression:36");
+	this._subs.push(sub);
+};
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level){
+	// File: expression_array.cpp lines: 160-165
+	var debug = this._debugAddSpace(level);
+	debug = debug + this.path() + " $elemMatch\n";
+	for (var i = 0; i < this._subs.length; i++) {
+		debug = debug + this._subs[i].debugString(level + 1);
+	}
+	return debug;
+};
+
+/**
+ *
+ * Get the given child in the internal array
+ * @method getChild
+ * @param i
+ *
+ */
+proto.getChild = function getChild(i){
+	// File: expression_array.h lines: 103-102
+	return this._subs[i];
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ * @param sub
+ *
+ */
+proto.init = function init(path, sub){
+	// File: expression_array.cpp lines: 121-124
+	this.initPath(path);
+	if (sub)
+		this.add(sub);
+	return {code:ErrorCodes.OK};
+};
+
+/**
+ *
+ * Check if one of the items in the input array matches everything in the internal array
+ * @method matchesArray
+ * @param anArray
+ * @param details
+ *
+ */
+proto.matchesArray = function matchesArray(anArray, details){
+	// File: expression_array.cpp lines: 137-149
+	for (var i in anArray) {
+		var inner = anArray[i];
+
+		if (this._arrayElementMatchesAll(inner)) {
+			if (details && details.needRecord()) {
+				details.setElemMatchKey(i);
+			}
+			return true;
+		}
+	}
+	return false;
+};
+
+/**
+ *
+ * Return the number of items in the internal array
+ * @method numChildren
+ *
+ */
+proto.numChildren = function numChildren(){
+	// File: expression_array.h lines: 102-101
+	return this._subs.length;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression_array.h lines: 91-97
+	var element = new ElemMatchValueMatchExpression();
+	element.init(this.path());
+	for (var i = 0; i < this._subs.length; ++i) {
+		element.add(this._subs[i].shallowClone());
+	}
+	return element;
+};

lib/pipeline/matcher/ElementPath.js

@@ -0,0 +1,232 @@
+"use strict";
+
+var FieldRef = require('./FieldRef');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ElementPath = module.exports = function ElementPath(){
+	this._fieldRef = new FieldRef();
+	this._shouldTraverseLeafArray = false;
+}, klass = ElementPath, base =  Object  , proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+// File: path.h lines: 41-41
+//         FieldRef _fieldRef;
+
+proto._fieldRef = undefined;
+
+
+// File: path.h lines: 42-42
+//         bool _shouldTraverseLeafArray;
+
+proto._shouldTraverseLeafArray = undefined;
+
+/**
+ * getFieldDottedOrArray
+ *
+ * @method getFieldDottedArray
+ * @param doc
+ * @param path
+ * @param idxPathObj This is an object with a pathID element. This allows for pass by ref in calling function.
+ * */
+klass.getFieldDottedOrArray = function getFieldDottedOrArray(doc, path, idxPathObj){
+	// File path_internal.cpp lines 31-72
+	if (path.numParts() === 0 ) { return doc; }
+
+	var res,curr = doc,
+		stop = false,
+		partNum = 0;
+	while (partNum < path.numParts() && !stop) {
+		res = curr[path.getPart( partNum)];
+		if(res == {}){
+			stop = true;
+		} else if (res instanceof Object) {
+			curr = res;
+			partNum++;
+		} else if (res instanceof Array) {
+			stop = true;
+		} else {
+			if (partNum + 1 < path.numParts() ) {
+				res = {};
+			}
+			stop = true;
+		}
+
+	}
+
+	//idxPathObj.pathID = partNum;
+	return res;
+};
+
+/**
+ * isAllDigits does what it says on the tin.
+ *
+ * @method isAllDigits
+ * @param str
+ */
+
+klass.isAllDigits = function isAllDigits ( str ){
+	// File path_internal.cpp lines 23-29
+	var digitCheck = /\D/g;
+	if (digitCheck.exec(str) === null){ return true; }
+	return false;
+};
+
+
+
+
+
+/**
+ *
+ * return the internal fieldRef object
+ * @method fieldRef
+ * @param
+ *
+ */
+proto.fieldRef = function fieldRef( /*  */ ){
+// File: path.h lines: 37-36
+	return this._fieldRef;
+};
+
+
+/**
+ *
+ * Initialize necessary items on this instance
+ * @method init
+ * @param path
+ *
+ */
+proto.init = function init( path ){  //  const StringData& path
+// File: path.cpp lines: 26-29
+	this._shouldTraverseLeafArray = true;
+	this._fieldRef.parse( path );
+	return {'code':'OK'};
+};
+
+
+/**
+ *
+ * Set whether paths should traverse leaves inside arrays
+ * @method setTraverseLeafArray
+ * @param
+ *
+ */
+proto.setTraverseLeafArray = function setTraverseLeafArray( b ){ //  bool b
+// File: path.h lines: 35-34
+	this._shouldTraverseLeafArray = b;
+};
+
+
+/**
+ *
+ * Return whether arrays should traverse leaf arrays
+ * @method shouldTraverseLeafArray
+ * @param
+ *
+ */
+proto.shouldTraverseLeafArray = function shouldTraverseLeafArray( /*  */ ){
+// File: path.h lines: 38-37
+	return this._shouldTraverseLeafArray;
+};
+
+proto.objAtPath = function objAtPath(doc) {
+	return klass.objAtPath(doc, this._fieldRef._path);
+};
+
+klass.objAtPath = function objAtPath(doc, path) {
+	if(path.length === 0) {
+		return doc;
+	}
+	if (path.length > 0 && Object.keys(doc).length === 0){
+		return {};
+	}
+	if (doc === null || doc === undefined) {
+		return doc;
+	}
+	var tpath = path.split('.');
+	return klass.objAtPath(doc[tpath[0]],tpath.slice(1).join('.'));
+};
+
+
+/**
+ *
+ * Helper to wrap our path into the static method
+ * @method _matches
+ * @param doc
+ * @param details
+ * @param function checker this function is used to check for a valid item at the end of the path
+ *
+ */
+proto._matches = function _matches(doc, details, checker) {
+	return klass._matches(doc, this._fieldRef._array, this._shouldTraverseLeafArray, details, checker);
+};
+
+/**
+ *
+ * _matches exists because we don't have pathIterators, so we need a recursive function call
+ * through the path pieces
+ * @method _matches
+ * @param doc
+ * @param path
+ * @param details
+ * @param function checker this function is used to check for a valid item at the end of the path
+ *
+ */
+klass._matches = function _matches(doc, path, shouldTraverseLeafArray, details, checker){
+	// File: expression_array.cpp lines: 34-53
+	var k, result, ii, il,
+		curr = doc,
+		item = doc;
+	for (k = 0; k < path.length; k++) {
+		if((curr instanceof Object) && (path[k] in curr)) {
+			item = curr[path[k]];
+		}
+		if(path[k].length === 0)
+			continue;
+		item = curr[path[k]];
+		if (item instanceof Object && item.constructor === Object) {
+			if (!(isNaN(parseInt(path[k], 10)))) {
+				result = checker(item[path[k]]);
+				if (result) {
+					if (details && details.needRecord())
+						details.setElemMatchKey(ii.toString());
+					return result;
+				}
+			}
+			curr = item;
+			continue;
+		} else if (item instanceof Object && item.constructor === Array) {
+			if (k == path.length-1) {
+				if ((shouldTraverseLeafArray) && (isNaN(parseInt(path[k], 10)))) {
+					for (ii = 0, il = item.length; ii < il; ii++) {
+						result = checker(item[ii]);
+						if (result) {
+							if (details && details.needRecord())
+								details.setElemMatchKey(ii.toString());
+							return result;
+						}
+					}
+					if(item.length === 0)
+						return checker({});
+					
+				}
+				curr = item;
+				break; // this is the end of the path, so check this array
+			} else if (!(isNaN(parseInt(path[k+1], 10)))) {
+				curr = item;
+				continue; // the *next* path section is an item in the array so we don't check this whole array
+			}
+			// otherwise, check each item in the array against the rest of the path
+			for(ii = 0, il = item.length; ii < il; ii++){
+				var subitem = item[ii];
+				if (subitem.constructor !== Object) continue;	// can't look for a subfield in a non-object value.
+				if (this._matches(subitem, path.slice(k), shouldTraverseLeafArray, null, checker)) { // check the item against the rest of the path
+					if (details && details.needRecord())
+						details.setElemMatchKey(ii.toString());
+					return true;
+				}
+			}
+			return false; // checked all items in the array and found no matches
+		}
+	}
+	return checker(item);
+};

lib/pipeline/matcher/EqualityMatchExpression.js

@@ -0,0 +1,24 @@
+"use strict";
+
+var ComparisonMatchExpression = require('./ComparisonMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var EqualityMatchExpression = module.exports = function EqualityMatchExpression(){
+	base.call(this);
+	this._matchType = 'EQ';
+}, klass = EqualityMatchExpression, base =  ComparisonMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Return a new instance of this class, with fields set the same as ourself
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /*  */ ){
+// File: expression_leaf.h lines: 98-101
+	var e = new EqualityMatchExpression();
+	e.init ( this.path(), this._rhs );
+	return e;
+};
+

lib/pipeline/matcher/ExistsMatchExpression.js

@@ -0,0 +1,81 @@
+"use strict";
+var LeafMatchExpression = require('./LeafMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ExistsMatchExpression = module.exports = function ExistsMatchExpression(){
+	base.call(this);
+	this._matchType = 'EXISTS';
+}, klass = ExistsMatchExpression, base =  LeafMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression_leaf.cpp lines: 286-294
+	return this._debugAddSpace( level ) + this.path() + " exists" + (this.getTag() ? " " + this.getTag().debugString() : "") + "\n";
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression_leaf.cpp lines: 297-302
+	if(this._matchType != other._matchType)	{
+		return false;
+	}
+	return this.path() == other.path();
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ * @param type
+ *
+ */
+proto.init = function init(path) {
+	// File: expression_leaf.cpp lines: 278-279
+	return this.initPath( path );
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e) {
+	// File: expression_leaf.cpp lines: 282-283
+	if(typeof(e) == 'undefined')
+		return false;
+	if(e === null)
+		return true;
+	if(typeof(e) == 'object')
+		return (Object.keys(e).length > 0);
+	else
+		return true;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression_leaf.h lines: 220-223
+	var e = new ExistsMatchExpression();
+	e.init(this.path());
+	return e;
+};
+

lib/pipeline/matcher/FalseMatchExpression.js

@@ -0,0 +1,70 @@
+"use strict";
+var MatchExpression = require('./MatchExpression');
+
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var FalseMatchExpression = module.exports = function FalseMatchExpression(){
+	base.call(this);
+	this._matchType = 'ALWAYS_FALSE';
+}, klass = FalseMatchExpression, base =  MatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression.cpp lines: 53-55
+	return this._debugAddSpace( level ) + "$false\n";
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression.h lines: 222-223
+	return other._matchType === 'ALWAYS_FALSE';
+};
+
+/**
+ *
+ * matches checks the input doc against the internal element path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc,details) {
+	// File: expression.h lines: 208-209
+	return false;
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e) {
+	// File: expression.h lines: 212-213
+	return false;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression.h lines: 216-217
+	return new FalseMatchExpression();
+};
+

lib/pipeline/matcher/FieldRef.js

@@ -0,0 +1,111 @@
+"use strict";
+
+var FieldRef = module.exports = function FieldRef (){
+	this._array = [];
+	this._path = '';
+}, klass = FieldRef, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+proto.parse = function parse( str ) {
+	this._array = str.split('.');
+	this._path = this._array.join('.');
+};
+
+proto.setPart = function setPart( i, part ) {
+	this._array[i] = part;
+	this._path = this._array.join('.');
+};
+
+proto.getPart = function getPArt( i ) {
+	return this._array[i];
+};
+
+
+proto.isPrefixOf = function isPrefixOf( other ) {
+	return ( other._path.indexOf(this.path) === 0 );
+};
+
+proto.commonPrefixSize = function commonPrefixSize ( other ) {
+	var i = 0;
+	while(other._array[i] == this._array[i]) { i++; }
+	return i;
+};
+
+proto.dottedField = function dottedField( ) {
+	var offset = 0;
+	if(arguments.length == 1){
+		offset = arguments[0];
+	}
+	return this._array.slice( offset ).join('.');
+};
+
+proto.equalsDottedField = function equalsDottedField ( other ) {
+	return this._path == other._path;
+};
+
+proto.compare = function compare( other ) {
+	return (this._path < other._path ? -1 : this._path > other._path ? 1 : 0);
+};
+
+proto.clear = function clear() {
+	this._path = '';
+	this._array = [];
+};
+
+proto.numParts = function numParts() {
+	return this._array.length;
+};
+
+proto.numReplaced = function numReplaced() {
+	throw new Error('Why?');
+};
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

lib/pipeline/matcher/GTEMatchExpression.js

@@ -0,0 +1,24 @@
+"use strict";
+
+var ComparisonMatchExpression = require('./ComparisonMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var GTEMatchExpression = module.exports = function GTEMatchExpression(){
+	base.call(this);
+	this._matchType = 'GTE';
+}, klass = GTEMatchExpression, base =  ComparisonMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Return a new instance of this class, with fields set the same as ourself
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /*  */ ){
+// File: expression_leaf.h lines: 141-144
+	var e = new GTEMatchExpression();
+	e.init( this.path(), this._rhs );
+	return e;
+};
+

lib/pipeline/matcher/GTMatchExpression.js

@@ -0,0 +1,25 @@
+"use strict";
+
+var ComparisonMatchExpression = require('./ComparisonMatchExpression.js');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var GTMatchExpression = module.exports = function GTMatchExpression(){
+	base.call(this);
+	this._matchType = 'GT';
+}, klass = GTMatchExpression, base = ComparisonMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+/**
+ *
+ * Return a new instance of this class, with fields set the same as ourself
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /* */ ){
+	// File: expression_leaf.h lines: 130-133
+	var e = new GTMatchExpression();
+	e.init( this.path(), this._rhs );
+	return e;
+};
+

lib/pipeline/matcher/InMatchExpression.js

@@ -0,0 +1,161 @@
+"use strict";
+var LeafMatchExpression = require('./LeafMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var InMatchExpression = module.exports = function InMatchExpression(){
+	base.call(this);
+	this._matchType = 'MATCH_IN';
+	this._arrayEntries = new ArrayFilterEntries();
+}, klass = InMatchExpression, base =  LeafMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes,
+	ArrayFilterEntries = require("./ArrayFilterEntries.js");
+
+// File: expression_leaf.h lines: 294-294
+proto._arrayEntries = null;
+
+/**
+ *
+ * Check if the input element matches a real element
+ * @method _matchesRealElement
+ * @param e
+ *
+ */
+proto._matchesRealElement = function _matchesRealElement(e) {
+	// File: expression_leaf.cpp lines: 422-431
+	if(this._arrayEntries.contains(e)) { // array wrapper.... so no e "in" array
+		return true;
+	}
+
+	for (var i = 0; i < this._arrayEntries.numRegexes(); i++) {
+		if(e.match && e.match(this._arrayEntries.regex(i)._regex)) {
+			return true;
+		} else if (e instanceof RegExp) {
+			if(e.toString() == this._arrayEntries.regex(i)._regex.toString()) {
+				return true;
+			}
+		}
+	}
+
+	if(typeof(e) == 'undefined') {
+		return true; // Every Set contains the Null Set, man.
+	}
+
+	return false;
+};
+
+/**
+ *
+ * Copy our array to the input array
+ * @method copyTo
+ * @param toFillIn
+ *
+ */
+proto.copyTo = function copyTo(toFillIn) {
+	// File: expression_leaf.cpp lines: 481-483
+	toFillIn.init(this.path());
+	this._arrayEntries.copyTo( toFillIn._arrayEntries );
+};
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression_leaf.cpp lines: 455-463
+	return this._debugAddSpace( level ) + this.path() + ";$in: TODO " + (this.getTag() ? this.getTag().debugString() : '') + "\n";
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression_leaf.cpp lines: 466-472
+	if ( other._matchType != 'MATCH_IN' ) {
+		return false;
+	}
+	return this.path() == other.path() && this._arrayEntries.equivalent( other._arrayEntries );
+};
+
+/**
+ *
+ * Return the _arrayEntries property
+ * @method getArrayFilterEntries
+ *
+ */
+proto.getArrayFilterEntries = function getArrayFilterEntries(){
+	// File: expression_leaf.h lines: 280-279
+	return this._arrayEntries;
+};
+
+/**
+ *
+ * Return the _arrayEntries property
+ * @method getData
+ *
+ */
+proto.getData = function getData(){
+	// File: expression_leaf.h lines: 290-289
+	return this._arrayEntries;
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ *
+ */
+proto.init = function init(path) {
+	// File: expression_leaf.cpp lines: 418-419
+	return this.initPath( path );
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e) {
+	// File: expression_leaf.cpp lines: 434-452
+	if( this._arrayEntries === null && typeof(e) == 'object' && Object.keys(e).length === 0) {
+		return true;
+	}
+	if (this._matchesRealElement( e )) {
+		return true;
+	}
+	/*if (e instanceof Array){
+		for (var i = 0; i < e.length; i++) {
+			if(this._matchesRealElement( e[i] )) {
+				return true;
+			}
+		}
+
+	}*/
+	return false;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression_leaf.cpp lines: 475-478
+	var e = new InMatchExpression();
+	this.copyTo( e );
+	return e;
+};
+

lib/pipeline/matcher/IndexKeyMatchableDocument.js

@@ -0,0 +1,58 @@
+"use strict";
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var IndexKeyMatchableDocument = module.exports = function IndexKeyMatchableDocument(pattern, doc){
+	this._pattern = pattern;
+	this._doc = doc;
+}, klass = IndexKeyMatchableDocument, base =  Object  , proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// File: matcher.cpp lines: 52-52
+proto._doc = undefined;
+
+// File: matcher.cpp lines: 51-51
+proto._pattern = undefined;
+
+// File: matcher.cpp lines: 52-52
+proto._doc = undefined;
+
+// File: matcher.cpp lines: 51-51
+proto._pattern = undefined;
+
+/**
+ *
+ * get the element at the input path
+ * @method _getElement
+ * @param path
+ *
+ */
+proto._getElement = function _getElement(path){
+	// File: matcher.cpp lines: 63-77
+	var patternElement, docElement;
+
+	for (var i in this._pattern) {
+		patternElement = this._pattern[i];
+		//verify( docIterator.more() );
+		if(i >= this._doc.length) throw new Error("Ran out of docs in IndexKeyMatchableDocument:35");
+		docElement = this._doc[i];
+
+		if (path.equalsDottedField(patternElement.fieldName())) {
+			return docElement;
+		}
+	}
+
+	return {};
+};
+
+/**
+ *
+ * This method returns a JSON representation of the Document
+ * @method toBSON
+ * @param
+ *
+ */
+proto.toJSON = function toJSON(){
+	// File: matcher.cpp lines: 39-42
+	// TODO: this isn't quite correct because of dots
+	// don't think it'll ever be called though
+	return this._doc.replaceFieldNames(this._pattern);
+};

lib/pipeline/matcher/LTEMatchExpression.js

@@ -0,0 +1,24 @@
+"use strict";
+
+var ComparisonMatchExpression = require('./ComparisonMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var LTEMatchExpression = module.exports = function LTEMatchExpression(){
+	base.call(this);
+	this._matchType = 'LTE';
+}, klass = LTEMatchExpression, base = ComparisonMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Return a new instance of this class, with fields set the same as ourself
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /* */ ){
+	// File: expression_leaf.h lines: 108-111
+	var e = new LTEMatchExpression();
+	e.init( this.path(), this._rhs );
+	return e;
+};
+

lib/pipeline/matcher/LTMatchExpression.js

@@ -0,0 +1,24 @@
+"use strict";
+
+var ComparisonMatchExpression = require('./ComparisonMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var LTMatchExpression = module.exports = function LTMatchExpression(){
+	base.call(this);
+	this._matchType = 'LT';
+}, klass = LTMatchExpression, base = ComparisonMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Return a new instance of this class, with fields set the same as ourself
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /* */ ){
+	// File: expression_leaf.h lines: 119-122
+	var e = new LTMatchExpression();
+	e.init( this.path(), this._rhs );
+	return e;
+};
+

lib/pipeline/matcher/LeafMatchExpression.js

@@ -0,0 +1,107 @@
+"use strict";
+
+var MatchExpression = require('./MatchExpression');
+var ElementPath = require('./ElementPath');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var LeafMatchExpression = module.exports = function LeafMatchExpression( type ){
+	base.call(this);
+	this._matchType = type;
+	this._elementPath = new ElementPath();
+}, klass = LeafMatchExpression, base =  MatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+// File: expression_leaf.h lines: 63-63
+//         ElementPath _elementPath;
+
+proto._elementPath = undefined;
+
+
+// File: expression_leaf.h lines: 62-62
+//         StringData _path;
+
+proto._path = undefined;
+
+
+/**
+ *
+ * Initialize the ElementPath to the input path
+ * @method initPath
+ * @param path
+ *
+ */
+proto.initPath = function initPath( path ) { //  const StringData& path
+// File: expression_leaf.cpp lines: 31-33
+	this._path = path;
+	if(this._elementPath === undefined){
+		this._elementPath = new ElementPath();
+	}
+	return this._elementPath.init( this._path );
+};
+
+
+/**
+ *
+ * Check whether the input doc matches
+ * @method matches
+ * @param doc
+ *
+ */
+proto.matches = function matches( doc, details ) { //  const MatchableDocument* doc, MatchDetails* details
+	// File: expression_leaf.cpp lines: 37-48
+	var self = this,
+		checker = function(element) {
+			/*if (element instanceof Array) {
+				for (var i = 0; i < element.length; i++) {
+					if(self.matchesSingleElement(element[i])) {
+						if(details && details.needRecord()) {
+							details.setElemMatchKey(i.toString());
+						}
+						return true;
+					}
+				}
+				return false;
+			}*/
+			if (!self.matchesSingleElement(element)) {
+				return false;
+			}
+			/*
+			if( details && details.needRecord() && (element instanceof Array)) {
+				details.setElemMatchKey( docKeys[i+1] );
+			}
+			*/
+			return true;
+		};
+	return this._elementPath._matches(doc, details, checker);
+	/*
+	var tDoc = ElementPath.objAtPath( doc, this._path );
+	if(tDoc instanceof RegExp || typeof(tDoc) != 'object') {
+		return this.matchesSingleElement(tDoc);
+	}
+		if(!this.matchesSingleElement( tDoc[docKeys[i]] ))
+			continue;
+		if( details && details.needRecord() && (tDoc instanceof Array) && i < docKeys.length-1 ) {
+			console.log('test2');
+			details.setElemMatchKey( docKeys[i+1] );
+		}
+		return true;
+	}
+	return false;
+	*/
+};
+
+
+
+
+/**
+ *
+ * Return the internal path
+ * @method path
+ * @param
+ *
+ */
+proto.path = function path( /*  */ ){
+// File: expression_leaf.h lines: 56-55
+	return this._path;
+};
+

lib/pipeline/matcher/ListOfMatchExpression.js

@@ -0,0 +1,106 @@
+"use strict";
+
+var MatchExpression = require('./MatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ListOfMatchExpression = module.exports = function ListOfMatchExpression(matchType){
+	base.call(this);
+	this._expressions = [];
+	this._matchType = matchType;
+}, klass = ListOfMatchExpression, base =  MatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// File: expression_tree.h lines: 56-56
+proto._expressions = undefined;
+
+/**
+ *
+ * Print the debug info from each expression in the list
+ * @method _debugList
+ * @param level
+ *
+ */
+proto._debugList = function _debugList(level){
+	// File: expression_tree.cpp lines: 40-42
+	for (var i = 0; i < this._expressions.length; i++ )
+		this._expressions[i].debugString(level + 1); // debug only takes level now
+};
+
+/**
+ *
+ * Append a new expression to our list
+ * @method add
+ * @param Expression
+ *
+ */
+proto.add = function add( exp ){
+	// File: expression_tree.cpp lines: 34-36
+	// verify(expression)
+	if(!exp)
+		throw new Error(exp + " failed verify on ListOfMatchExpression:34");
+	if(this._expressions) {
+		this._expressions.push(exp);
+	} else {
+		this._expressions = [exp];
+	}
+
+};
+
+/**
+ *
+ * Empty us out
+ * @method clearAndRelease
+ *
+ */
+proto.clearAndRelease = function clearAndRelease(){
+	// File: expression_tree.h lines: 45-44
+	this._expressions = []; // empty the expressions
+};
+
+/**
+ *
+ * Check if the input list is considered the same as this one
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other){
+	// File: expression_tree.cpp lines: 45-59
+	if (this._matchType != other._matchType)
+		return false;
+
+	var realOther = new ListOfMatchExpression(other);
+
+	if (this._expressions.length != realOther._expressions.length)
+		return false;
+
+	// TODO: order doesn't matter
+	for (var i = 0; i < this._expressions.length; i++ )
+		if (!this._expressions[i].equivalent(realOther._expressions[i]))
+			return false;
+
+	return true;
+};
+
+/**
+ *
+ * Get an item from the expressions
+ * @method getChild
+ * @param
+ *
+ */
+proto.getChild = function getChild(i){
+	// File: expression_tree.h lines: 48-47
+	return this._expressions[i];
+};
+
+/**
+ *
+ * Get the length of the list
+ * @method numChildren
+ * @param
+ *
+ */
+proto.numChildren = function numChildren(){
+	// File: expression_tree.h lines: 47-46
+	return this._expressions.length;
+};

lib/pipeline/matcher/MatchDetails.js

@@ -0,0 +1,122 @@
+"use strict";
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var MatchDetails = module.exports = function (){
+	// File: match_details.cpp lines: 27-29
+	this._elemMatchKeyRequested = false;
+	this.resetOutput();
+}, klass = MatchDetails, base =  Object  , proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// File: match_details.h lines: 60-60
+proto._elemMatchKey = undefined;
+
+// File: match_details.h lines: 59-59
+proto._elemMatchKeyRequested = undefined;
+
+// File: match_details.h lines: 58-58
+proto._loadedRecord = undefined;
+
+/**
+ *
+ * Return the _elemMatchKey property
+ * @method elemMatchKey
+ *
+ */
+proto.elemMatchKey = function elemMatchKey(){
+	// File: match_details.cpp lines: 41-43
+	if (!this.hasElemMatchKey()) throw new Error("no elem match key MatchDetails:29");
+	return this._elemMatchKey;
+};
+
+/**
+ *
+ * Return the _elemMatchKey property so we can check if exists
+ * @method hasElemMatchKey
+ *
+ */
+proto.hasElemMatchKey = function hasElemMatchKey(){
+	// File: match_details.cpp lines: 37-38
+	return this._elemMatchKey;
+};
+
+/**
+ *
+ * Return the _loadedRecord property
+ * @method hasLoadedRecord
+ *
+ */
+proto.hasLoadedRecord = function hasLoadedRecord(){
+	// File: match_details.h lines: 41-40
+	return this._loadedRecord;
+};
+
+/**
+ *
+ * Return the _elemMatchKeyRequested property
+ * @method needRecord
+ *
+ */
+proto.needRecord = function needRecord(){
+	// File: match_details.h lines: 45-44
+	return this._elemMatchKeyRequested;
+};
+
+/**
+ *
+ * Set the _elemMatchKeyRequested property to true
+ * @method requestElemMatchKey
+ *
+ */
+proto.requestElemMatchKey = function requestElemMatchKey(){
+	// File: match_details.h lines: 50-49
+	this._elemMatchKeyRequested = true;
+};
+
+/**
+ *
+ * Set _loadedRecord to false and _elemMatchKey to undefined
+ * @method resetOutput
+ *
+ */
+proto.resetOutput = function resetOutput(){
+	// File: match_details.cpp lines: 32-34
+	this._loadedRecord = false;
+	this._elemMatchKey = undefined;
+};
+
+/**
+ *
+ * If we request an _elemMatchKey then set it to the input
+ * @method setElemMatchKey
+ * @param elemMatchKey
+ *
+ */
+proto.setElemMatchKey = function setElemMatchKey(elemMatchKey){
+	// File: match_details.cpp lines: 46-49
+	if ( this._elemMatchKeyRequested ) {
+		this._elemMatchKey = elemMatchKey;
+	}
+};
+
+/**
+ *
+ * Set the _loadedRecord property
+ * @method setLoadedRecord
+ * @param loadedRecord
+ *
+ */
+proto.setLoadedRecord = function setLoadedRecord(loadedRecord){
+	// File: match_details.h lines: 39-38
+	this._loadedRecord = loadedRecord;
+};
+
+/**
+ *
+ * Return a string representation of ourselves
+ * @method toString
+ *
+ */
+proto.toString = function toString(){
+	// File: match_details.cpp lines: 52-57
+	return "loadedRecord: " + this._loadedRecord + " " + "elemMatchKeyRequested: " + this._elemMatchKeyRequested + " " + "elemMatchKey: " + ( this._elemMatchKey ? this._elemMatchKey : "NONE" ) + " ";
+};

lib/pipeline/matcher/MatchExpression.js

@@ -0,0 +1,172 @@
+"use strict";
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var MatchExpression = module.exports = function MatchExpression( type ){
+	this._matchType = type;
+}, klass = MatchExpression, base =  Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes;
+
+	// File: expression.h lines: 172-172
+proto._matchType = undefined;
+
+// File: expression.h lines: 173-173
+proto._tagData = undefined;
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto._debugAddSpace = function _debugAddSpace(level){
+	// File: expression.cpp lines: 37-39
+	return new Array( level + 1).join("    ");
+};
+
+/**
+ *
+ * Get our child elements
+ * @method getChild
+ *
+ */
+proto.getChild = function getChild() {
+	// File: expression.h lines: 78-77
+	throw new Error('Virtual function called.');
+};
+
+
+/**
+ *
+ * Return the _tagData property
+ * @method getTag
+ *
+ */
+proto.getTag = function getTag(){
+	// File: expression.h lines: 159-158
+	return this._tagData;
+};
+
+/**
+ *
+ * Return if our _matchType needs an array
+ * @method isArray
+ *
+ */
+proto.isArray = function isArray(){
+	// File: expression.h lines: 111-113
+	switch (this._matchType){
+		case 'SIZE':
+		case 'ALL':
+		case 'ELEM_MATCH_VALUE':
+		case 'ELEM_MATCH_OBJECT':
+			return true;
+		default:
+			return false;
+	}
+
+	return false;
+};
+
+/**
+ *
+ * Check if we do not need an array, and we are not a logical element (leaves are very emotional)
+ * @method isLeaf
+ *
+ */
+proto.isLeaf = function isLeaf(){
+	// File: expression.h lines: 124-125
+	return !this.isArray() && !this.isLogical();
+};
+
+/**
+ *
+ * Check if we are a vulcan
+ * @method isLogical
+ *
+ */
+proto.isLogical = function isLogical(){
+	// File: expression.h lines: 100-101
+	switch( this._matchType ){
+		case 'AND':
+		case 'OR':
+		case 'NOT':
+		case 'NOR':
+			return true;
+		default:
+			return false;
+	}
+	return false;
+};
+
+/**
+ *
+ * Return the _matchType property
+ * @method matchType
+ *
+ */
+proto.matchType = function matchType(){
+	// File: expression.h lines: 67-66
+	return this._matchType;
+};
+
+/**
+ *
+ * Wrapper around matches function
+ * @method matchesBSON
+ * @param
+ *
+ */
+proto.matchesBSON = function matchesBSON(doc, details){
+	// File: expression.cpp lines: 42-44
+	return this.matches(doc, details);
+};
+
+/**
+ *
+ * Return the number of children we have
+ * @method numChildren
+ *
+ */
+proto.numChildren = function numChildren( ){
+	// File: expression.h lines: 73-72
+	return 0;
+};
+
+/**
+ *
+ * Return our internal path
+ * @method path
+ *
+ */
+proto.path = function path( ){
+	// File: expression.h lines: 83-82
+	return '';
+};
+
+/**
+ *
+ * Set the _tagData property
+ * @method setTag
+ * @param data
+ *
+ */
+proto.setTag = function setTag(data){
+	// File: expression.h lines: 158-157
+	this._tagData = data;
+};
+
+/**
+ *
+ * Call the debugString method
+ * @method toString
+ *
+ */
+proto.toString = function toString(){
+	// File: expression.cpp lines: 31-34
+	return this.debugString( 0 );
+};
+

lib/pipeline/matcher/MatchExpressionParser.js

@@ -0,0 +1,735 @@
+"use strict";
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var MatchExpressionParser = module.exports = function (){
+
+}, klass = MatchExpressionParser, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes,
+	AndMatchExpression = require("./AndMatchExpression.js"),
+	MatchExpression = require("./MatchExpression.js"),
+	OrMatchExpression = require("./OrMatchExpression.js"),
+	ModMatchExpression = require("./ModMatchExpression.js"),
+	NorMatchExpression = require("./NorMatchExpression.js"),
+	NotMatchExpression = require("./NotMatchExpression.js"),
+	LTMatchExpression = require("./LTMatchExpression.js"),
+	LTEMatchExpression = require("./LTEMatchExpression.js"),
+	GTMatchExpression = require("./GTMatchExpression.js"),
+	GTEMatchExpression = require("./GTEMatchExpression.js"),
+	InMatchExpression = require("./InMatchExpression.js"),
+	SizeMatchExpression = require("./SizeMatchExpression.js"),
+	TypeMatchExpression = require("./TypeMatchExpression.js"),
+	ExistsMatchExpression = require("./ExistsMatchExpression.js"),
+	EqualityMatchExpression = require("./EqualityMatchExpression.js"),
+	ArrayMatchingMatchExpression = require("./ArrayMatchingMatchExpression.js"),
+	RegexMatchExpression = require("./RegexMatchExpression.js"),
+	FalseMatchExpression = require("./FalseMatchExpression.js"),
+	ComparisonMatchExpression = require("./ComparisonMatchExpression.js"),
+	ElemMatchValueMatchExpression = require("./ElemMatchValueMatchExpression.js"),
+	ElemMatchObjectMatchExpression = require("./ElemMatchObjectMatchExpression.js"),
+	AllElemMatchOp = require("./AllElemMatchOp.js"),
+	AtomicMatchExpression = require("./AtomicMatchExpression.js");
+
+/**
+ *
+ * Check if the input element is an expression
+ * @method _isExpressionDocument
+ * @param element
+ *
+ */
+proto._isExpressionDocument = function _isExpressionDocument(element){
+	// File: expression_parser.cpp lines: 340-355
+	if (!(element instanceof Object))
+		return false;
+
+	if (Object.keys(element).length === 0)
+		return false;
+
+	var name = Object.keys(element)[0];
+	if (name[0] != '$')
+		return false;
+
+	if ("$ref" == name)
+		return false;
+
+	return true;
+};
+
+/**
+ *
+ * Parse the input object into individual elements
+ * @method _parse
+ * @param obj
+ * @param topLevel
+ *
+ */
+proto._parse = function _parse(obj, topLevel){
+	// File: expression_parser.cpp lines: 217-319
+	var rest, temp, status, element, eq, real;
+	var root = new AndMatchExpression();
+	var objkeys = Object.keys(obj);
+	var currname, currval;
+	for (var i = 0; i < objkeys.length; i++) {
+		currname = objkeys[i];
+		currval = obj[currname];
+		if (currname[0] == '$' ) {
+			rest = currname.substr(1);
+
+			// TODO: optimize if block?
+			if ("or" == rest) {
+				if (!(currval instanceof Array))
+					return {code:ErrorCodes.BAD_VALUE, description:"$or needs an array"};
+				temp = new OrMatchExpression();
+				status = this._parseTreeList(currval, temp);
+				if (status.code != ErrorCodes.OK)
+					return status;
+				root.add(temp);
+			}
+			else if ("and" == rest) {
+				if (!(currval instanceof Array))
+					return {code:ErrorCodes.BAD_VALUE, description:"and needs an array"};
+				temp = new AndMatchExpression();
+				status = this._parseTreeList(currval, temp);
+				if (status.code != ErrorCodes.OK)
+					return status;
+				root.add(temp);
+			}
+			else if ("nor" == rest) {
+				if (!(currval instanceof Array))
+					return {code:ErrorCodes.BAD_VALUE, description:"and needs an array"};
+				temp = new NorMatchExpression();
+				status = this._parseTreeList(currval, temp);
+				if (status.code != ErrorCodes.OK)
+					return status;
+				root.add(temp);
+			}
+			else if (("atomic" == rest) || ("isolated" == rest)) {
+				if (!topLevel)
+					return {code:ErrorCodes.BAD_VALUE, description:"$atomic/$isolated has to be at the top level"};
+				if (element)
+					root.add(new AtomicMatchExpression());
+			}
+			else if ("where" == rest) {
+				/*
+				if ( !topLevel )
+					return StatusWithMatchExpression( ErrorCodes::BAD_VALUE, "$where has to be at the top level" );
+				*/
+
+				return {'code':'FAILED_TO_PARSE', 'desc':'Where unimplimented.'};
+				/*
+				status = this.expressionParserWhereCallback(element);
+				if (status.code != ErrorCodes.OK)
+					return status;
+				root.add(status.result);*/
+			}
+			else if ("comment" == rest) {
+				1+1;
+			}
+			else {
+				return {code:ErrorCodes.BAD_VALUE, description:"unknown top level operator: " + currname};
+			}
+
+			continue;
+		}
+
+		if (this._isExpressionDocument(currval)) {
+			status = this._parseSub(currname, currval, root);
+			if (status.code != ErrorCodes.OK)
+				return status;
+			continue;
+		}
+
+		if (currval instanceof RegExp) {
+			status = this._parseRegexElement(currname, currval);
+			if (status.code != ErrorCodes.OK)
+				return status;
+			root.add(status.result);
+			continue;
+		}
+
+		eq = new EqualityMatchExpression();
+		status = eq.init(currname, currval);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		root.add(eq);
+	}
+
+	if (root.numChildren() == 1) {
+		return {code:ErrorCodes.OK, result:root.getChild(0)};
+	}
+
+	return {code:ErrorCodes.OK, result:root};
+};
+
+/**
+ *
+ * Parse the $all element
+ * @method _parseAll
+ * @param name
+ * @param element
+ *
+ */
+proto._parseAll = function _parseAll(name, element){
+	// File: expression_parser.cpp lines: 512-583
+	var status, i;
+
+	if (!(element instanceof Array))
+		return {code:ErrorCodes.BAD_VALUE, description:"$all needs an array"};
+
+	var arr = element;
+	if ((arr[0] instanceof Object) && ("$elemMatch" == Object.keys(arr[0])[0])) {
+		// $all : [ { $elemMatch : {} } ... ]
+
+		var temp = new AllElemMatchOp();
+		status = temp.init(name);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		for (i = 0; i < arr.length; i++) {
+			var hopefullyElemMatchElement = arr[i];
+
+			if (!(hopefullyElemMatchElement instanceof Object)) {
+				// $all : [ { $elemMatch : ... }, 5 ]
+				return {code:ErrorCodes.BAD_VALUE, description:"$all/$elemMatch has to be consistent"};
+			}
+
+			if ("$elemMatch" != Object.keys(hopefullyElemMatchElement)[0]) {
+				// $all : [ { $elemMatch : ... }, { x : 5 } ]
+				return {code:ErrorCodes.BAD_VALUE, description:"$all/$elemMatch has to be consistent"};
+			}
+
+			status = this._parseElemMatch("", hopefullyElemMatchElement.$elemMatch ); // TODO: wrong way to do this?
+			if (status.code != ErrorCodes.OK)
+				return status;
+			temp.add(status.result);
+		}
+
+		return {code:ErrorCodes.OK, result:temp};
+	}
+
+	var myAnd = new AndMatchExpression();
+	for (i = 0; i < arr.length; i++) {
+		var e = arr[i];
+
+		if (e instanceof RegExp) {
+			var r = new RegexMatchExpression();
+			status = r.init(name, e);
+			if (status.code != ErrorCodes.OK)
+				return status;
+			myAnd.add(r);
+		}
+		else if ((e instanceof Object) && (typeof(Object.keys(e)[0] == 'string' && Object.keys(e)[0][0] == '$' ))) {
+			return {code:ErrorCodes.BAD_VALUE, description:"no $ expressions in $all"};
+		}
+		else {
+			var x = new EqualityMatchExpression();
+			status = x.init(name, e);
+			if (status.code != ErrorCodes.OK)
+				return status;
+			myAnd.add(x);
+		}
+	}
+
+	if (myAnd.numChildren() === 0) {
+		return {code:ErrorCodes.OK, result:new FalseMatchExpression()};
+	}
+
+	return {code:ErrorCodes.OK, result:myAnd};
+};
+
+/**
+ *
+ * Parse the input array and add new RegexMatchExpressions to entries
+ * @method _parseArrayFilterEntries
+ * @param entries
+ * @param theArray
+ *
+ */
+proto._parseArrayFilterEntries = function _parseArrayFilterEntries(entries, theArray){
+	// File: expression_parser.cpp lines: 445-468
+	var status, e, r;
+	for (var i = 0; i < theArray.length; i++) {
+		e = theArray[i];
+
+		if (e instanceof RegExp ) {
+			r = new RegexMatchExpression();
+			status = r.init("", e);
+			if (status.code != ErrorCodes.OK)
+				return status;
+			status = entries.addRegex(r);
+			if (status.code != ErrorCodes.OK)
+				return status;
+		}
+		else {
+			status = entries.addEquality(e);
+			if (status.code != ErrorCodes.OK)
+				return status;
+		}
+	}
+	return {code:ErrorCodes.OK};
+};
+
+/**
+ *
+ * Parse the input ComparisonMatchExpression
+ * @method _parseComparison
+ * @param name
+ * @param cmp
+ * @param element
+ *
+ */
+proto._parseComparison = function _parseComparison(name, cmp, element){
+	// File: expression_parser.cpp lines: 34-43
+	var temp = new ComparisonMatchExpression(cmp);
+
+	var status = temp.init(name, element);
+	if (status.code != ErrorCodes.OK)
+		return status;
+
+	return {code:ErrorCodes.OK, result:temp};
+};
+
+/**
+ *
+ * Parse an element match into the appropriate expression
+ * @method _parseElemMatch
+ * @param name
+ * @param element
+ *
+ */
+proto._parseElemMatch = function _parseElemMatch(name, element){
+	// File: expression_parser.cpp lines: 471-509
+	var temp, status;
+	if (!(element instanceof Object))
+		return {code:ErrorCodes.BAD_VALUE, description:"$elemMatch needs an Object"};
+
+	if (this._isExpressionDocument(element)) {
+		// value case
+
+		var theAnd = new AndMatchExpression();
+		status = this._parseSub("", element, theAnd);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		temp = new ElemMatchValueMatchExpression();
+		status = temp.init(name);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		for (var i = 0; i < theAnd.numChildren(); i++ ) {
+			temp.add(theAnd.getChild(i));
+		}
+		theAnd.clearAndRelease();
+
+		return {code:ErrorCodes.OK, result:temp};
+	}
+
+	// object case
+
+	status = this._parse(element, false);
+	if (status.code != ErrorCodes.OK)
+		return status;
+
+	temp = new ElemMatchObjectMatchExpression();
+	status = temp.init(name, status.result);
+	if (status.code != ErrorCodes.OK)
+		return status;
+
+	return {code:ErrorCodes.OK, result:temp};
+};
+
+/**
+ *
+ * Parse a ModMatchExpression
+ * @method _parseMOD
+ * @param name
+ * @param element
+ *
+ */
+proto._parseMOD = function _parseMOD(name, element){
+	// File: expression_parser.cpp lines: 360-387
+	var d,r;
+	if (!(element instanceof Array))
+		return {code:ErrorCodes.BAD_VALUE, result:"malformed mod, needs to be an array"};
+	if (element.length < 2)
+		return {code:ErrorCodes.BAD_VALUE, result:"malformed mod, not enough elements"};
+	if (element.length > 2)
+		return {code:ErrorCodes.BAD_VALUE, result:"malformed mod, too many elements"};
+	if ((typeof(element[0]) != 'number')) {
+		return {code:ErrorCodes.BAD_VALUE, result:"malformed mod, divisor not a number"};
+	} else {
+		d = element[0];
+	}
+	if (( typeof(element[1]) != 'number') ) {
+		r = 0;
+	} else {
+		r = element[1];
+	}
+
+	var temp = new ModMatchExpression();
+	var status = temp.init( name, d, r);
+	if (status.code != ErrorCodes.OK)
+		return status;
+
+	return {code:ErrorCodes.OK, result:temp};
+};
+
+/**
+ *
+ * Parse a NotMatchExpression
+ * @method _parseNot
+ * @param name
+ * @param element
+ *
+ */
+proto._parseNot = function _parseNot(name, element){
+	// File: expression_parser_tree.cpp lines: 55-91
+	var status;
+	if (element instanceof RegExp) {
+		status = this._parseRegexElement(name, element);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		var n = new NotMatchExpression();
+		status = n.init(status.result);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		return {code:ErrorCodes.OK, result:n};
+	}
+
+	if (!(element instanceof Object))
+		return {code:ErrorCodes.BAD_VALUE, result:"$not needs a regex or a document"};
+
+	if (element == {})
+		return {code:ErrorCodes.BAD_VALUE, result:"$not cannot be empty"};
+
+	var theAnd = new AndMatchExpression();
+	status = this._parseSub(name, element, theAnd);
+	if (status.code != ErrorCodes.OK)
+		return status;
+
+	// TODO: this seems arbitrary?
+	// tested in jstests/not2.js
+	for (var i = 0; i < theAnd.numChildren(); i++) {
+		if (theAnd.getChild(i).matchType == MatchExpression.REGEX) {
+			return {code:ErrorCodes.BAD_VALUE, result:"$not cannot have a regex"};
+		}
+	}
+	var theNot = new NotMatchExpression();
+	status = theNot.init(theAnd);
+	if (status.code != ErrorCodes.OK)
+		return status;
+
+	return {code:ErrorCodes.OK, result:theNot};
+};
+
+/**
+ *
+ * Parse a RegexMatchExpression
+ * @method _parseRegexDocument
+ * @param name
+ * @param doc
+ *
+ */
+proto._parseRegexDocument = function _parseRegexDocument(name, doc){
+	// File: expression_parser.cpp lines: 402-442
+	var regex = '', regexOptions = '', e;
+
+	if(doc.$regex) {
+		e = doc.$regex;
+		if(e instanceof RegExp) {
+			var str = e.toString(),
+				flagIndex = 0;
+			for (var c = str.length; c > 0; c--){
+				if (str[c] == '/') {
+					flagIndex = c;
+					break;
+				}
+			}
+			regex = (flagIndex? str : str.substr(1, flagIndex-1));
+			regexOptions = str.substr(flagIndex, str.length);
+		} else if (typeof(e) == 'string') {
+			regex = e;
+		} else {
+			return {code:ErrorCodes.BAD_VALUE, description:"$regex has to be a string"};
+		}
+	}
+
+	if(doc.$options) {
+		e = doc.$options;
+		if(typeof(e) == 'string') {
+			regexOptions = e;
+		} else {
+			return {code:ErrorCodes.BAD_VALUE, description:"$options has to be a string"};
+		}
+
+	}
+
+	var temp = new RegexMatchExpression();
+	var status = temp.init(name, regex, regexOptions);
+	if (status.code != ErrorCodes.OK)
+		return status;
+
+	return {code:ErrorCodes.OK, result:temp};
+};
+
+/**
+ *
+ * Parse an element into a RegexMatchExpression
+ * @method _parseRegexElement
+ * @param name
+ * @param element
+ *
+ */
+proto._parseRegexElement = function _parseRegexElement(name, element){
+	// File: expression_parser.cpp lines: 390-399
+	if (!(element instanceof RegExp))
+		return {code:ErrorCodes.BAD_VALUE, description:"not a regex"};
+
+	var str = element.toString(),
+		flagIndex = 0;
+	for (var c = str.length; c > 0; c--){
+		if (str[c] == '/') {
+			flagIndex = c;
+			break;
+		}
+	}
+	var regex = str.substr(1, flagIndex-1),
+		regexOptions = str.substr(flagIndex+1, str.length),
+		temp = new RegexMatchExpression(),
+		status = temp.init(name, regex, regexOptions);
+
+	if (status.code != ErrorCodes.OK)
+		return status;
+
+	return {code:ErrorCodes.OK, result:temp};
+};
+
+/**
+ *
+ * Parse a sub expression
+ * @method _parseSub
+ * @param name
+ * @param sub
+ * @param root
+ *
+ */
+proto._parseSub = function _parseSub(name, sub, root){
+	// File: expression_parser.cpp lines: 322-337
+	var subkeys = Object.keys(sub),
+		currname, currval;
+
+	for (var i = 0; i < subkeys.length; i++) {
+		currname = subkeys[i];
+		currval = sub[currname];
+		var deep = {};
+		deep[currname] = currval;
+
+		var status = this._parseSubField(sub, root, name, deep);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		if (status.result)
+			root.add(status.result);
+	}
+
+	return {code:ErrorCodes.OK, result:root};
+};
+
+/**
+ *
+ * Parse a sub expression field
+ * @method _parseSubField
+ * @param context
+ * @param andSoFar
+ * @param name
+ * @param element
+ *
+ */
+proto._parseSubField = function _parseSubField(context, andSoFar, name, element){
+	// File: expression_parser.cpp lines: 46-214
+	// TODO: these should move to getGtLtOp, or its replacement
+	var currname = Object.keys(element)[0];
+	var currval = element[currname];
+	if ("$eq" == currname)
+		return this._parseComparison(name, 'EQ', currval);
+
+	if ("$not" == currname) {
+		return this._parseNot(name, currval);
+	}
+
+	var status, temp, temp2;
+	switch (currname) {
+	case '$lt':
+		return this._parseComparison(name, 'LT', currval);
+	case '$lte':
+		return this._parseComparison(name, 'LTE', currval);
+	case '$gt':
+		return this._parseComparison(name, 'GT', currval);
+	case '$gte':
+		return this._parseComparison(name, 'GTE', currval);
+	case '$ne':
+		status = this._parseComparison(name, 'EQ', currval);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		var n = new NotMatchExpression();
+		status = n.init(status.result);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		return {code:ErrorCodes.OK, result:n};
+	case '$eq':
+		return this._parseComparison(name, 'EQ', currval);
+
+	case '$in':
+		if (!(currval instanceof Array))
+			return {code:ErrorCodes.BAD_VALUE, description:"$in needs an array"};
+		temp = new InMatchExpression();
+		status = temp.init(name);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		status = this._parseArrayFilterEntries(temp.getArrayFilterEntries(), currval);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		return {code:ErrorCodes.OK, result:temp};
+
+	case '$nin':
+		if (!(currval instanceof Array))
+			return {code:ErrorCodes.BAD_VALUE, description:"$nin needs an array"};
+		temp = new InMatchExpression();
+		status = temp.init(name);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		status = this._parseArrayFilterEntries(temp.getArrayFilterEntries(), currval);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		temp2 = new NotMatchExpression();
+		status = temp2.init(temp);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		return {code:ErrorCodes.OK, result:temp2};
+
+	case '$size':
+		var size = 0;
+		if ( typeof(currval) === 'string')
+			// matching old odd semantics
+			size = 0;
+		else if (typeof(currval) === 'number')
+			size = currval;
+		else {
+			return {code:ErrorCodes.BAD_VALUE, description:"$size needs a number"};
+		}
+
+		temp = new SizeMatchExpression();
+		status = temp.init(name, size);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		return {code:ErrorCodes.OK, result:temp};
+
+	case '$exists':
+		if (currval == {})
+			return {code:ErrorCodes.BAD_VALUE, description:"$exists can't be eoo"};
+		temp = new ExistsMatchExpression();
+		status = temp.init(name);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		if (currval)
+			return {code:ErrorCodes.OK, result:temp};
+		temp2 = new NotMatchExpression();
+		status = temp2.init(temp);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		return {code:ErrorCodes.OK, result:temp2};
+
+	case '$type':
+		if (typeof(currval) != 'number')
+			return {code:ErrorCodes.BAD_VALUE, description:"$type has to be a number"};
+		var type = currval;
+		temp = new TypeMatchExpression();
+		status = temp.init(name, type);
+		if (status.code != ErrorCodes.OK)
+			return status;
+		return {code:ErrorCodes.OK, result:temp};
+
+	case '$mod':
+		return this._parseMOD(name, currval);
+
+	case '$options':
+		// TODO: try to optimize this
+		// we have to do this since $options can be before or after a $regex
+		// but we validate here
+		for(var i = 0; i < Object.keys(context).length; i++) {
+			temp = Object.keys(context)[i];
+			if (temp == '$regex')
+				return {code:ErrorCodes.OK, result:null};
+		}
+
+		return {code:ErrorCodes.BAD_VALUE, description:"$options needs a $regex"};
+
+	case '$regex':
+		return this._parseRegexDocument(name, context);
+
+	case '$elemMatch':
+		return this._parseElemMatch(name, currval);
+
+	case '$all':
+		return this._parseAll(name, currval);
+
+	case '$geoWithin':
+	case '$geoIntersects':
+	case '$near':
+	case '$nearSphere':
+		var x = 'Temporary value until Geo fns implimented.';
+		return this.expressionParserGeoCallback(name, x, context);
+	default:
+		return {code:ErrorCodes.BAD_VALUE, description:"not handled: " + element};
+	} // end switch
+
+
+	return {code:ErrorCodes.BAD_VALUE, description:"not handled: " + element};
+};
+
+/**
+ *
+ * Parse a list of parsable elements
+ * @method _parseTreeList
+ * @param arr
+ * @param out
+ *
+ */
+proto._parseTreeList = function _parseTreeList(arr, out){
+	// File: expression_parser_tree.cpp lines: 33-52
+	if (arr.length === 0)
+		return {code:ErrorCodes.BAD_VALUE, description:"$and/$or/$nor must be a nonempty array"};
+
+	var status, element;
+	for (var i = 0; i < arr.length; i++) {
+		element = arr[i];
+
+		if (!(element instanceof Object))
+			return {code:ErrorCodes.BAD_VALUE, description:"$or/$and/$nor entries need to be full objects"};
+
+		status = this._parse(element, false);
+		if (status.code != ErrorCodes.OK)
+			return status;
+
+		out.add(status.result);
+	}
+	return {code:ErrorCodes.OK};
+};
+
+/**
+ *
+ * Wrapper for _parse
+ * @method parse
+ * @param obj
+ *
+ */
+proto.parse = function parse(obj){
+	// File: expression_parser.h lines: 40-41
+	return this._parse(obj, true);
+};

lib/pipeline/matcher/Matcher2.js

@@ -0,0 +1,326 @@
+"use strict";
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var Matcher2 = module.exports = function Matcher2(pattern, nested){
+	// File: matcher.cpp lines: 83-92
+	this._pattern = pattern;
+	this.parser = new MatchExpressionParser();
+	var result = this.parser.parse(pattern);
+	if (result.code != ErrorCodes.OK)
+		return {code:16810, description:"bad query: " + result};
+	this._expression = result.result;
+}, klass = Matcher2, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes,
+	MatchExpression = require("./MatchExpression.js"),
+	MatchExpressionParser = require("./MatchExpressionParser.js"),
+	FalseMatchExpression = require("./FalseMatchExpression.js"),
+	ComparisonMatchExpression = require("./ComparisonMatchExpression.js"),
+	InMatchExpression = require("./InMatchExpression.js"),
+	AndMatchExpression = require("./AndMatchExpression.js"),
+	OrMatchExpression = require("./OrMatchExpression.js"),
+	IndexKeyMatchableDocument = require('./IndexKeyMatchableDocument.js'),
+	ListOfMatchExpression = require('./ListOfMatchExpression.js'),
+	LeafMatchExpression = require("./LeafMatchExpression.js");
+
+// File: matcher.h lines: 82-82
+proto._expression = undefined;
+
+// File: matcher.h lines: 80-80
+proto._indexKey = undefined;
+
+// File: matcher.h lines: 79-79
+proto._pattern = undefined;
+
+// File: matcher.h lines: 84-84
+proto._spliceInfo = undefined;
+
+/**
+ *
+ * Figure out where our index is
+ * @method _spliceForIndex
+ * @param keys
+ * @param full
+ * @param spliceInfo
+ *
+ */
+proto._spliceForIndex = function _spliceForIndex(keys, full, spliceInfo){
+	// File: matcher.cpp lines: 236-380
+	var dup, i, obj, lme;
+	switch (full) {
+		case MatchExpression.ALWAYS_FALSE:
+			return new FalseMatchExpression();
+
+		case MatchExpression.GEO_NEAR:
+		case MatchExpression.NOT:
+		case MatchExpression.NOR:
+			// maybe?
+			return null;
+
+		case MatchExpression.OR:
+
+		case MatchExpression.AND:
+			dup = new ListOfMatchExpression();
+			for (i = 0; i < full.numChildren(); i++) {
+				var sub = this._spliceForIndex(keys, full.getChild(i), spliceInfo);
+				if (!sub)
+					continue;
+				if (!dup.get()) {
+					if (full.matchType() == MatchExpression.AND)
+						dup.reset(new AndMatchExpression());
+					else
+						dup.reset(new OrMatchExpression());
+				}
+				dup.add(sub);
+			}
+			if (dup.get()) {
+				if (full.matchType() == MatchExpression.OR &&  dup.numChildren() != full.numChildren()) {
+					// TODO: I think this should actuall get a list of all the fields
+					// and make sure that's the same
+					// with an $or, have to make sure its all or nothing
+					return null;
+				}
+				return dup.release();
+			}
+			return null;
+
+		case MatchExpression.EQ:
+			var cmp = new ComparisonMatchExpression(full);
+
+			if (cmp.getRHS().type() == Array) {
+				// need to convert array to an $in
+
+				if (!keys.count(cmp.path().toString()))
+					return null;
+
+				var newIn = new InMatchExpression();
+				newIn.init(cmp.path());
+
+				if (newIn.getArrayFilterEntries().addEquality(cmp.getRHS()).isOK())
+					return null;
+
+				if (cmp.getRHS().Obj().isEmpty())
+					newIn.getArrayFilterEntries().addEquality(undefined);
+
+				obj = cmp.getRHS().Obj();
+				for(i in obj) {
+					var s = newIn.getArrayFilterEntries().addEquality( obj[i].next() );
+					if (s.code != ErrorCodes.OK)
+						return null;
+				}
+
+				return newIn.release();
+			}
+			else if (cmp.getRHS().type() === null) {
+				//spliceInfo.hasNullEquality = true;
+				return null;
+			}
+			break;
+
+		case MatchExpression.LTE:
+		case MatchExpression.LT:
+		case MatchExpression.GT:
+		case MatchExpression.GTE:
+			cmp = new ComparisonMatchExpression(full);
+
+			if ( cmp.getRHS().type() === null) {
+				// null and indexes don't play nice
+				//spliceInfo.hasNullEquality = true;
+				return null;
+			}
+			break;
+
+		case MatchExpression.REGEX:
+		case MatchExpression.MOD:
+			lme = new LeafMatchExpression(full);
+			if (!keys.count(lme.path().toString()))
+				return null;
+			return lme.shallowClone();
+
+		case MatchExpression.MATCH_IN:
+			lme = new LeafMatchExpression(full);
+			if (!keys.count(lme.path().toString()))
+				return null;
+			var cloned = new InMatchExpression(lme.shallowClone());
+			if (cloned.getArrayFilterEntries().hasEmptyArray())
+				cloned.getArrayFilterEntries().addEquality(undefined);
+
+			// since { $in : [[1]] } matches [1], need to explode
+			for (i = cloned.getArrayFilterEntries().equalities().begin(); i != cloned.getArrayFilterEntries().equalities().end(); ++i) {
+				var x = cloned[i];
+				if (x.type() == Array) {
+					for(var j in x) {
+						cloned.getArrayFilterEntries().addEquality(x[j]);
+					}
+				}
+			}
+
+			return cloned;
+
+		case MatchExpression.ALL:
+			// TODO: convert to $in
+			return null;
+
+		case MatchExpression.ELEM_MATCH_OBJECT:
+		case MatchExpression.ELEM_MATCH_VALUE:
+			// future
+			return null;
+
+		case MatchExpression.GEO:
+		case MatchExpression.SIZE:
+		case MatchExpression.EXISTS:
+		case MatchExpression.NIN:
+		case MatchExpression.TYPE_OPERATOR:
+		case MatchExpression.ATOMIC:
+		case MatchExpression.WHERE:
+			// no go
+			return null;
+	}
+
+	return null;
+};
+
+/**
+ *
+ * return if our _expression property is atomic or not
+ * @method atomic
+ *
+ */
+proto.atomic = function atomic(){
+	// File: matcher.cpp lines: 120-133
+	if (!this._expression)
+		return false;
+
+	if (this._expression.matchType() == MatchExpression.ATOMIC)
+		return true;
+
+	// we only go down one level
+	for (var i = 0; i < this._expression.numChildren(); i++) {
+		if (this._expression.getChild(i).matchType() == MatchExpression.ATOMIC)
+			return true;
+	}
+
+	return false;
+};
+
+/**
+ *
+ * Return the _pattern property
+ * @method getQuery
+ *
+ */
+proto.getQuery = function getQuery(){
+	// File: matcher.h lines: 65-64
+	return this._pattern;
+};
+
+/**
+ *
+ * Check if we exist
+ * @method hasExistsFalse
+ *
+ */
+proto.hasExistsFalse = function hasExistsFalse(){
+	// File: matcher.cpp lines: 172-180
+	if (this._spliceInfo.hasNullEquality) {
+		// { a : NULL } is very dangerous as it may not got indexed in some cases
+		// so we just totally ignore
+		return true;
+	}
+
+	return this._isExistsFalse(this._expression.get(), false, this._expression.matchType() == MatchExpression.AND ? -1 : 0);
+};
+
+/**
+ *
+ * Find if we have a matching key inside us
+ * @method keyMatch
+ * @param docMatcher
+ *
+ */
+proto.keyMatch = function keyMatch(docMatcher){
+	// File: matcher.cpp lines: 199-206
+	if (!this._expression)
+		return docMatcher._expression.get() === null;
+	if (!docMatcher._expression)
+		return false;
+	if (this._spliceInfo.hasNullEquality)
+		return false;
+	return this._expression.equivalent(docMatcher._expression.get());
+};
+
+/**
+ *
+ * matches checks the input doc against the internal element path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc, details){
+	// File: matcher.cpp lines: 105-116
+	if (!this._expression)
+		return true;
+
+	if (this._indexKey == {})
+		return this._expression.matchesBSON(doc, details);
+
+	if ((doc != {}) && (Object.keys(doc)[0]))
+		return this._expression.matchesBSON(doc, details);
+
+	var mydoc = new IndexKeyMatchableDocument(this._indexKey, doc);
+	return this._expression.matches(mydoc, details);
+};
+
+/**
+ *
+ * Check if we are a simple match
+ * @method singleSimpleCriterion
+ *
+ */
+proto.singleSimpleCriterion = function singleSimpleCriterion(){
+	// File: matcher.cpp lines: 184-196
+	if (!this._expression)
+		return false;
+
+	if (this._expression.matchType() == MatchExpression.EQ)
+		return true;
+
+	if (this._expression.matchType() == MatchExpression.AND && this._expression.numChildren() == 1 && this._expression.getChild(0).matchType() == MatchExpression.EQ)
+		return true;
+
+	return false;
+};
+
+/**
+ *
+ * Wrapper around _spliceForIndex
+ * @method spliceForIndex
+ * @param key
+ * @param full
+ * @param spliceInfo
+ *
+ */
+proto.spliceForIndex = function spliceForIndex(key, full, spliceInfo){
+	// File: matcher.cpp lines: 209-217
+	var keys = [],
+		e, i;
+	for (i in key) {
+		e = key[i];
+		keys.insert(e.fieldName());
+	}
+	return this._spliceForIndex(keys, full, spliceInfo);
+};
+
+/**
+ *
+ * Convert _pattern into a string
+ * @method toString
+ *
+ */
+proto.toString = function toString(){
+	// File: matcher.h lines: 66-65
+	return this._pattern.toString();
+};