Resolve CursorDocumentSource conflict

lib/Errors.js

@@ -0,0 +1,65 @@
+"use strict";
+
+/*
+ * This file defines valid error codes used by mongo
+ * mongo/base/error_codes.err
+ **/
+
+// Error codes
+var ErrorCodes = {
+	OK                           : "OK"                           ,
+	INTERNAL_ERROR               : "INTERNAL_ERROR"               ,
+	BAD_VALUE                    : "BAD_VALUE"                    ,
+	DUPLICATE_KEY                : "DUPLICATE_KEY"                ,
+	NO_SUCH_KEY                  : "NO_SUCH_KEY"                  ,
+	GRAPH_CONTAINS_CYCLE         : "GRAPH_CONTAINS_CYCLE"         ,
+	HOST_UNREACHABLE             : "HOST_UNREACHABLE"             ,
+	HOST_NOT_FOUND               : "HOST_NOT_FOUND"               ,
+	UNKNOWN_ERROR                : "UNKNOWN_ERROR"                ,
+	FAILED_TO_PARSE              : "FAILED_TO_PARSE"              ,
+	CANNOT_MUTATE_OBJECT         : "CANNOT_MUTATE_OBJECT"         ,
+	USER_NOT_FOUND               : "USER_NOT_FOUND"               ,
+	UNSUPPORTED_FORMAT           : "UNSUPPORTED_FORMAT"           ,
+	UNAUTHORIZED                 : "UNAUTHORIZED"                 ,
+	TYPE_MISMATCH                : "TYPE_MISMATCH"                ,
+	OVERFLOW                     : "OVERFLOW"                     ,
+	INVALID_LENGTH               : "INVALID_LENGTH"               ,
+	PROTOCOL_ERROR               : "PROTOCOL_ERROR"               ,
+	AUTHENTICATION_FAILED        : "AUTHENTICATION_FAILED"        ,
+	CANNOT_REUSE_OBJECT          : "CANNOT_REUSE_OBJECT"          ,
+	ILLEGAL_OPERATION            : "ILLEGAL_OPERATION"            ,
+	EMPTY_ARRAY_OPERATION        : "EMPTY_ARRAY_OPERATION"        ,
+	INVALID_B_S_O_N              : "INVALID_B_S_O_N"              ,
+	ALREADY_INITIALIZED          : "ALREADY_INITIALIZED"          ,
+	LOCK_TIMEOUT                 : "LOCK_TIMEOUT"                 ,
+	REMOTE_VALIDATION_ERROR      : "REMOTE_VALIDATION_ERROR"      ,
+	NAMESPACE_NOT_FOUND          : "NAMESPACE_NOT_FOUND"          ,
+	INDEX_NOT_FOUND              : "INDEX_NOT_FOUND"              ,
+	PATH_NOT_VIABLE              : "PATH_NOT_VIABLE"              ,
+	NON_EXISTENT_PATH            : "NON_EXISTENT_PATH"            ,
+	INVALID_PATH                 : "INVALID_PATH"                 ,
+	ROLE_NOT_FOUND               : "ROLE_NOT_FOUND"               ,
+	ROLES_NOT_RELATED            : "ROLES_NOT_RELATED"            ,
+	PRIVILEGE_NOT_FOUND          : "PRIVILEGE_NOT_FOUND"          ,
+	CANNOT_BACKFILL_ARRAY        : "CANNOT_BACKFILL_ARRAY"        ,
+	USER_MODIFICATION_FAILED     : "USER_MODIFICATION_FAILED"     ,
+	REMOTE_CHANGE_DETECTED       : "REMOTE_CHANGE_DETECTED"       ,
+	FILE_RENAME_FAILED           : "FILE_RENAME_FAILED"           ,
+	FILE_NOT_OPEN                : "FILE_NOT_OPEN"                ,
+	FILE_STREAM_FAILED           : "FILE_STREAM_FAILED"           ,
+	CONFLICTING_UPDATE_OPERATORS : "CONFLICTING_UPDATE_OPERATORS" ,
+	FILE_ALREADY_OPEN            : "FILE_ALREADY_OPEN"            ,
+	LOG_WRITE_FAILED             : "LOG_WRITE_FAILED"             ,
+	CURSOR_NOT_FOUND             : "CURSOR_NOT_FOUND"             ,
+	KEY_NOT_FOUND                : "KEY_NOT_FOUND"                ,
+},
+
+// Classes of errors
+ErrorClass = {
+	NETWORK_ERROR: ["HOST_UNREACHABLE", "HOST_NOT_FOUND"],
+};
+
+module.exports = {
+	ErrorCodes: ErrorCodes,
+	ErrorClass: ErrorClass
+};

lib/index.js

@@ -42,20 +42,28 @@ exports = module.exports = function aggregate(pipeline, ctx, inputs, callback) {
 				src = inputs;
 			}else{
 				try{
-					pipelineInst.collectionName = inputs;	//NOTE: use the given `inputs` directly; not really a "name" but we don't really have collection names in mungedb-aggregate
-					src = exports.pipeline.PipelineD.prepareCursorSource(pipelineInst, pipelineInst.ctx);
+					ctx.ns = inputs;	//NOTE: use the given `inputs` directly; hacking so that the cursor source will be our inputs instead of the context namespace
+					src = exports.pipeline.PipelineD.prepareCursorSource(pipelineInst, ctx);
 				}catch(err){
 					return callback(err);
 				}
 			}
 
-			// run the pipeline against the input src
-			var results = pipelineInst.run(src, callback === exports.SYNC_CALLBACK ? undefined : function aggregated(err, results){
+			var runCallback;
+			if (!callback) {
+				runCallback = exports.SYNC_CALLBACK;
+				pipelineInst.SYNC_MODE = true;
+			} else {
+				runCallback = function aggregated(err, results){
 				if(err) return callback(err);
 				return callback(null, results.result);
-			});
-			pipelineInst = null; // unset so that subsequent calls can rebuild the pipeline
-			return results;
+		};
+			}
+
+			// run the pipeline against
+			pipelineInst.stitch();
+			var results = pipelineInst.run(runCallback);
+			return results ? results.result : undefined;
 		};
 	if(inputs) return aggregator(ctx, inputs, callback);
 	return aggregator;
@@ -75,5 +83,8 @@ exports.Cursor = require("./Cursor");
 exports.pipeline = require("./pipeline/");
 
 // version info
-exports.version = "r2.4.0-rc0";
-exports.gitVersion = "cb8efcd6a2f05d35655ed9f9b947cc4a99ade8db";
+exports.version = "r2.5.4";
+exports.gitVersion = "ffd52e5f46cf2ba74ba931c78da62d4a7f480d8e";
+
+// error code constants
+exports.ERRORS = require('./Errors.js');

lib/pipeline/DepsTracker.js

@@ -0,0 +1,105 @@
+"use strict";
+
+/**
+ * Allows components in an aggregation pipeline to report what they need from their input.
+ *
+ * @class DepsTracker
+ * @namespace mungedb-aggregate.pipeline
+ * @module mungedb-aggregate
+ * @constructor
+ */
+var DepsTracker = module.exports = function DepsTracker() {
+	// fields is a set of strings
+	this.fields = {};
+	this.needWholeDocument = false;
+	this.needTextScore = false;
+}, klass = DepsTracker, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+var ParsedDeps = require("./ParsedDeps");
+
+/**
+ * Returns a projection object covering the dependencies tracked by this class.
+ * @method toProjection
+ * @return {Object} projection of caller's dependencies
+ */
+proto.toProjection = function toProjection() {
+	var proj = {};
+
+	// if(this.needTextScore) {
+		// bb.append(Document::metaFieldTextScore, BSON("$meta" << "textScore"));
+	// }
+
+	if (this.needWholeDocument) {
+		return proj;
+	}
+
+	if (Object.keys(this.fields).length === 0) {
+		// Projection language lacks good a way to say no fields needed. This fakes it.
+		proj._id = 0;
+		proj.$noFieldsNeeded = 1;
+		return proj;
+	}
+
+	var last = "",
+		needId = false;
+
+	Object.keys(this.fields).sort().forEach(function (it) {
+		if (it.slice(0,3) == "_id" && (it.length == 3 || it.charAt(3) == ".")) {
+			// _id and subfields are handled specially due in part to SERVER-7502
+			needId = true;
+			return;
+		}
+
+		if (last !== "" && it.slice(0, last.length) === last) {
+			// we are including a parent of *it so we don't need to include this
+			// field explicitly. In fact, due to SERVER-6527 if we included this
+			// field, the parent wouldn't be fully included. This logic relies
+			// on on set iterators going in lexicographic order so that a string
+			// is always directly before of all fields it prefixes.
+			return;
+		}
+
+		last = it + ".";
+		proj[it] = 1;
+	});
+
+	if (needId)
+		proj._id = 1;
+	else
+		proj._id = 0;
+
+	return proj;
+};
+
+/**
+ * Takes a depsTracker and builds a simple recursive lookup table out of it.
+ * @method toParsedDeps
+ * @return {ParsedDeps}
+ */
+proto.toParsedDeps = function toParsedDeps() {
+	var doc = {};
+
+	if (this.needWholeDocument || this.needTextScore) {
+		// can't use ParsedDeps in this case
+		// TODO: not sure what appropriate equivalent to boost::none is
+		return;
+	}
+
+	var last = "";
+	Object.keys(this.fields).sort().forEach(function (it) {
+		if (last !== "" && it.slice(0, last.length) === last) {
+			// we are including a parent of *it so we don't need to include this
+			// field explicitly. In fact, due to SERVER-6527 if we included this
+			// field, the parent wouldn't be fully included. This logic relies
+			// on on set iterators going in lexicographic order so that a string
+			// is always directly before of all fields it prefixes.
+			return;
+		}
+
+		last = it + ".";
+		// TODO: set nested field to true; i.e. a.b.c = true, not a = true
+		doc[it] = true;
+	});
+
+	return new ParsedDeps(doc);
+};

lib/pipeline/Document.js

@@ -11,10 +11,9 @@ var Document = module.exports = function Document(){
 	if(this.constructor == Document) throw new Error("Never create instances! Use static helpers only.");
 }, klass = Document, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// DEPENDENCIES
-var Value = require("./Value");
+var Value = require("./Value"),
+	FieldPath = require("./FieldPath");
 
-// STATIC MEMBERS
 /**
  * Shared "_id"
  * @static
@@ -22,18 +21,66 @@ var Value = require("./Value");
  **/
 klass.ID_PROPERTY_NAME = "_id";
 
+//SKIPPED: DocumentStorage
+
 /**
- * Compare two documents.
+ * Return JSON representation of this Document
+ * @method toJson
+ * @returns {Object} JSON representation of this Document
+ **/
+klass.toJson = function toJson(doc) {
+ 	return JSON.parse(JSON.stringify(doc));
+};
+
+//SKIPPED: metaFieldTextScore
+//SKIPPED: toBsonWithMetaData
+//SKIPPED: fromBsonWithMetaData
+
+//SKIPPED: most of MutableDocument except for getNestedField and setNestedField, squashed into Document here (because that's how they use it)
+function getNestedFieldHelper(obj, path) {
+	// NOTE: DEVIATION FROM MONGO: from MutableDocument; similar but necessarily different
+	var keys = Array.isArray(path) ? path : (path instanceof FieldPath ? path.fields : path.split(".")),
+		lastKey = keys[keys.length - 1];
+	for (var i = 0, l = keys.length - 1, cur = obj; i < l && cur instanceof Object; i++) {
+		var next = cur[keys[i]];
+		if (!(next instanceof Object)) return undefined;
+		cur = next;
+	}
+	return cur[lastKey];
+}
+klass.getNestedField = getNestedFieldHelper;  // NOTE: ours is static so these are the same
+klass.setNestedField = function setNestedField(obj, path, val) {
+	// NOTE: DEVIATION FROM MONGO: from MutableDocument; similar but necessarily different
+	var keys = Array.isArray(path) ? path : (path instanceof FieldPath ? path.fields : path.split(".")),
+		lastKey = keys[keys.length - 1];
+	for (var i = 0, l = keys.length - 1, cur = obj; i < l && cur instanceof Object; i++) {
+		var next = cur[keys[i]];
+		if (!(next instanceof Object)) cur[keys[i]] = next = {};
+		cur = next;
+	}
+	cur[lastKey] = val;
+	return val;
+};
+//SKIPPED: getApproximateSize -- not implementing mem usage right now
+//SKIPPED: hash_combine
+
+/** Compare two documents.
  *
- * BSON document field order is significant, so this just goes through the fields in order.
- * The comparison is done in roughly the same way as strings are compared, but comparing one field at a time instead of one character at a time.
+ *  BSON document field order is significant, so this just goes through
+ *  the fields in order.  The comparison is done in roughly the same way
+ *  as strings are compared, but comparing one field at a time instead
+ *  of one character at a time.
+ *
+ *  Note: This does not consider metadata when comparing documents.
  *
- * @static
  * @method compare
- * @param rL left document
- * @param rR right document
- * @returns an integer less than zero, zero, or an integer greater than zero, depending on whether rL < rR, rL == rR, or rL > rR
- **/
+ * @static
+ * @param l {Object}  left document
+ * @param r {Object} right document
+ * @returns an integer less than zero, zero, or an integer greater than
+ *           zero, depending on whether lhs < rhs, lhs == rhs, or lhs > rhs
+ *  Warning: may return values other than -1, 0, or 1
+ */
 klass.compare = function compare(l, r){	//TODO: might be able to replace this with a straight compare of docs using JSON.stringify()
 	var lPropNames = Object.getOwnPropertyNames(l),
 		lPropNamesLength = lPropNames.length,
@@ -48,42 +95,72 @@ klass.compare = function compare(l, r){	//TODO: might be able to replace this wi
 
 		if (i >= rPropNamesLength) return 1; // right document is shorter
 
-		var nameCmp = Value.compare(lPropNames[i], rPropNames[i]);
+		var rField = rPropNames[i],
+			lField = lPropNames[i];
+		var nameCmp = Value.compare(lField, rField);
 		if (nameCmp !== 0) return nameCmp; // field names are unequal
 
-		var valueCmp = Value.compare(l[lPropNames[i]], r[rPropNames[i]]);
+		var valueCmp = Value.compare(l[lPropNames[i]], r[rField]);
 		if (valueCmp) return valueCmp; // fields are unequal
 	}
+};
 
-	/* NOTREACHED */
-	throw new Error("This should never happen");	//verify(false)
-//		return 0;
+//SKIPPED: toString
+
+klass.serializeForSorter = function serializeForSorter(doc) {
+	//NOTE: DEVIATION FROM MONGO: they take a buffer to output the current instance into, ours is static and takes a doc and returns the serialized output
+	return JSON.stringify(doc);
+};
+
+klass.deserializeForSorter = function deserializeForSorter(docStr, sorterDeserializeSettings) {
+	return JSON.parse(docStr);
 };
 
+//SKIPPED: swap
+//SKIPPED: []
+//SKIPPED: getField -- inline as:  obj[key]
+//SKIPPED: getNestedField -- use fieldPath? might need to implement this...
+//SKIPPED: size -- need this? Number of fields in this document. O(n) -- recursive
+klass.empty = function(obj) {
+	return Object.keys(obj).length === 0;
+};
+//SKIPPED: operator <<
+//SKIPPED: positionOf
+
 /**
  * Clone a document
+ * This should only be called by MutableDocument and tests
+ * The new document shares all the fields' values with the original.
+ * This is not a deep copy.  Only the fields on the top-level document
+ * are cloned.
  * @static
  * @method clone
- * @param document
- **/
-klass.clone = function(document){
+ * @param doc
+ */
+klass.clone = function clone(doc) {
 	var obj = {};
-	for(var key in document){
-		if(document.hasOwnProperty(key)){
-			var withObjVal = document[key];
-			if(withObjVal === null) { // necessary to handle null values without failing
-				obj[key] = withObjVal;
-			}
-			else if(withObjVal.constructor === Object){
-				obj[key] = Document.clone(withObjVal);
-			}else{
-				obj[key] = withObjVal;
-			}
+	for (var key in doc) {
+		if (doc.hasOwnProperty(key)) {
+			obj[key] = doc[key];
 		}
 	}
 	return obj;
 };
+klass.cloneDeep = function cloneDeep(doc) {	//there are casese this is actually what we want
+	var obj = {};
+	for (var key in doc) {
+		if (doc.hasOwnProperty(key)) {
+			var val = doc[key];
+			obj[key] = val instanceof Object && val.constructor === Object ? Document.clone(val) : val;
+		}
+	}
+	return obj;
+};
+
+//SKIPPED: hasTextScore
+//SKIPPED: getTextScore
+
+//SKIPPED: memUsageForSorter -- not implementing mem usage right now
+//SKIPPED: getOwned -- not implementing mem usage right now
 
-//	proto.addField = function addField(){ throw new Error("Instead of `Document#addField(key,val)` you should just use `obj[key] = val`"); }
-//	proto.setField = function addField(){ throw new Error("Instead of `Document#setField(key,val)` you should just use `obj[key] = val`"); }
-//  proto.getField = function getField(){ throw new Error("Instead of `Document#getField(key)` you should just use `var val = obj[key];`"); }
+//SKIPPED: getPtr

lib/pipeline/FieldPath.js

@@ -11,62 +11,65 @@
  * @module mungedb-aggregate
  * @constructor
  * @param fieldPath the dotted field path string or non empty pre-split vector.
- **/
+ */
 var FieldPath = module.exports = function FieldPath(path) {
-	var fields = typeof path === "object" && typeof path.length === "number" ? path : path.split(".");
-	if(fields.length === 0) throw new Error("FieldPath cannot be constructed from an empty vector (String or Array).; code 16409");
-	for(var i = 0, n = fields.length; i < n; ++i){
-		var field = fields[i];
-		if(field.length === 0) throw new Error("FieldPath field names may not be empty strings; code 15998");
-		if(field[0] == "$") throw new Error("FieldPath field names may not start with '$'; code 16410");
-		if(field.indexOf("\0") != -1) throw new Error("FieldPath field names may not contain '\\0'; code 16411");
-		if(field.indexOf(".") != -1) throw new Error("FieldPath field names may not contain '.'; code 16412");
+	var fieldNames = typeof path === "object" && typeof path.length === "number" ? path : path.split(".");
+	if (fieldNames.length === 0) throw new Error("FieldPath cannot be constructed from an empty vector (String or Array).; massert code 16409");
+	this.fieldNames = [];
+	for (var i = 0, n = fieldNames.length; i < n; ++i) {
+		this._pushFieldName(fieldNames[i]);
 	}
-	this.path = path;
-	this.fields = fields;
 }, klass = FieldPath, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// STATIC MEMBERS
 klass.PREFIX = "$";
 
-// PROTOTYPE MEMBERS
 /**
  * Get the full path.
- *
  * @method getPath
  * @param fieldPrefix whether or not to include the field prefix
  * @returns the complete field path
- **/
-proto.getPath = function getPath(withPrefix) {
-	return ( !! withPrefix ? FieldPath.PREFIX : "") + this.fields.join(".");
+ */
+proto.getPath = function getPath(fieldPrefix) {
+	return (!!fieldPrefix ? FieldPath.PREFIX : "") + this.fieldNames.join(".");
 };
 
+//SKIPPED: writePath - merged into getPath
+
 /**
  * A FieldPath like this but missing the first element (useful for recursion). Precondition getPathLength() > 1.
- *
  * @method tail
- **/
+ */
 proto.tail = function tail() {
-	return new FieldPath(this.fields.slice(1));
+	return new FieldPath(this.fieldNames.slice(1));
 };
 
 /**
  * Get a particular path element from the path.
- *
  * @method getFieldName
  * @param i the zero based index of the path element.
  * @returns the path element
- **/
-proto.getFieldName = function getFieldName(i){	//TODO: eventually replace this with just using .fields[i] directly
-	return this.fields[i];
+ */
+proto.getFieldName = function getFieldName(i) {	//TODO: eventually replace this with just using .fieldNames[i] directly
+	return this.fieldNames[i];
+};
+
+klass._uassertValidFieldName = function _uassertValidFieldName(fieldName) {
+	if (fieldName.length === 0) throw new Error("FieldPath field names may not be empty strings; code 15998");
+	if (fieldName[0] === "$") throw new Error("FieldPath field names may not start with '$'; code 16410");
+	if (fieldName.indexOf("\0") !== -1) throw new Error("FieldPath field names may not contain '\\0'; code 16411");
+	if (fieldName.indexOf(".") !== -1) throw new Error("FieldPath field names may not contain '.'; code 16412");
+};
+
+proto._pushFieldName = function _pushFieldName(fieldName) {
+	klass._uassertValidFieldName(fieldName);
+	this.fieldNames.push(fieldName);
 };
 
 /**
  * Get the number of path elements in the field path.
- *
  * @method getPathLength
  * @returns the number of path elements
- **/
+ */
 proto.getPathLength = function getPathLength() {
-	return this.fields.length;
+	return this.fieldNames.length;
 };

lib/pipeline/ParsedDeps.js

@@ -0,0 +1,85 @@
+"use strict";
+
+/**
+ * This class is designed to quickly extract the needed fields into a Document.
+ * It should only be created by a call to DepsTracker.toParsedDeps.
+ *
+ * @class ParsedDeps
+ * @namespace mungedb-aggregate.pipeline
+ * @module mungedb-aggregate
+ * @constructor
+ * @param {Object} fields	The fields needed in a Document
+ */
+var ParsedDeps = module.exports = function ParsedDeps(fields) {
+	this._fields = fields;
+}, klass = ParsedDeps, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+var Value = require("./Value");
+
+/**
+ * Extracts fields from the input into a new Document, based on the caller.
+ *
+ * @method extractFields
+ * @param {Object} input	The JSON object to extract from
+ * @return {Document}
+ */
+proto.extractFields = function extractFields(input) {
+	return proto._documentHelper(input, this._fields);
+};
+
+/**
+ * Private: Handles array-type values for extractFields()
+ *
+ * @method _arrayHelper
+ * @param {Object} array	Array to iterate over
+ * @param {Object} neededFields
+ * @return {Array}
+ */
+proto._arrayHelper = function _arrayHelper(array, neededFields) {
+	var values = [];
+
+	for (var it in array) {
+		if (it instanceof Array)
+			values.push(_arrayHelper(it, neededFields));
+		else if (it instanceof Object)
+			values.push(proto._documentHelper(it, neededFields));
+	}
+
+	return values;
+};
+
+/**
+ * Private: Handles object-type values for extractFields()
+ *
+ * @method _documentHelper
+ * @param {Object} json	Object to iterate over and filter
+ * @param {Object} neededFields	Fields to not exclude
+ * @return {Document}
+ */
+proto._documentHelper = function _documentHelper(json, neededFields) {
+	var doc = {};
+
+	for (var fieldName in json) {
+		var jsonElement = json[fieldName],
+			isNeeded = neededFields[fieldName];
+
+		if (isNeeded === undefined)
+			continue;
+
+		if (Value.getType(isNeeded) === 'boolean') {
+			doc[fieldName] = jsonElement;
+			continue;
+		}
+
+		if (!isNeeded instanceof Object) throw new Error("dassert failure");
+
+		if (Value.getType(isNeeded) === 'object') {
+			if (jsonElement instanceof Array)
+				doc[fieldName] = proto._arrayHelper(jsonElement, isNeeded);
+			if (jsonElement instanceof Object)
+				doc[fieldName] = proto._documentHelper(jsonElement, isNeeded);
+		}
+	}
+
+	return doc;
+};

lib/pipeline/Pipeline.js

@@ -1,5 +1,4 @@
 "use strict";
-var async = require("async");
 
 /**
  * mongodb "commands" (sent via db.$cmd.findOne(...)) subclass to make a command.  define a singleton object for it.
@@ -10,11 +9,11 @@ var async = require("async");
  **/
 // CONSTRUCTOR
 var Pipeline = module.exports = function Pipeline(theCtx){
-	this.collectionName = null;
-	this.sourceVector = null;
+	this.sources = null;
 	this.explain = false;
 	this.splitMongodPipeline = false;
 	this.ctx = theCtx;
+	this.SYNC_MODE = false;
 }, klass = Pipeline, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 var DocumentSource = require("./documentSources/DocumentSource"),
@@ -24,24 +23,156 @@ var DocumentSource = require("./documentSources/DocumentSource"),
 	SkipDocumentSource = require('./documentSources/SkipDocumentSource'),
 	UnwindDocumentSource = require('./documentSources/UnwindDocumentSource'),
 	GroupDocumentSource = require('./documentSources/GroupDocumentSource'),
+	OutDocumentSource = require('./documentSources/OutDocumentSource'),
+	GeoNearDocumentSource = require('./documentSources/GeoNearDocumentSource'),
+	RedactDocumentSource = require('./documentSources/RedactDocumentSource'),
 	SortDocumentSource = require('./documentSources/SortDocumentSource');
 
 klass.COMMAND_NAME = "aggregate";
 klass.PIPELINE_NAME = "pipeline";
 klass.EXPLAIN_NAME = "explain";
 klass.FROM_ROUTER_NAME = "fromRouter";
-klass.SPLIT_MONGOD_PIPELINE_NAME = "splitMongodPipeline";
 klass.SERVER_PIPELINE_NAME = "serverPipeline";
 klass.MONGOS_PIPELINE_NAME = "mongosPipeline";
 
 klass.stageDesc = {};//attaching this to the class for test cases
+klass.stageDesc[GeoNearDocumentSource.geoNearName] = GeoNearDocumentSource.createFromJson;
+klass.stageDesc[GroupDocumentSource.groupName] = GroupDocumentSource.createFromJson;
 klass.stageDesc[LimitDocumentSource.limitName] = LimitDocumentSource.createFromJson;
 klass.stageDesc[MatchDocumentSource.matchName] = MatchDocumentSource.createFromJson;
+klass.stageDesc[OutDocumentSource.outName] = OutDocumentSource.createFromJson;
 klass.stageDesc[ProjectDocumentSource.projectName] = ProjectDocumentSource.createFromJson;
+klass.stageDesc[RedactDocumentSource.redactName] = ProjectDocumentSource.createFromJson;
 klass.stageDesc[SkipDocumentSource.skipName] = SkipDocumentSource.createFromJson;
-klass.stageDesc[UnwindDocumentSource.unwindName] = UnwindDocumentSource.createFromJson;
-klass.stageDesc[GroupDocumentSource.groupName] = GroupDocumentSource.createFromJson;
 klass.stageDesc[SortDocumentSource.sortName] = SortDocumentSource.createFromJson;
+klass.stageDesc[UnwindDocumentSource.unwindName] = UnwindDocumentSource.createFromJson;
+klass.nStageDesc = Object.keys(klass.stageDesc).length;
+
+klass.optimizations = {};
+klass.optimizations.local = {};
+
+/**
+ * Moves $match before $sort when they are placed next to one another
+ * @static
+ * @method moveMatchBeforeSort
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.moveMatchBeforeSort = function moveMatchBeforeSort(pipelineInst) {
+	var sources = pipelineInst.sources;
+	for(var srcn = sources.length, srci = 1; srci < srcn; ++srci) {
+		var source = sources[srci];
+		if(source.constructor === MatchDocumentSource) {
+			var previous = sources[srci - 1];
+			if(previous && previous.constructor === SortDocumentSource) { //Added check that previous exists
+				/* swap this item with the previous */
+				sources[srci] = previous;
+				sources[srci-1] = source;
+			}
+		}
+	}
+};
+
+/**
+ * Moves $limit before $skip when they are placed next to one another
+ * @static
+ * @method moveLimitBeforeSkip
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.moveLimitBeforeSkip = function moveLimitBeforeSkip(pipelineInst) {
+	var sources = pipelineInst.sources;
+	if(sources.length === 0) return;
+	for(var i = sources.length - 1; i >= 1 /* not looking at 0 */; i--) {
+		var limit = sources[i].constructor === LimitDocumentSource ? sources[i] : undefined,
+			skip = sources[i-1].constructor === SkipDocumentSource ? sources[i-1] : undefined;
+		if(limit && skip) {
+			limit.setLimit(limit.getLimit() + skip.getSkip());
+			sources[i-1] = limit;
+			sources[i] = skip;
+
+			// Start at back again. This is needed to handle cases with more than 1 $limit
+			// (S means skip, L means limit)
+			//
+			// These two would work without second pass (assuming back to front ordering)
+			// SL   -> LS
+			// SSL  -> LSS
+			//
+			// The following cases need a second pass to handle the second limit
+			// SLL  -> LLS
+			// SSLL -> LLSS
+			// SLSL -> LLSS
+			i = sources.length; // decremented before next pass
+		}
+	}
+};
+
+/**
+ * Attempts to coalesce every pipeline stage into the previous pipeline stage, starting after the first
+ * @static
+ * @method coalesceAdjacent
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.coalesceAdjacent = function coalesceAdjacent(pipelineInst) {
+	var sources = pipelineInst.sources;
+	if(sources.length === 0) return;
+
+	// move all sources to a temporary list
+	var moveSrc = sources.pop(),
+		tempSources = [];
+	while(moveSrc) {
+		tempSources.unshift(moveSrc);
+		moveSrc = sources.pop();
+	}
+
+	// move the first one to the final list
+	sources.push(tempSources[0]);
+
+	// run through the sources, coalescing them or keeping them
+	for(var tempn = tempSources.length, tempi = 1; tempi < tempn; ++tempi) {
+		// If we can't coalesce the source with the last, then move it
+		// to the final list, and make it the new last.  (If we succeeded,
+		// then we're still on the same last, and there's no need to move
+		// or do anything with the source -- the destruction of tempSources
+		// will take care of the rest.)
+		var lastSource = sources[sources.length-1],
+			tempSrc = tempSources[tempi];
+		if(!(lastSource && tempSrc)) {
+			throw new Error('Must have a last and current source'); // verify(lastSource && tempSrc);
+		}
+		if(!lastSource.coalesce(tempSrc)) sources.push(tempSrc);
+	}
+};
+
+/**
+ * Iterates over sources in the pipelineInst, optimizing each
+ * @static
+ * @method optimizeEachDocumentSource
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.optimizeEachDocumentSource = function optimizeEachDocumentSource(pipelineInst) {
+	var sources = pipelineInst.sources;
+	for(var srci = 0, srcn = sources.length; srci < srcn; ++srci) {
+		sources[srci].optimize();
+	}
+};
+
+/**
+ * Auto-places a $match before a $redact when the $redact is the first item in a pipeline
+ * @static
+ * @method duplicateMatchBeforeInitalRedact
+ * @param pipelineInst An instance of a Pipeline
+ **/
+klass.optimizations.local.duplicateMatchBeforeInitalRedact = function duplicateMatchBeforeInitalRedact(pipelineInst) {
+	var sources = pipelineInst.sources;
+	if(sources.length >= 2 && sources[0].constructor === RedactDocumentSource) {
+		if(sources[1].constructor === MatchDocumentSource) {
+			var match = sources[1],
+				redactSafePortion = match.redactSafePortion();
+			if(Object.keys(redactSafePortion).length > 0) {
+				sources.shift(MatchDocumentSource.createFromJson(redactSafePortion, pipelineInst.ctx));
+			}
+		}
+	}
+};
 
 /**
  * Create an `Array` of `DocumentSource`s from the given JSON pipeline
@@ -52,7 +183,7 @@ klass.stageDesc[SortDocumentSource.sortName] = SortDocumentSource.createFromJson
  * @returns {Array}  The parsed `DocumentSource`s
  **/
 klass.parseDocumentSources = function parseDocumentSources(pipeline, ctx){
-	var sourceVector = [];
+	var sources = [];
 	for (var nSteps = pipeline.length, iStep = 0; iStep < nSteps; ++iStep) {
 		// pull out the pipeline element as an object
 		var pipeElement = pipeline[iStep];
@@ -70,11 +201,14 @@ klass.parseDocumentSources = function parseDocumentSources(pipeline, ctx){
 
 		// Parse the stage
 		var stage = desc(stageSpec, ctx);
-		if (!stage) throw new Error("Stage must not be undefined!");
-		stage.setPipelineStep(iStep);
-		sourceVector.push(stage);
+		if (!stage) throw new Error("Stage must not be undefined!"); // verify(stage)
+		sources.push(stage);
+
+		if(stage.constructor === OutDocumentSource && iStep !== nSteps - 1) {
+			throw new Error("$out can only be the final stage in the pipeline; code 16435");
+		}
 	}
-	return sourceVector;
+	return sources;
 };
 
 /**
@@ -99,12 +233,15 @@ klass.parseCommand = function parseCommand(cmdObj, ctx){
 	var pipeline;
 	for(var fieldName in cmdObj){
 		var cmdElement = cmdObj[fieldName];
-		if(fieldName == klass.COMMAND_NAME)						pipelineInst.collectionName = cmdElement;		//look for the aggregation command
+		if(fieldName[0] == "$")									continue;
+		else if(fieldName == "cursor")							continue;
+		else if(fieldName == klass.COMMAND_NAME)				continue;										//look for the aggregation command
 		else if(fieldName == klass.PIPELINE_NAME)				pipeline = cmdElement;							//check for the pipeline of JSON doc srcs
 		else if(fieldName == klass.EXPLAIN_NAME)				pipelineInst.explain = cmdElement;				//check for explain option
-		else if(fieldName == klass.FROM_ROUTER_NAME)			pipelineInst.fromRouter = cmdElement;			//if the request came from the router, we're in a shard
-		else if(fieldName == klass.SPLIT_MONGOD_PIPELINE_NAME)	pipelineInst.splitMongodPipeline = cmdElement;	//check for debug options
-		// NOTE: DEVIATION FROM MONGO: Not implementing: "Ignore $auth information sent along with the command. The authentication system will use it, it's not a part of the pipeline."
+		else if(fieldName == klass.FROM_ROUTER_NAME)			ctx.inShard = cmdElement;						//if the request came from the router, we're in a shard
+		else if(fieldName == "allowDiskUsage") {
+			if(typeof cmdElement !== 'boolean') throw new Error("allowDiskUsage must be a bool, not a " + typeof allowDiskUsage+ "; uassert code 16949");
+		}
 		else throw new Error("unrecognized field " + JSON.stringify(fieldName));
 	}
 
@@ -113,64 +250,13 @@ klass.parseCommand = function parseCommand(cmdObj, ctx){
 	 * Set up the specified document source pipeline.
 	 **/
 	// NOTE: DEVIATION FROM MONGO: split this into a separate function to simplify and better allow for extensions (now in parseDocumentSources)
-	var sourceVector = pipelineInst.sourceVector = Pipeline.parseDocumentSources(pipeline, ctx);
-
-	/* if there aren't any pipeline stages, there's nothing more to do */
-	if (!sourceVector.length) return pipelineInst;
-
-	/* Move filters up where possible.
-	CW TODO -- move filter past projections where possible, and noting corresponding field renaming.
-	*/
-
-	/*
-	Wherever there is a match immediately following a sort, swap them.
-	This means we sort fewer items.  Neither changes the documents in the stream, so this transformation shouldn't affect the result.
-	We do this first, because then when we coalesce operators below, any adjacent matches will be combined.
-	*/
-	for(var srcn = sourceVector.length, srci = 1; srci < srcn; ++srci) {
-		var source = sourceVector[srci];
-		if (source instanceof MatchDocumentSource) {
-			var previous = sourceVector[srci - 1];
-			if (previous instanceof SortDocumentSource) {
-				/* swap this item with the previous */
-				sourceVector[srci - 1] = source;
-				sourceVector[srci] = previous;
-			}
-		}
-	}
+	var sources = pipelineInst.sources = Pipeline.parseDocumentSources(pipeline, ctx);
 
-	/*
-	Coalesce adjacent filters where possible.  Two adjacent filters are equivalent to one filter whose predicate is the conjunction of the two original filters' predicates.
-	For now, capture this by giving any DocumentSource the option to absorb it's successor; this will also allow adjacent projections to coalesce when possible.
-	Run through the DocumentSources, and give each one the opportunity to coalesce with its successor.  If successful, remove the successor.
-	Move all document sources to a temporary list.
-	*/
-	var tempVector = sourceVector.slice(0);
-	sourceVector.length = 0;
-
-	// move the first one to the final list
-	sourceVector.push(tempVector[0]);
-
-	// run through the sources, coalescing them or keeping them
-	for(var tempn = tempVector.length, tempi = 1; tempi < tempn; ++tempi) {
-		/*
-		If we can't coalesce the source with the last, then move it to the final list, and make it the new last.
-		(If we succeeded, then we're still on the same last, and there's no need to move or do anything with the source -- the destruction of tempVector will take care of the rest.)
-		*/
-		var lastSource = sourceVector[sourceVector.length - 1],
-			temp = tempVector[tempi];
-		if (!temp || !lastSource) throw new Error("null document sources found");
-		if (!lastSource.coalesce(temp)){
-			sourceVector.push(temp);
-		}
-	}
-
-	// optimize the elements in the pipeline
-	for(var i = 0, l = sourceVector.length; i<l; i++) {
-		var iter = sourceVector[i];
-		if (!iter) throw new Error("Pipeline received empty document as argument");
-		iter.optimize();
-	}
+	klass.optimizations.local.moveMatchBeforeSort(pipelineInst);
+	klass.optimizations.local.moveLimitBeforeSkip(pipelineInst);
+	klass.optimizations.local.coalesceAdjacent(pipelineInst);
+	klass.optimizations.local.optimizeEachDocumentSource(pipelineInst);
+	klass.optimizations.local.duplicateMatchBeforeInitalRedact(pipelineInst);
 
 	return pipelineInst;
 };
@@ -186,86 +272,154 @@ function ifError(err) {
 }
 
 /**
- * Run the pipeline
+ * Gets the initial $match query when $match is the first pipeline stage
  * @method run
  * @param	inputSource		{DocumentSource}	The input document source for the pipeline
  * @param	[callback]		{Function}			Optional callback function if using async extensions
+ * @return {Object}	An empty object or the match spec
 **/
-proto.run = function run(inputSource, callback){
-	if (inputSource && !(inputSource instanceof DocumentSource)) throw new Error("arg `inputSource` must be an instance of DocumentSource");
-	if (!callback) callback = klass.SYNC_CALLBACK;
-	var self = this;
-	if (callback === klass.SYNC_CALLBACK) { // SYNCHRONOUS MODE
-		inputSource.setSource(undefined, ifError);	//TODO: HACK: temp solution to the fact that we need to initialize our source since we're using setSource as a workaround for the lack of real async cursors
-		var source = inputSource;
-		for(var i = 0, l = self.sourceVector.length; i < l; i++){
-			var temp = self.sourceVector[i];
-			temp.setSource(source, ifError);
-			source = temp;
-		}
-		/*
-		Iterate through the resulting documents, and add them to the result.
-		We do this even if we're doing an explain, in order to capture the document counts and other stats.
-		However, we don't capture the result documents for explain.
-		*/
-		var resultArray = [];
-		try{
-			for(var hasDoc = !source.eof(); hasDoc; hasDoc = source.advance()) {
-				var document = source.getCurrent();
-				resultArray.push(document);	// add the document to the result set
-				//Commenting out this assertion for munge.  MUHAHAHA!!!
-				// object will be too large, assert. the extra 1KB is for headers
-				//if(resultArray.len() < BSONObjMaxUserSize - 1024) throw new Error("aggregation result exceeds maximum document size (" + BSONObjMaxUserSize / (1024 * 1024) + "MB); code 16389");
+proto.getInitialQuery = function getInitialQuery() {
+	var sources = this.sources;
+	if(sources.length === 0) {
+		return {};
+	}
+
+	/* look for an initial $match */
+	var match = sources[0].constructor === MatchDocumentSource ? sources[0] : undefined;
+	if(!match) return {};
+
+	return match.getQuery();
+};
+
+/**
+ * Creates the JSON representation of the pipeline
+ * @method run
+ * @param	inputSource		{DocumentSource}	The input document source for the pipeline
+ * @param	[callback]		{Function}			Optional callback function if using async extensions
+ * @return {Object}	An empty object or the match spec
+**/
+proto.serialize = function serialize() {
+	var serialized = {},
+		array = [];
+
+	// create an array out of the pipeline operations
+	this.sources.forEach(function(source) {
+		source.serializeToArray(array);
+	});
+
+	serialized[klass.COMMAND_NAME] = this.ctx && this.ctx.ns && this.ctx.ns.coll ? this.ctx.ns.coll : '';
+	serialized[klass.PIPELINE_NAME] = array;
+
+	if(this.explain) serialized[klass.EXPLAIN_NAME] = this.explain;
+
+	return serialized;
+};
+
+/**
+ * Points each source at its previous source
+ * @method stitch
+**/
+proto.stitch = function stitch() {
+	if(this.sources.length <= 0) throw new Error("should not have an empty pipeline; massert code 16600");
+
+	/* chain together the sources we found */
+	var prevSource = this.sources[0];
+	for(var srci = 1, srcn = this.sources.length; srci < srcn; srci++) {
+		var tempSource = this.sources[srci];
+		tempSource.setSource(prevSource);
+		prevSource = tempSource;
+	}
+};
+
+/**
+ * Run the pipeline
+ * @method run
+ * @param callback {Function} Optional. Run the pipeline in async mode; callback(err, result)
+ * @return result {Object} The result of executing the pipeline
+**/
+proto.run = function run(callback) {
+	// should not get here in the explain case
+	if(this.explain) throw new Error("Should not be running a pipeline in explain mode!");
+
+	/* NOTE: DEVIATION FROM MONGO SOURCE. WE'RE SUPPORTING SYNC AND ASYNC */
+	if(this.SYNC_MODE) {
+		callback();
+		return this._runSync();
+	} else {
+		return this._runAsync(callback);
+	}
+};
+
+/**
+ * Get the last document source in the pipeline
+ * @method _getFinalSource
+ * @return {Object}		The DocumentSource at the end of the pipeline
+ * @private
+**/
+proto._getFinalSource = function _getFinalSource() {
+	return this.sources[this.sources.length - 1];
+};
+
+/**
+ * Run the pipeline synchronously
+ * @method _runSync
+ * @return {Object}		The results object {result:resultArray}
+ * @private
+**/
+proto._runSync = function _runSync(callback) {
+	var resultArray = [],
+		finalSource = this._getFinalSource(),
+		handleErr = function(err) {
+			if(err) throw err;
+		},
+		next;
+	while((next = finalSource.getNext(handleErr)) !== DocumentSource.EOF) {
+		resultArray.push(next);
+	}
+	return {result:resultArray};
+};
+
+/**
+ * Run the pipeline asynchronously
+ * @method _runAsync
+ * @param callback {Function} callback(err, resultObject)
+ * @private
+**/
+proto._runAsync = function _runAsync(callback) {
+	var resultArray = [],
+		finalSource = this._getFinalSource(),
+		gotNext = function(err, doc) {
+			if(err) return callback(err);
+			if(doc !== DocumentSource.EOF) {
+				resultArray.push(doc);
+				return setImmediate(function() { //setImmediate to avoid callstack size issues
+					finalSource.getNext(gotNext);
+				});
+			} else {
+				return callback(null, {result:resultArray});
 			}
-		} catch (err) {
-			return callback(err);
-		}
-		var result = {
-			result: resultArray
-//			,ok: true;	//not actually in here... where does this come from?
 		};
-		return callback(null, result);
-	} else {	// ASYNCHRONOUS MODE	//TODO: move this up to a higher level package?
-		return inputSource.setSource(undefined, function(err){	//TODO: HACK: temp solution to the fact that we need to initialize our source since we're using setSource as a workaround for the lack of real async cursors
-			if (err) return callback(err);
-			// chain together the sources we found
-			var source = inputSource;
-			async.eachSeries(
-				self.sourceVector,
-				function eachSrc(temp, next){
-					temp.setSource(source, function(err){
-						if (err) return next(err);
-						source = temp;
-						return next();
-					});
-				},
-				function doneSrcs(err){ //source is left pointing at the last source in the chain
-					if (err) return callback(err);
-					/*
-					Iterate through the resulting documents, and add them to the result.
-					We do this even if we're doing an explain, in order to capture the document counts and other stats.
-					However, we don't capture the result documents for explain.
-					*/
-					// the array in which the aggregation results reside
-					var resultArray = [];
-					try{
-						for(var hasDoc = !source.eof(); hasDoc; hasDoc = source.advance()) {
-							var document = source.getCurrent();
-							resultArray.push(document);	// add the document to the result set
-							//Commenting out this assertion for munge.  MUHAHAHA!!!
-							// object will be too large, assert. the extra 1KB is for headers
-							//if(resultArray.len() < BSONObjMaxUserSize - 1024) throw new Error("aggregation result exceeds maximum document size (" + BSONObjMaxUserSize / (1024 * 1024) + "MB); code 16389");
-						}
-					} catch (err) {
-						return callback(err);
-					}
-					var result = {
-						result: resultArray
-	//					,ok: true;	//not actually in here... where does this come from?
-					};
-					return callback(null, result);
-				}
-			);
-		});
-	}
+	finalSource.getNext(gotNext);
+};
+
+/**
+ * Get the pipeline explanation
+ * @method writeExplainOps
+ * @return {Array}	An array of source explanations
+**/
+proto.writeExplainOps = function writeExplainOps() {
+	var array = [];
+	this.sources.forEach(function(source) {
+		source.serializeToArray(array, /*explain=*/true);
+	});
+	return array;
+};
+
+/**
+ * Set the source of documents for the pipeline
+ * @method addInitialSource
+ * @param source {DocumentSource}
+**/
+proto.addInitialSource = function addInitialSource(source) {
+	this.sources.unshift(source);
 };

lib/pipeline/PipelineD.js

@@ -20,6 +20,7 @@ var DocumentSource = require('./documentSources/DocumentSource'),
  * Create a Cursor wrapped in a DocumentSourceCursor, which is suitable to be the first source for a pipeline to begin with.
  * This source will feed the execution of the pipeline.
  *
+ * //NOTE: Not doing anything here, as we don't use any of these cursor source features
  * //NOTE: DEVIATION FROM THE MONGO: We don't have special optimized cursors; You could support something similar by overriding `Pipeline#run` to call `DocumentSource#coalesce` on the `inputSource` if you really need it.
  *
  * This method looks for early pipeline stages that can be folded into
@@ -32,48 +33,62 @@ var DocumentSource = require('./documentSources/DocumentSource'),
  * @param ctx       {Object}    Context for expressions
  * @returns	{CursorDocumentSource}	the cursor that was created
 **/
-klass.prepareCursorSource = function prepareCursorSource(pipeline, /*dbName,*/ expCtx){
+klass.prepareCursorSource = function prepareCursorSource(pipeline, expCtx){
 
-	var sources = pipeline.sourceVector;
+	var sources = pipeline.sources;
 
-	//NOTE: SKIPPED: look for initial match
-	//NOTE: SKIPPED: create a query object
+	// NOTE: SKIPPED: look for initial match
+	// NOTE: SKIPPED: create a query object
 
-	//Look for an initial simple project; we'll avoid constructing Values for fields that won't make it through the projection
+	// Look for an initial simple project; we'll avoid constructing Values for fields that won't make it through the projection
 	var projection = {};
-	var deps = [];
+	var dependencies;
+	var deps = {};
 	var status = DocumentSource.GetDepsReturn.SEE_NEXT;
-	for (var i=0; i < sources.length && status != DocumentSource.GetDepsReturn.EXHAUSTIVE; i++) {
+	for (var i=0; i < sources.length && status !== DocumentSource.GetDepsReturn.EXHAUSTIVE; i++) {
 		status = sources[i].getDependencies(deps);
+		if(Object.keys(deps).length === 0) {
+			status = DocumentSource.GetDepsReturn.NOT_SUPPORTED;
+		}
 	}
-	if (status == DocumentSource.GetDepsReturn.EXHAUSTIVE) {
+	if (status === DocumentSource.GetDepsReturn.EXHAUSTIVE) {
 		projection = DocumentSource.depsToProjection(deps);
+		dependencies = DocumentSource.parseDeps(deps);
 	}
 
-	//NOTE: SKIPPED: Look for an initial sort
-	//NOTE: SKIPPED: Create the sort object
+	// NOTE: SKIPPED: Look for an initial sort
+	// NOTE: SKIPPED: Create the sort object
 
-//	//get the full "namespace" name
-//	var fullName = dbName + "." + pipeline.collectionName;
+	//get the full "namespace" name
+	// var fullName = dbName + "." + pipeline.collectionName;
 
-	//NOTE: SKIPPED: if(DEV) log messages
+	// NOTE: SKIPPED: if(DEV) log messages
 
-	//Create the necessary context to use a Cursor
-	//NOTE: SKIPPED: pSortedCursor bit
-	//NOTE: SKIPPED: pUnsortedCursor bit
-	var cursorWithContext = new CursorDocumentSource.CursorWithContext(/*fullName*/);
+	// Create the necessary context to use a Cursor
+	// NOTE: SKIPPED: pSortedCursor bit
+	// NOTE: SKIPPED: pUnsortedCursor bit
 
-	// Now add the Cursor to cursorWithContext
-	cursorWithContext._cursor = new Cursor( pipeline.collectionName );
+	// NOTE: Deviating from mongo here. We're passing in a source or set of documents instead of collection name in the ctx.ns field
+	var source;
+	if(expCtx.ns instanceof DocumentSource){
+		source = expCtx.ns;
+	} else {
+		var cursorWithContext = new CursorDocumentSource.CursorWithContext(/*fullName*/);
 
-	// wrap the cursor with a DocumentSource and return that
-	var source = new CursorDocumentSource( cursorWithContext, expCtx );
+		// Now add the Cursor to cursorWithContext
+		cursorWithContext._cursor = new Cursor( expCtx.ns );	//NOTE: collectionName will likely be an array of documents in munge
 
-//	source.namespace = fullName;
+		// wrap the cursor with a DocumentSource and return that
+		source = new CursorDocumentSource( cursorWithContext, expCtx );
 
-	//NOTE: SKIPPED: Note the query and sort
+		// NOTE: SKIPPED: Note the query and sort
 
-	if (Object.keys(projection).length) source.setProjection(projection);
+		if (Object.keys(projection).length) source.setProjection(projection, dependencies);
 
-	return source;
+		while(sources.length > 0 && source.coalesce(sources[0])) { //Note: Attempting to coalesce into the cursor source
+			sources.shift();
+		}
+	}
+
+	pipeline.addInitialSource(source);
 };

lib/pipeline/Value.js

@@ -11,34 +11,36 @@ var Value = module.exports = function Value(){
 	if(this.constructor == Value) throw new Error("Never create instances of this! Use the static helpers only.");
 }, klass = Value, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// PRIVATE STUFF
-function getTypeVerifier(type, IClass, isStrict) {
-	return function verifyType(value) {
-		if (typeof(value) != type) throw new Error("typeof value is not: " + type + "; actual: " + typeof(value));
-		if (typeof(IClass) == "function" && !(isStrict ? value.constructor == IClass : value instanceof IClass)) throw new Error("instanceof value is not: " + IClass.name + "; actual: " + value.constructor.name);
-		return value;
-	};
-}
-
-// STATIC MEMBERS
-klass.verifyNumber = getTypeVerifier("number", Number);	//NOTE: replaces #getDouble(), #getInt(), and #getLong()
-klass.verifyString = getTypeVerifier("string", String);
-klass.verifyDocument = getTypeVerifier("object", Object, true);	//TODO: change to verifyObject? since we're not using actual Document instances
-klass.verifyArray = getTypeVerifier("object", Array, true);
-klass.verifyDate = getTypeVerifier("object", Date, true);
-klass.verifyRegExp = getTypeVerifier("object", RegExp, true);	//NOTE: renamed from #getRegex()
-//TODO:	klass.verifyOid = ...?
-//TODO:	klass.VerifyTimestamp = ...?
-klass.verifyBool = getTypeVerifier("boolean", Boolean, true);
+var Document;  // loaded lazily below //TODO: a dirty hack; need to investigate and clean up
+
+//SKIPPED: ValueStorage -- probably not required; use JSON?
+//SKIPPED: createIntOrLong -- not required; use Number
+//SKIPPED: operator <Array>[] -- not required; use arr[i]
+//SKIPPED: operator <Object>[] -- not required; use obj[key]
+//SKIPPED: operator << -- not required
+//SKIPPED: addToBsonObj -- not required; use obj[key] = <val>
+//SKIPPED: addToBsonArray -- not required; use arr.push(<val>)
 
+/** Coerce a value to a bool using BSONElement::trueValue() rules.
+ * Some types unsupported.  SERVER-6120
+ * @method coerceToBool
+ * @static
+ */
 klass.coerceToBool = function coerceToBool(value) {
 	if (typeof(value) == "string") return true;
 	return !!value;	// including null or undefined
 };
-klass.coerceToInt =
-klass.coerceToLong =
-klass.coerceToDouble =
-klass._coerceToNumber = function _coerceToNumber(value) { //NOTE: replaces .coerceToInt(), .coerceToLong(), and .coerceToDouble()
+
+/** Coercion operators to extract values with fuzzy type logic.
+ *  These currently assert if called on an unconvertible type.
+ *  TODO: decided how to handle unsupported types.
+ */
+klass.coerceToWholeNumber = function coerceToInt(value) {
+	return klass.coerceToNumber(value) | 0;
+};
+klass.coerceToInt = klass.coerceToWholeNumber;
+klass.coerceToLong = klass.coerceToWholeNumber;
+klass.coerceToNumber = function coerceToNumber(value) {
 	if (value === null) return 0;
 	switch (typeof(value)) {
 	case "undefined":
@@ -59,99 +61,56 @@ klass._coerceToNumber = function _coerceToNumber(value) { //NOTE: replaces .coer
 		throw new Error("can't convert from BSON type " + typeof(value) + " to int; codes 16003, 16004, 16005");
 	}
 };
+klass.coerceToDouble = klass.coerceToNumber;
 klass.coerceToDate = function coerceToDate(value) {
-	//TODO: Support Timestamp BSON type?
 	if (value instanceof Date) return value;
 	throw new Error("can't convert from BSON type " + typeof(value) + " to Date; uassert code 16006");
 };
-//TODO: klass.coerceToTimeT = ...?   try to use as Date first rather than having coerceToDate return Date.parse  or dateObj.getTime() or similar
-//TODO:	klass.coerceToTm = ...?
+//SKIPPED: coerceToTimeT -- not required; just use Date
+//SKIPPED: coerceToTm -- not required; just use Date
+//SKIPPED: tmToISODateString -- not required; just use Date
 klass.coerceToString = function coerceToString(value) {
-	if (value === null) return "";
-	switch (typeof(value)) {
-	case "undefined":
-		return "";
-	case "number":
-		return value.toString();
-	case "string":
-		return value;
-	default:
-		throw new Error("can't convert from BSON type " + typeof(value) + " to String; uassert code 16007");
-	}
-};
-
-
-klass.canonicalize = function canonicalize(x) {
-	var xType = typeof(x);
-	if(xType == "object") xType = x === null ? "null" : x.constructor.name;
-	switch (xType) {
-		case "MinKey":
-			return -1;
-		case "MaxKey":
-			return 127;
-		case "EOO":
-		case "undefined":
-		case undefined:
-			return 0;
-		case "jstNULL":
-		case "null":
-		case "Null":
-			return 5;
-		case "NumberDouble":
-		case "NumberInt":
-		case "NumberLong":
+	var type = typeof(value);
+	if (type == "object") type = value === null ? "null" : value.constructor.name;
+	switch (type) {
+		//TODO: BSON numbers?
 		case "number":
-			return 10;
-		case "Symbol":
+			return value.toString();
+
+		//TODO: BSON Code?
+		//TODO: BSON Symbol?
 		case "string":
-			return 15;
-		case "Object":
-			return 20;
-		case "Array":
-			return 25;
-		case "Binary":
-			return 30;
-		case "ObjectId":
-			return 35;
-		case "ObjectID":
-			return 35;
-		case "boolean":
-		case "Boolean":
-			return 40;
+			return value;
+
+		//TODO: BSON Timestamp?
 		case "Date":
-		case "Timestamp":
-			return 45;
-		case "RegEx":
-		case "RegExp":
-			return 50;
-		case "DBRef":
-			return 55;
-		case "Code":
-			return 60;
-		case "CodeWScope":
-			return 65;
+			return value.toISOString().split(".")[0];
+
+		case "null":
+		case "undefined":
+			return "";
+
 		default:
-			// Default value for Object
-			return 20;  
+			throw new Error("can't convert from BSON type " + typeof(value) + " to String; uassert code 16007");
 	}
 };
+//SKIPPED: coerceToTimestamp
 
+/**
+ * Helper function for Value.compare
+ * @method cmp
+ * @static
+ */
 klass.cmp = function cmp(l, r){
 	return l < r ? -1 : l > r ? 1 : 0;
 };
 
-//TODO:	klass.coerceToTimestamp = ...?
-
-/**
- * Compare two Values.
- *
+/** Compare two Values.
  * @static
  * @method compare
- * @param rL left value
- * @param rR right value
- * @returns an integer less than zero, zero, or an integer greater than zero, depending on whether rL < rR, rL == rR, or rL > rR
- **/
-var Document;  // loaded lazily below //TODO: a dirty hack; need to investigate and clean up
+ * @returns an integer less than zero, zero, or an integer greater than zero, depending on whether lhs < rhs, lhs == rhs, or lhs > rhs
+ * Warning: may return values other than -1, 0, or 1
+ */
 klass.compare = function compare(l, r) {
 	//NOTE: deviation from mongo code: we have to do some coercing for null "types" because of javascript
 	var lt = l === null ? "null" : typeof(l),
@@ -170,7 +129,22 @@ klass.compare = function compare(l, r) {
 		if (isNaN(r)) return 1;
 		return klass.cmp(l,r);
 	}
-
+	// Compare MinKey and MaxKey cases
+	if (l instanceof Object && ["MinKey", "MaxKey"].indexOf(l.constructor.name) !== -1) {
+		if (l.constructor.name == r.constructor.name) {
+			return 0;
+		} else if (l.constructor.name === "MinKey") {
+			return -1;
+		} else {
+			return 1; // Must be MaxKey, which is greater than everything but MaxKey (which r cannot be)
+		}
+	}
+	// hack: These should really get converted to their BSON type ids and then compared, we use int vs object in queries
+	if (lt === "number" && rt === "object"){
+		return -1;
+	} else if (lt === "object" && rt === "number") {
+		return 1;
+	}
 	// CW TODO for now, only compare like values
 	if (lt !== rt) throw new Error("can't compare values of BSON types [" + lt + " " + l.constructor.name + "] and [" + rt + ":" + r.constructor.name + "]; code 16016");
 	// Compare everything else
@@ -178,7 +152,7 @@ klass.compare = function compare(l, r) {
 	case "number":
 		throw new Error("number types should have been handled earlier!");
 	case "string":
-		return klass.cmp(l,r);
+		return klass.cmp(l, r);
 	case "boolean":
 		return l == r ? 0 : l ? 1 : -1;
 	case "undefined": //NOTE: deviation from mongo code: we are comparing null to null or undefined to undefined (otherwise the ret stuff above would have caught it)
@@ -208,8 +182,99 @@ klass.compare = function compare(l, r) {
 
 };
 
-//TODO:	klass.hashCombine = ...?
-//TODO:	klass.getWidestNumeric = ...?
-//TODO:	klass.getApproximateSize = ...?
-//TODO:	klass.addRef = ...?
-//TODO:	klass.release = ...?
+//SKIPPED: hash_combine
+//SKIPPED: getWidestNumeric
+//SKIPPED: getApproximateSize
+//SKIPPED: toString
+//SKIPPED: operator <<
+//SKIPPED: serializeForSorter
+//SKIPPED: deserializeForSorter
+
+/**
+ * Takes an array and removes items and adds them to returned array.
+ * @method consume
+ * @static
+ * @param consumed {Array} The array to be copied, emptied.
+ **/
+klass.consume = function consume(consumed) {
+	return consumed.splice(0);
+};
+
+//NOTE: DEVIATION FROM MONGO: many of these do not apply or are inlined (code where relevant)
+// missing(val):  val == undefined
+// nullish(val):  val == null || val == undefined
+// numeric(val):  typeof val == "number"
+klass.getType = function getType(v) {
+	var t = typeof v;
+	if (t == "object") t = (v === null ? "null" : v.constructor.name || t);
+	return t;
+};
+// getArrayLength(arr): arr.length
+// getString(val): val.toString()   //NOTE: same for getStringData(val) I think
+// getOid
+// getBool
+// getDate
+// getTimestamp
+// getRegex(re):  re.source
+// getRegexFlags(re):  re.toString().slice(-re.toString().lastIndexOf('/') + 2)
+// getSymbol
+// getCode
+// getInt
+// getLong
+//NOTE: also, because of this we are not throwing if the type does not match like the mongo code would but maybe that's okay
+
+// from bsontypes
+klass.canonicalize = function canonicalize(x) {
+	var xType = typeof(x);
+	if (xType == "object") xType = x === null ? "null" : x.constructor.name;
+	switch (xType) {
+		case "MinKey":
+			return -1;
+		case "MaxKey":
+			return 127;
+		case "EOO":
+		case "undefined":
+		case undefined:
+			return 0;
+		case "jstNULL":
+		case "null":
+		case "Null":
+			return 5;
+		case "NumberDouble":
+		case "NumberInt":
+		case "NumberLong":
+		case "number":
+			return 10;
+		case "Symbol":
+		case "string":
+			return 15;
+		case "Object":
+			return 20;
+		case "Array":
+			return 25;
+		case "Binary":
+			return 30;
+		case "ObjectId":
+			return 35;
+		case "ObjectID":
+			return 35;
+		case "boolean":
+		case "Boolean":
+			return 40;
+		case "Date":
+		case "Timestamp":
+			return 45;
+		case "RegEx":
+		case "RegExp":
+			return 50;
+		case "DBRef":
+			return 55;
+		case "Code":
+			return 60;
+		case "CodeWScope":
+			return 65;
+		default:
+			// Default value for Object
+			return 20;
+	}
+};

lib/pipeline/accumulators/Accumulator.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * A base class for all pipeline accumulators. Uses NaryExpression as a base class.
  *
  * @class Accumulator
@@ -10,26 +10,26 @@
  **/
 var Accumulator = module.exports = function Accumulator(){
 	if (arguments.length !== 0) throw new Error("zero args expected");
+	this._memUsageBytes = 0;
 	base.call(this);
-}, klass = Accumulator, base = require("../expressions/NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = Accumulator, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
 // var Value = require("../Value"),
 
+proto.memUsageForSorter = function memUsageForSorter() {
+	return this._memUsageBytes;
+};
+
 proto.getFactory = function getFactory(){
 	return klass;	// using the ctor rather than a separate .create() method
 };
 
-/**
- * Adds the operand after checking the current limit
- * The equal is there because it checks *before* adding the requested argument.
- * Cannot use checkArgLimit because Accumulator must return a different error code.
- *
- * @param expr the operand to add
- **/
-proto.addOperand = function addOperand(expr) {
-	if (this.operands.length >= 1) throw new Error("code 15943; group accumulator " + this.getOpName() + " only accepts one operand");
-	base.prototype.addOperand.call(this, expr);
+/** Process input and update internal state.
+ * merging should be true when processing outputs from getValue(true).
+ */
+proto.process = function process(input, merging){
+	this.processInternal(input, merging);
 };
 
 proto.toJSON = function toJSON(isExpressionRequired){
@@ -39,47 +39,37 @@ proto.toJSON = function toJSON(isExpressionRequired){
 };
 
 /**
- * Convenience method for doing this for accumulators.  The pattern
- * is always the same, so a common implementation works, but requires
- * knowing the operator name.
+ * If this function is not overridden in the sub classes,
+ * then throw an error
  *
- * @param {Object} pBuilder the builder to add to
- * @param {String} fieldName the projected name
- * @param {String} opName the operator name
- * @param {Boolean} requireExpression pass down if the expression is needed
  **/
-//	proto.opToBson = function opToBson(pBuilder, opName, fieldName, requireExpression) {
-//		if (this.operands.length == 1) throw new Error("this should never happen");
-//		var builder = new BSONObjBuilder();
-//		this.operands[0].addToBsonObj(builder, opName, requireExpression);
-//		pBuilder.append(fieldName, builder.done());
-//	};
+proto.getOpName = function getOpName() {
+	throw new Error("You need to define this function on your accumulator");
+};
 
 /**
- * Wrapper around opToBson
+ * If this function is not overridden in the sub classes,
+ * then throw an error
  *
- * @param {Object} pBuilder the builder to add to
- * @param {String} fieldName the projected name
- * @param {Boolean} requireExpression pass down if the expression is needed
  **/
-//	proto.addToBsonObj = function addToBsonObj(pBuilder, fieldName, requireExpression) {
-//		this.opToBson(pBuilder, this.getOpName(), fieldName, requireExpression);
-//	};
+proto.getValue = function getValue(toBeMerged) {
+	throw new Error("You need to define this function on your accumulator");
+};
 
 /**
- * Make sure that nobody adds an accumulator to an array
+ * If this function is not overridden in the sub classes,
+ * then throw an error
  *
- * @param {Object} pBuilder the builder to add to
  **/
-proto.addToBsonArray = function addToBsonArray(pBuilder) {
-	if (false) throw new Error("this should never happen"); // these can't appear in arrays
+proto.processInternal = function processInternal(input, merging) {
+	throw new Error("You need to define this function on your accumulator");
 };
 
-/** 
+/**
  * If this function is not overridden in the sub classes,
  * then throw an error
  *
  **/
-proto.getValue = function getValue() {
+proto.reset = function reset() {
 	throw new Error("You need to define this function on your accumulator");
 };

lib/pipeline/accumulators/AddToSetAccumulator.js

@@ -1,21 +1,28 @@
 "use strict";
 
-/** 
+/**
  * Create an expression that finds the sum of n operands.
- * @class AddSoSetAccumulator
+ * @class AddToSetAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
  * @module mungedb-aggregate
  * @constructor
 **/
 var AddToSetAccumulator = module.exports = function AddToSetAccumulator(/* ctx */){
 	if (arguments.length !== 0) throw new Error("zero args expected");
-	this.set = {};
+	this.set = [];
 	//this.itr = undefined; /* Shoudln't need an iterator for the set */
 	//this.ctx = undefined; /* Not using the context object currently as it is related to sharding */
 	base.call(this);
 }, klass = AddToSetAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// PROTOTYPE MEMBERS
+// NOTE: Skipping the create function, using the constructor instead
+
+// DEPENDENCIES
+var Value = require("../Value");
+
+
+// MEMBER FUNCTIONS
+
 proto.getOpName = function getOpName(){
 	return "$addToSet";
 };
@@ -24,17 +31,29 @@ proto.getFactory = function getFactory(){
 	return klass;	// using the ctor rather than a separate .create() method
 };
 
-proto.evaluate = function evaluate(doc) {
-	if (arguments.length !== 1) throw new Error("One and only one arg expected");
-	var rhs = this.operands[0].evaluate(doc);
-	if (rhs === undefined) return;
-	this.set[JSON.stringify(rhs)] = rhs;
+
+proto.contains = function contains(value) {
+	var set = this.set;
+	for (var i = 0, l = set.length; i < l; ++i) {
+		if (Value.compare(set[i], value) === 0) {
+			return true;
+		}
+	}
+	return false;
 };
 
-proto.getValue = function getValue() {
-	var setValues = [];
-	for (var setKey in this.set) {
-		setValues.push(this.set[setKey]);
+proto.processInternal = function processInternal(input, merging) {
+	if (! this.contains(input)) {
+		this.set.push(input);
 	}
-	return setValues;
 };
+
+proto.getValue = function getValue(toBeMerged) {
+	return this.set;
+};
+
+proto.reset = function reset() {
+	this.set = [];
+};
+
+

lib/pipeline/accumulators/AvgAccumulator.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * A class for constructing accumulators to calculate avg.
  * @class AvgAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
@@ -11,23 +11,54 @@ var AvgAccumulator = module.exports = function AvgAccumulator(){
 	this.subTotalName = "subTotal";
 	this.countName = "count";
 	this.totalIsANumber = true;
+	this.total = 0;
+	this.count = 0;
 	base.call(this);
-}, klass = AvgAccumulator, SumAccumulator = require("./SumAccumulator"), base = SumAccumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = AvgAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
+// NOTE: Skipping the create function, using the constructor instead
+
+// DEPENDENCIES
+var Value = require("../Value");
+
+// MEMBER FUNCTIONS
+proto.processInternal = function processInternal(input, merging) {
+	if (!merging) {
+		if (typeof input !== "number") {
+			return;
+		}
+		this.total += input;
+		this.count += 1;
+	} else {
+		Value.verifyDocument(input);
+		this.total += input[this.subTotalName];
+		this.count += input[this.countName];
+	}
 };
 
-proto.getValue = function getValue(){
-	if (this.totalIsANumber && this.count > 0) {
-		return this.total / this.count;
-	} else if (this.count === 0) {
-		return 0;
+proto.getValue = function getValue(toBeMerged){
+	if (!toBeMerged) {
+		if (this.totalIsANumber && this.count > 0) {
+			return this.total / this.count;
+		} else if (this.count === 0) {
+			return 0;
+		} else {
+			throw new Error("$sum resulted in a non-numeric type");
+		}
 	} else {
-		throw new Error("$sum resulted in a non-numeric type");
+		var ret = {};
+		ret[this.subTotalName] = this.total;
+		ret[this.countName] = this.count;
+
+		return ret;
 	}
 };
 
+proto.reset = function reset() {
+	this.total = 0;
+	this.count = 0;
+};
+
 proto.getOpName = function getOpName(){
 	return "$avg";
 };

lib/pipeline/accumulators/FirstAccumulator.js

@@ -1,18 +1,22 @@
 "use strict";
 
-/** 
- * Constructor for FirstAccumulator, wraps SingleValueAccumulator's constructor and adds flag to track whether we have started or not
+/**
+ * Constructor for FirstAccumulator, wraps Accumulator and adds flag to track whether we have started or not
  * @class FirstAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
  * @module mungedb-aggregate
  * @constructor
  **/
 var FirstAccumulator = module.exports = function FirstAccumulator(){
+	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-	this.started = 0; //TODO: hack to get around falsy values making us keep going
-}, klass = FirstAccumulator, base = require("./SingleValueAccumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	this._haveFirst = false;
+	this._first = undefined;
+}, klass = FirstAccumulator, base = require("./Accumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// PROTOTYPE MEMBERS
+// NOTE: Skipping the create function, using the constructor instead
+
+// MEMBER FUNCTIONS
 proto.getOpName = function getOpName(){
 	return "$first";
 };
@@ -21,19 +25,23 @@ proto.getFactory = function getFactory(){
 	return klass;	// using the ctor rather than a separate .create() method
 };
 
-/** 
- * Takes a document and returns the first value in the document
- * @param {Object} doc the document source
- * @return the first value
- **/
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
 
-	// only remember the first value seen
-	if (!base.prototype.getValue.call(this) && this.started === 0) {
-		this.value = this.operands[0].evaluate(doc);
-		this.started = 1;
+proto.processInternal = function processInternal(input, merging) {
+	/* only remember the first value seen */
+	if (!this._haveFirst) {
+		// can't use pValue.missing() since we want the first value even if missing
+		this._haveFirst = true;
+		this._first = input;
+		//this._memUsageBytes = sizeof(*this) + input.getApproximateSize() - sizeof(Value);
 	}
+};
+
+proto.getValue = function getValue(toBeMerged) {
+	return this._first;
+};
 
-	return this.value;
+proto.reset = function reset() {
+	this._haveFirst = false;
+	this._first = undefined;
+	this._memUsageBytes = 0;
 };

lib/pipeline/accumulators/LastAccumulator.js

@@ -9,13 +9,24 @@
  **/
 var LastAccumulator = module.exports = function LastAccumulator(){
 	base.call(this);
-}, klass = LastAccumulator, SingleValueAccumulator = require("./SingleValueAccumulator"), base = SingleValueAccumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	this.value = undefined;
+}, klass = LastAccumulator, base = require("./Accumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
-	this.value = this.operands[0].evaluate(doc);
+// NOTE: Skipping the create function, using the constructor instead
+
+// MEMBER FUNCTIONS
+proto.processInternal = function processInternal(input, merging){
+	this.value = input;
+};
+
+proto.getValue = function getValue() {
+	return this.value;
 };
 
 proto.getOpName = function getOpName(){
 	return "$last";
 };
+
+proto.reset = function reset() {
+	this.value = undefined;
+};

lib/pipeline/accumulators/MinMaxAccumulator.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * Constructor for MinMaxAccumulator, wraps SingleValueAccumulator's constructor and adds flag to track whether we have started or not
  * @class MinMaxAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
@@ -12,12 +12,14 @@ var MinMaxAccumulator = module.exports = function MinMaxAccumulator(sense){
 	base.call(this);
 	this.sense = sense; /* 1 for min, -1 for max; used to "scale" comparison */
 	if (this.sense !== 1 && this.sense !== -1) throw new Error("this should never happen");
-}, klass = MinMaxAccumulator, base = require("./SingleValueAccumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = MinMaxAccumulator, base = require("./Accumulator"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// NOTE: Skipping the create function, using the constructor instead
 
 // DEPENDENCIES
 var Value = require("../Value");
 
-// PROTOTYPE MEMBERS
+// MEMBER FUNCTIONS
 proto.getOpName = function getOpName(){
 	if (this.sense == 1) return "$min";
 	return "$max";
@@ -31,22 +33,22 @@ klass.createMax = function createMax(){
 	return new MinMaxAccumulator(-1);
 };
 
-/** 
- * Takes a document and returns the first value in the document
- * @param {Object} doc the document source
- * @return the first value
- **/
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
-	var prhs = this.operands[0].evaluate(doc);
+proto.reset = function reset() {
+	this.value = undefined;
+};
+
+proto.getValue = function getValue(toBeMerged) {
+	return this.value;
+};
 
+proto.processInternal = function processInternal(input, merging) {
 	// if this is the first value, just use it
 	if (!this.hasOwnProperty('value')) {
-		this.value = prhs;
+		this.value = input;
 	} else {
 		// compare with the current value; swap if appropriate
-		var cmp = Value.compare(this.value, prhs) * this.sense;
-		if (cmp > 0) this.value = prhs;
+		var cmp = Value.compare(this.value, input) * this.sense;
+		if (cmp > 0) this.value = input;
 	}
 
 	return this.value;

lib/pipeline/accumulators/PushAccumulator.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * Constructor for PushAccumulator. Pushes items onto an array.
  * @class PushAccumulator
  * @namespace mungedb-aggregate.pipeline.accumulators
@@ -12,17 +12,40 @@ var PushAccumulator = module.exports = function PushAccumulator(){
 	base.call(this);
 }, klass = PushAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
-	var v = this.operands[0].evaluate(doc);
-	if (v !== undefined) this.values.push(v);
-	return null;
-};
+// NOTE: Skipping the create function, using the constructor instead
 
-proto.getValue = function getValue(){
+// MEMBER FUNCTIONS
+proto.getValue = function getValue(toBeMerged){
 	return this.values;
 };
 
 proto.getOpName = function getOpName(){
 	return "$push";
 };
+
+proto.getFactory = function getFactory(){
+	return klass;	// using the ctor rather than a separate .create() method
+};
+
+
+proto.processInternal = function processInternal(input, merging) {
+	if (!merging) {
+		if (input !== undefined) {
+			this.values.push(input);
+			//_memUsageBytes += input.getApproximateSize();
+		}
+	}
+	else {
+		// If we're merging, we need to take apart the arrays we
+		// receive and put their elements into the array we are collecting.
+		// If we didn't, then we'd get an array of arrays, with one array
+		// from each merge source.
+		if (!(input instanceof Array)) throw new Error("input is not an Array during merge in PushAccumulator:35");
+
+		this.values = this.values.concat(input);
+
+		//for (size_t i=0; i < vec.size(); i++) {
+			//_memUsageBytes += vec[i].getApproximateSize();
+		//}
+	}
+};

lib/pipeline/accumulators/SingleValueAccumulator.js

@@ -1,24 +0,0 @@
-"use strict";
-
-/**
- * This isn't a finished accumulator, but rather a convenient base class
- * for others such as $first, $last, $max, $min, and similar.  It just
- * provides a holder for a single Value, and the getter for that.  The
- * holder is protected so derived classes can manipulate it.
- *
- * @class SingleValueAccumulator
- * @namespace mungedb-aggregate.pipeline.accumulators
- * @module mungedb-aggregate
- * @constructor
-**/
-var SingleValueAccumulator = module.exports = function SingleValueAccumulator(){
-	if (arguments.length > 1) throw new Error("expects a single value");
-	base.call(this);
-}, klass = SingleValueAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
-
-// DEPENDENCIES
-var Value = require("../Value");
-
-proto.getValue = function getValue(){
-	return this.value;
-};

lib/pipeline/accumulators/SumAccumulator.js

@@ -14,22 +14,20 @@ var SumAccumulator = module.exports = function SumAccumulator(){
 	base.call(this);
 }, klass = SumAccumulator, Accumulator = require("./Accumulator"), base = Accumulator, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-proto.evaluate = function evaluate(doc){
-	if (this.operands.length != 1) throw new Error("this should never happen");
-	var v = this.operands[0].evaluate(doc);
+// NOTE: Skipping the create function, using the constructor instead
 
-	if (typeof v !== "number") { // do nothing with non-numeric types
-		return 0;
-	} else {
+// MEMBER FUNCTIONS
+proto.processInternal = function processInternal(input, merging) {
+	if(typeof input === "number"){ // do nothing with non-numeric types
 		this.totalIsANumber = true;
-		this.total += v;
+		this.total += input;
 	}
 	this.count++;
 
 	return 0;
 };
 
-proto.getValue = function getValue(){
+proto.getValue = function getValue(toBeMerged){
 	if (this.totalIsANumber) {
 		return this.total;
 	}

lib/pipeline/documentSources/CursorDocumentSource.js

@@ -1,5 +1,12 @@
 "use strict";
 
+var DocumentSource = require('./DocumentSource'),
+	LimitDocumentSource = require('./LimitDocumentSource');
+
+// Mimicking max memory size from mongo/db/query/new_find.cpp
+// Need to actually decide some size for this?
+var MAX_BATCH_DOCS = 150;
+
 /**
  * Constructs and returns Documents from the objects produced by a supplied Cursor.
  * An object of this type may only be used by one thread, see SERVER-6123.
@@ -30,6 +37,10 @@ var CursorDocumentSource = module.exports = CursorDocumentSource = function Curs
 	this._projection = null;
 
 	this._cursorWithContext = cursorWithContext;
+	this._curIdx = 0;
+	this._currentBatch = [];
+	this._limit = undefined;
+	this._docsAddedToBatches = 0;
 
 	if (!this._cursorWithContext || !this._cursorWithContext._cursor) throw new Error("CursorDocumentSource requires a valid cursorWithContext");
 
@@ -59,6 +70,43 @@ klass.CursorWithContext = (function (){
  **/
 proto.dispose = function dispose() {
 	this._cursorWithContext = null;
+	this._currentBatch = [];
+	this._curIdx = 0;
+};
+
+proto.getSourceName = function getSourceName() {
+	return "$cursor";
+};
+
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	if (this._currentBatch.length <= this._curIdx) {
+		this.loadBatch();
+
+		if (this._currentBatch.length <= this._curIdx) {
+			callback(null, DocumentSource.EOF);
+			return DocumentSource.EOF;
+		}
+	}
+
+	// Don't unshift. It's expensiver.
+	var out = this._currentBatch[this._curIdx];
+	this._curIdx++;
+
+	callback(null, out);
+	return out;
+};
+
+proto.coalesce = function coalesce(nextSource) {
+	if (this._limit) {
+		return this._limit.coalesce(nextSource);
+	} else if (nextSource instanceof LimitDocumentSource) {
+		this._limit = nextSource;
+		return this._limit;
+	} else {
+		return false;
+	}
 };
 
 ///**
@@ -78,9 +126,10 @@ proto.dispose = function dispose() {
 // * @method	setQuery
 // * @param	{Object}	pBsonObj	the query to record
 // **/
-//proto.setQuery = function setQuery(pBsonObj) {};
-//
-//
+proto.setQuery = function setQuery(query) {
+	this._query = query;
+};
+
 ///**
 // * Record the sort that was specified for the cursor this wraps, if any.
 // * This should be captured after any optimizations are applied to
@@ -98,7 +147,7 @@ proto.dispose = function dispose() {
  * @method	setProjection
  * @param	{Object}	projection
  **/
-proto.setProjection = function setProjection(projection) {
+proto.setProjection = function setProjection(projection, deps) {
 
 	if (this._projection){
 		throw new Error("projection is already set");
@@ -113,42 +162,10 @@ proto.setProjection = function setProjection(projection) {
 //	this.cursor().fields = this._projection;
 
 	this._projection = projection;  //just for testing
+	this._dependencies = deps;
 };
 
 //----------------virtuals from DocumentSource--------------
-/**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	if (!this.current) this.findNext(); // if we haven't gotten the first one yet, do so now
-	return (this.current === null);
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-	if (!this.current) this.findNext(); // if we haven't gotten the first one yet, do so now
-	this.findNext();
-	return (this.current !== null);
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	if (!this.current) this.findNext(); // if we haven't gotten the first one yet, do so now
-	return this.current;
-};
 
 /**
  * Set the underlying source this source should use to get Documents
@@ -165,74 +182,76 @@ proto.getCurrent = function getCurrent() {
  * @param source   {DocumentSource}  the underlying source to use
  * @param callback  {Function}        a `mungedb-aggregate`-specific extension to the API to half-way support reading from async sources
  **/
-proto.setSource = function setSource(theSource, callback) {
+proto.setSource = function setSource(theSource) {
 	if (theSource) throw new Error("CursorDocumentSource doesn't take a source"); //TODO: This needs to put back without the if once async is fully and properly supported
-	if (callback) return setImmediate(callback);
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToBsonArray()
- * to add this object to a pipeline being represented in BSON.
- *
- * @method	sourceToJson
- * @param	{Object} pBuilder	BSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(pBuilder, explain) {
-	/* this has no analog in the BSON world, so only allow it for explain */
-	//if (explain){
-	////we are not currently supporting explain in mungedb-aggregate
-	//}
+proto.serialize = function serialize(explain) {
+	if (!explain)
+		return null;
+
+	if (!this._cursorWithContext)
+		throw new Error("code 17135; Cursor deleted.");
+
+	// A stab at what mongo wants
+	return {
+		query: this._query,
+		sort: this._sort ? this._sort : null,
+		limit: this._limit ? this._limit : null,
+		fields: this._projection ? this._projection : null,
+		indexonly: false,
+		cursorType: this._cursorWithContext ? "cursor" : null
+	};
 };
 
-//----------------private--------------
+// LimitDocumentSource has the setLimit function which trickles down to any documentsource
+proto.getLimit = function getLimit() {
+	return this._limit ? this._limit.getLimit() : -1;
+};
 
-proto.findNext = function findNext(){
+//----------------private--------------
 
-	if ( !this._cursorWithContext ) {
-		this.current = null;
-		return;
-	}
+//proto.chunkMgr = function chunkMgr(){};
 
-	for( ; this.cursor().ok(); this.cursor().advance() ) {
+//proto.canUseCoveredIndex = function canUseCoveredIndex(){};
 
-		//yieldSometimes();
-//		if ( !this.cursor().ok() ) {
-//			// The cursor was exhausted during the yield.
-//			break;
-//		}
+//proto.yieldSometimes = function yieldSometimes(){};
 
-//		if ( !this.cursor().currentMatches() || this.cursor().currentIsDup() )
-//			continue;
+proto.loadBatch = function loadBatch() {
+	var nDocs = 0,
+		cursor = this._cursorWithContext ? this._cursorWithContext._cursor : null;
 
+	if (!cursor)
+		return this.dispose();
 
-		// grab the matching document
-		var documentObj;
-//		if (this.canUseCoveredIndex()) { ...  Dont need any of this, I think
+	for(;cursor.ok(); cursor.advance()) {
+		if (!cursor.ok())
+			break;
 
-		documentObj = this.cursor().current();
-		this.current = documentObj;
-		this.cursor().advance();
-		return;
-	}
+		// these methods do not exist
+		// if (!cursor.currentMatches() || cursor.currentIsDup())
+		// continue;
 
-	// If we got here, there aren't any more documents.
-	// The CursorWithContext (and its read lock) must be released, see SERVER-6123.
-	this.dispose();
-	this.current = null;
-};
+		var next = cursor.current();
+		this._currentBatch.push(this._projection ? base.documentFromJsonWithDeps(next, this._dependencies) : next);
 
-proto.cursor = function cursor(){
-	if( this._cursorWithContext && this._cursorWithContext._cursor){
-		return this._cursorWithContext._cursor;
-	}
-	throw new Error("cursor not defined");
-};
+		if (this._limit) {
+			this._docsAddedToBatches++;
+			if (this._docsAddedToBatches == this._limit.getLimit())
+				break;
 
-//proto.chunkMgr = function chunkMgr(){};
+			if (this._docsAddedToBatches >= this._limit.getLimit()) {
+				throw new Error("added documents to the batch over limit size");
+			}
+		}
 
-//proto.canUseCoveredIndex = function canUseCoveredIndex(){};
+		// Mongo uses number of bytes, but that doesn't make sense here. Yield when nDocs is over a threshold
+		if (nDocs > MAX_BATCH_DOCS) {
+			this._curIdx++; // advance the deque
+			nDocs++;
+			return;
+		}
+	}
 
-//proto.yieldSometimes = function yieldSometimes(){};
+	this._cursorWithContext = undefined;	//NOTE: Trying to emulate erasing the cursor; not exactly how mongo does it
+};

lib/pipeline/documentSources/DocumentSource.js

@@ -37,6 +37,22 @@ var DocumentSource = module.exports = function DocumentSource(expCtx){
 
 }, klass = DocumentSource, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
+/**
+ * Use EOF as boost::none for document sources to signal the end of their document stream.
+ **/
+klass.EOF = (function() {
+	/**
+	 * Represents a non-value in a document stream
+	 * @class EOF
+	 * @namespace mungedb-aggregate.pipeline.documentSources.DocumentSource
+	 * @module mungedb-aggregate
+	 * @constructor
+	 **/
+	var klass = function EOF(){
+	};
+	return klass;
+})();
+
 /*
 class DocumentSource :
 public IntrusiveCounterUnsigned,
@@ -67,32 +83,13 @@ proto.getPipelineStep = function getPipelineStep() {
 };
 
 /**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	throw new Error("not implemented");
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
+ * Returns the next Document if there is one or DocumentSource.EOF if at EOF.
  *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	//pExpCtx->checkForInterrupt(); // might not return
-	return false;
-};
-
-/**
  * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
+ * @method	getNext
  * @returns	{Document}	the current Document without advancing
  **/
-proto.getCurrent = function getCurrent() {
+proto.getNext = function getNext(callback) {
 	throw new Error("not implemented");
 };
 
@@ -136,10 +133,9 @@ proto.getSourceName = function getSourceName() {
  * @method	setSource
  * @param	{DocumentSource}	source	the underlying source to use
  **/
-proto.setSource = function setSource(theSource, callback) {
+proto.setSource = function setSource(theSource) {
 	if (this.source) throw new Error("It is an error to set the source more than once");
 	this.source = theSource;
-	if (callback) return setTimeout(callback, 0);
 };
 
 /**
@@ -189,71 +185,34 @@ proto.getDependencies = function getDependencies(deps) {
 	return klass.GetDepsReturn.NOT_SUPPORTED;
 };
 
-/**
- * This takes dependencies from getDependencies and
- * returns a projection that includes all of them
- *
- * @method	depsToProjection
- * @param	{Object} deps	set (unique array) of strings
- * @returns	{Object}	JSONObj
- **/
-klass.depsToProjection = function depsToProjection(deps) {
-	var bb = {};
-	if (deps._id === undefined)
-		bb._id = 0;
-
-	var last = "";
-	Object.keys(deps).sort().forEach(function(it){
-		if (last !== "" && it.slice(0, last.length) === last){
-			// we are including a parent of *it so we don't need to
-			// include this field explicitly. In fact, due to
-			// SERVER-6527 if we included this field, the parent
-			// wouldn't be fully included.
-			return;
-		}
-		last = it + ".";
-		bb[it] = 1;
-	});
-
-	return bb;
+proto._serialize = function _serialize(explain) {
+	throw new Error("not implemented");
 };
 
-/**
- * Add the DocumentSource to the array builder.
- * The default implementation calls sourceToJson() in order to
- * convert the inner part of the object which will be added to the
- * array being built here.
- *
- * @method	addToJsonArray
- * @param	{Array} pBuilder	JSONArrayBuilder: the array builder to add the operation to.
- * @param	{Boolean}	explain	create explain output
- * @returns	{Object}
- **/
-proto.addToJsonArray = function addToJsonArray(pBuilder, explain) {
-	pBuilder.push(this.sourceToJson({}, explain));
+proto.serializeToArray = function serializeToArray(array, explain) {
+	var entry = this.serialize(explain);
+	if (entry) {
+		array.push(entry);
+	}
 };
 
 /**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
+ * A function compatible as a getNext for document sources.
+ * Does nothing except pass the documents through. To use,
+ * Attach this function on a DocumentSource prototype.
  *
- * @method	sourceToJson
- * @param	{Object} pBuilder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(pBuilder, explain) {
-	throw new Error("not implemented");
-};
-
-/**
- * Convert the DocumentSource instance to it's JSON Object representation; Used by the standard JSON.stringify() function
- * @method toJSON
- * @return {String} a JSON-encoded String that represents the DocumentSource
- **/
-proto.toJSON = function toJSON(){
-	var obj = {};
-	this.sourceToJson(obj);
-	return obj;
+ * @method GET_NEXT_PASS_THROUGH
+ * @param callback {Function}
+ * @param callback.err {Error} An error or falsey
+ * @param callback.doc {Object} The source's next object or DocumentSource.EOF
+ **/
+klass.GET_NEXT_PASS_THROUGH = function GET_NEXT_PASS_THROUGH(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var out;
+	this.source.getNext(function(err, doc) {
+		out = doc;
+		return callback(err, doc);
+	});
+	return out; // For the sync people in da house
 };

lib/pipeline/documentSources/FilterBaseDocumentSource.js

@@ -1,114 +0,0 @@
-"use strict";
-
-/**
- * A base class for filter document sources
- * @class FilterBaseDocumentSource
- * @namespace mungedb-aggregate.pipeline.documentSources
- * @module mungedb-aggregate
- * @constructor
- * @param [ctx] {ExpressionContext}
- **/
-var FilterBaseDocumentSource = module.exports = function FilterBaseDocumentSource(ctx){
-	if (arguments.length > 1) throw new Error("up to one arg expected");
-	base.call(this, ctx);
-	this.unstarted = true;
-	this.hasNext = false;
-	this.current = null;
-}, klass = FilterBaseDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
-
-//TODO: Need to implement coalesce()
-//TODO: Need to implement optimize()
-
-/**
- * Find the next acceptable source document, if there are any left.
- * @method findNext
- **/
-proto.findNext = function findNext() {
-	/* only do this the first time */
-	if (this.unstarted) {
-		this.hasNext = !this.source.eof();
-		this.unstarted = false;
-	}
-
-	while(this.hasNext) {
-		var document = this.source.getCurrent();
-		this.hasNext = this.source.advance();
-
-		if (this.accept(document)) {
-			this.current = document;
-			return;
-		}
-	}
-
-	this.current = null;
-};
-
-/**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	if (this.unstarted)
-		this.findNext();
-	return (this.current === null);
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-
-	if (this.unstarted)
-		this.findNext();
-
-	/**
-	* This looks weird after the above, but is correct.  Note that calling
-	* getCurrent() when first starting already yields the first document
-	* in the collection.  Calling advance() without using getCurrent()
-	* first will skip over the first item.
-	**/
-	this.findNext();
-	return (this.current !== null);
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	if (this.unstarted)
-		this.findNext();
-	if (this.current === null) throw new Error("This should never happen");
-	return this.current;
-};
-
-/**
-* Test the given document against the predicate and report if it should be accepted or not.
-* @param {object} document the document to test
-* @returns {bool} true if the document matches the filter, false otherwise
-**/
-proto.accept = function accept(document) {
-	throw new Error("not implemented");
-};
-
-/**
-* Create a JSONObj suitable for Matcher construction.
-*
-* This is used after filter analysis has moved as many filters to
-* as early a point as possible in the document processing pipeline.
-* See db/Matcher.h and the associated wiki documentation for the
-* format.  This conversion is used to move back to the low-level
-* find() Cursor mechanism.
-*
-* @param builder the builder to write to
-**/
-proto.toMatcherJson = function toMatcherJson(builder) {
-	throw new Error("not implemented");
-};

lib/pipeline/documentSources/GeoNearDocumentSource.js

@@ -0,0 +1,131 @@
+"use strict";
+
+var async = require('async'),
+	DocumentSource = require('./DocumentSource'),
+	FieldPath = require('../FieldPath');
+
+/**
+ * @class GeoNearDocumentSource
+ * @namespace mungedb-aggregate.pipeline.documentSources
+ * @module mungedb-aggregate
+ * @constructor
+ * @param [ctx] {ExpressionContext}
+ **/
+var GeoNearDocumentSource = module.exports = function GeoNearDocumentSource(ctx) {
+	if (arguments.length > 1) throw new Error("up to one arg expected");
+	base.call(this, ctx);
+	// mongo defaults
+	this.coordsIsArray = false;
+	this.limit = 100;
+	this.maxDistance = -1.0;
+	this.spherical = false;
+	this.distanceMultiplier = 1.0;
+	this.uniqueDocs = true;
+}, klass = GeoNearDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+klass.geoNearName = "$geoNear";
+
+proto.getSourceName = function() {
+	return klass.geoNearName;
+};
+
+proto.getNext = DocumentSource.GET_NEXT_PASS_THROUGH;
+
+proto.setSource = function(docSource) {
+	throw new Error('code 16602; $geoNear is only allowed as the first pipeline stage');
+};
+
+proto.isValidInitialSource = function() {
+	return true;
+};
+
+proto.serialize = function(explain) {
+	var result = {};
+
+	if (this.coordsIsArray)
+		result.near = this.near;
+	else
+		result.near = [this.near];
+
+	// not in buildGeoNearCmd
+	result.distanceField = this.distanceField.getPath(false);
+
+	result.limit = this.limit;
+
+	if (this.maxDistance > 0)
+		result.maxDistance = this.maxDistance;
+
+	if (this.query)
+		result.query = this.query;
+
+	if (this.spherical)
+		result.spherical = this.spherical;
+
+	if (this.distanceMultiplier)
+		result.distanceMultiplier = this.distanceMultiplier;
+
+	if (this.includeLocs) {
+		if (typeof this.includeLocs !== 'string')
+			throw new Error('code 16607; $geoNear requires that \'includeLocs\' option is a String');
+		result.includeLocs = this.includeLocs.getPath(false);
+	}
+
+	if (this.uniqueDocs)
+		result.uniqueDocs = this.uniqueDocs;
+
+	var sourceName = this.getSourceName(),
+		returnObj = {};
+	returnObj[sourceName] = result;
+
+	return returnObj;
+};
+
+klass.createFromJson = function(jsonElement, ctx) {
+	var out = new GeoNearDocumentSource(ctx);
+	out.parseOptions(jsonElement);
+	return out;
+};
+
+proto.parseOptions = function(options) {
+
+	// near and distanceField are required
+	if (!options.near || !Array.isArray(options.near))
+		throw new Error('code 16605; $geoNear requires a \'near\' option as an Array');
+	this.coordsIsArray = options.near instanceof Array;
+
+	if (typeof options.distanceField !== 'string')
+		throw new Error('code 16606: $geoNear  a \'distanceNear\' option as a String');
+	this.distanceField = new FieldPath(options.distanceField);
+
+	// remaining fields are optional
+
+	// num and limits are synonyms
+	if (typeof options.limit === 'number')
+		this.limit = options.limit;
+	if (typeof options.num === 'number')
+		this.limit = options.num;
+
+	if (typeof options.maxDistance === 'number')
+		this.maxDistance = options.maxDistance;
+
+	if (options.query instanceof Object)
+		this.query = options.query;
+
+	if (options.spherical)
+		this.spherical = options.spherical;
+
+	if (typeof options.distanceMultiplier === 'number')
+		this.distanceMultiplier = options.distanceMultiplier;
+
+	if (options.includeLocs) {
+		if (typeof options.includeLocs !== 'string')
+			throw new Error('code 16607; $geoNear requires that \'includeLocs\' option is a String');
+		this.includeLocs = new FieldPath(options.includeLocs);
+	}
+
+	if (options.uniqueDocs)
+		this.uniqueDocs;
+};
+
+
+

lib/pipeline/documentSources/GroupDocumentSource.js

@@ -4,16 +4,20 @@ var DocumentSource = require("./DocumentSource"),
 	Document = require("../Document"),
 	Expression = require("../expressions/Expression"),
 	ConstantExpression = require("../expressions/ConstantExpression"),
-	FieldPathExpression = require("../expressions/FieldPathExpression");
-
+	FieldPathExpression = require("../expressions/FieldPathExpression"),
+	Variables = require("../expressions/Variables"),
+	VariablesIdGenerator = require("../expressions/VariablesIdGenerator"),
+	VariablesParseState = require("../expressions/VariablesParseState"),
+	async = require("async");
 
 /**
  * A class for grouping documents together
+ *
  * @class GroupDocumentSource
  * @namespace mungedb-aggregate.pipeline.documentSources
  * @module mungedb-aggregate
  * @constructor
- * @param [ctx] {ExpressionContext}
+ * @param [expCtx] {ExpressionContext}
  **/
 var GroupDocumentSource = module.exports = function GroupDocumentSource(expCtx) {
 	if (arguments.length > 1) throw new Error("up to one arg expected");
@@ -24,7 +28,7 @@ var GroupDocumentSource = module.exports = function GroupDocumentSource(expCtx)
 	this.groups = {}; // GroupsType Value -> Accumulators[]
 	this.groupsKeys = []; // This is to faciliate easier look up of groups
 	this.originalGroupsKeys = []; // This stores the original group key un-hashed/stringified/whatever
-
+	this._variables = null;
 	this.fieldNames = [];
 	this.accumulatorFactories = [];
 	this.expressions = [];
@@ -38,7 +42,7 @@ klass.groupOps = {
 	"$avg": Accumulators.Avg,
 	"$first": Accumulators.First,
 	"$last": Accumulators.Last,
-	"$max": Accumulators.MinMax.createMax,
+	"$max": Accumulators.MinMax.createMax, // $min and $max have special constructors because they share base features
 	"$min": Accumulators.MinMax.createMin,
 	"$push": Accumulators.Push,
 	"$sum": Accumulators.Sum
@@ -46,43 +50,142 @@ klass.groupOps = {
 
 klass.groupName = "$group";
 
+/**
+ * Factory for making GroupDocumentSources
+ *
+ * @method create
+ * @static
+ * @param [expCtx] {ExpressionContext}
+ **/
+klass.create = function create(expCtx) {
+	return new GroupDocumentSource(expCtx);
+};
+
+/**
+ * Factory for making GroupDocumentSources
+ *
+ * @method getSourceName
+ * @return {GroupDocumentSource}
+ **/
 proto.getSourceName = function getSourceName() {
 	return klass.groupName;
 };
 
 /**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
+ * Gets the next document or DocumentSource.EOF if none
  *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
+ * @method getNext
+ * @return {Object}
  **/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	var idExp = this.idExpression,
-		insides = {
-			_id: idExp ? idExp.toJSON() : {}
+proto.getNext = function getNext(callback) {
+	var self = this;
+	async.series([
+		function(next) {
+			if (!self.populated)
+				self.populate(function(err) {
+					return next(err);
+				});
+			else
+				return next();
 		},
-		aFac = this.accumulatorFactories,
-		aFacLen = aFac.length;
+		function(next) {
+			if(Object.keys(self.groups).length === 0) {
+				return next(null, DocumentSource.EOF);
+			}
+
+			//Note: Skipped the spilled logic
+
+			if(self.currentGroupsKeysIndex === self.groupsKeys.length) {
+				return next(null, DocumentSource.EOF);
+			}
+
+			var id = self.groupsKeys[self.currentGroupsKeysIndex],
+				accumulators = self.groups[id],
+				out = self.makeDocument(id, accumulators /*,mergeableOutput*/);
+
+			if(++self.currentGroupsKeysIndex === self.groupsKeys.length) {
+				self.dispose();
+			}
+
+			return next(null, out);
+		}
+	], function(err, results) {
+		callback(err, results[1]);
+	});
+};
+
+/**
+ * Sets this source as apparently empty
+ *
+ * @method dispose
+ **/
+proto.dispose = function dispose() {
+	//NOTE: Skipped 'freeing' our resources; at best we could remove some references to things, but our parent will probably forget us anyways!
 
-	for(var i=0; i < aFacLen; ++i) {
-		var acc = new aFac[i](/*pExpCtx*/);
-		acc.addOperand(this.expressions[i]);
+	// make us look done
+	this.currentGroupsKeysIndex = this.groupsKeys.length;
 
-		insides[this.fieldNames[i]] = acc.toJSON(true);
+	// free our source's resources
+	this.source.dispose();
+};
+
+/**
+ * Optimizes the expressions in the group
+ * @method optimize
+ **/
+proto.optimize = function optimize() {
+	var self = this;
+	self.idExpression = self.idExpression.optimize();
+	self.expressions.forEach(function(expression, i) {
+		self.expressions[i] = expression.optimize();
+	});
+};
+
+/**
+ * Create an object that represents the document source.  The object
+ * will have a single field whose name is the source's name.
+ *
+ * @method	serialize
+ * @param explain {Boolean} Create explain output
+ **/
+proto.serialize = function serialize(explain) {
+	var insides = {};
+
+	// add the _id
+	insides._id = this.idExpression.serialize(explain);
+
+	//add the remaining fields
+	var aFacs = this.accumulatorFactories,
+		aFacLen = aFacs.length;
+
+	for(var i=0; i < aFacLen; i++) {
+		var aFac = aFacs[i](),
+			serialExpression = this.expressions[i].serialize(explain), //Get the accumulator's expression
+			serialAccumulator = {}; //Where we'll put the expression
+		serialAccumulator[aFac.getOpName()] = serialExpression;
+		insides[this.fieldNames[i]] = serialAccumulator;
 	}
 
-	builder[this.getSourceName()] = insides;
+	var serialSource = {};
+	serialSource[this.getSourceName()] = insides;
+	return serialSource;
 };
 
-klass.createFromJson = function createFromJson(groupObj, ctx) {
-	if (!(groupObj instanceof Object && groupObj.constructor === Object)) throw new Error("a group's fields must be specified in an object");
+/**
+ * Creates a GroupDocumentSource from the given elem
+ *
+ * @method	createFromJson
+ * @param elem {Object} The group specification object; the right hand side of the $group
+ **/
+klass.createFromJson = function createFromJson(elem, expCtx) {
+	if (!(elem instanceof Object && elem.constructor === Object)) throw new Error("a group's fields must be specified in an object");
+
+	var group = GroupDocumentSource.create(expCtx),
+		idSet = false;
 
-	var idSet = false,
-		group = new GroupDocumentSource(ctx);
+	var groupObj = elem,
+		idGenerator = new VariablesIdGenerator(),
+		vps = new VariablesParseState(idGenerator);
 
 	for (var groupFieldName in groupObj) {
 		if (groupObj.hasOwnProperty(groupFieldName)) {
@@ -93,106 +196,168 @@ klass.createFromJson = function createFromJson(groupObj, ctx) {
 				if(idSet) throw new Error("15948 a group's _id may only be specified once");
 
 				if (groupField instanceof Object && groupField.constructor === Object) {
+					/*
+						Use the projection-like set of field paths to create the
+						group-by key.
+					*/
 					var objCtx = new Expression.ObjectCtx({isDocumentOk:true});
-					group.idExpression = Expression.parseObject(groupField, objCtx);
+					group.setIdExpression(Expression.parseObject(groupField, objCtx, vps));
 					idSet = true;
 
 				} else if (typeof groupField === "string") {
-					if (groupField[0] !== "$") {
-						group.idExpression = new ConstantExpression(groupField);
-					} else {
-						var pathString = Expression.removeFieldPrefix(groupField);
-						group.idExpression = new FieldPathExpression(pathString);
-					}
-					idSet = true;
-
-				} else {
-					var typeStr = group._getTypeStr(groupField);
-					switch (typeStr) {
-						case "number":
-						case "string":
-						case "boolean":
-						case "Object":
-						case "object": // null returns "object" Xp
-						case "Array":
-							group.idExpression = new ConstantExpression(groupField);
-							idSet = true;
-							break;
-						default:
-							throw new Error("a group's _id may not include fields of type " + typeStr  + "");
+					if (groupField[0] === "$") {
+						group.setIdExpression(FieldPathExpression.parse(groupField, vps));
+						idSet = true;
 					}
 				}
 
+				if (!idSet) {
+					// constant id - single group
+					group.setIdExpression(ConstantExpression.create(groupField));
+					idSet = true;
+				}
 
 			} else {
+				/*
+					Treat as a projection field with the additional ability to
+					add aggregation operators.
+				*/
 				if (groupFieldName.indexOf(".") !== -1) throw new Error("16414 the group aggregate field name '" + groupFieldName + "' cannot contain '.'");
 				if (groupFieldName[0] === "$") throw new Error("15950 the group aggregate field name '" + groupFieldName + "' cannot be an operator name");
 				if (group._getTypeStr(groupFieldName) === "Object") throw new Error("15951 the group aggregate field '" + groupFieldName + "' must be defined as an expression inside an object");
 
-				var subFieldCount = 0;
-				for (var subFieldName in groupField) {
-					if (groupField.hasOwnProperty(subFieldName)) {
-						var subField = groupField[subFieldName],
-							op = klass.groupOps[subFieldName];
-						if (!op) throw new Error("15952 unknown group operator '" + subFieldName + "'");
+				var subElementCount = 0;
+				for (var subElementName in groupField) {
+					if (groupField.hasOwnProperty(subElementName)) {
+						var subElement = groupField[subElementName],
+							op = klass.groupOps[subElementName];
+						if (!op) throw new Error("15952 unknown group operator '" + subElementName + "'");
 
 						var groupExpression,
-							subFieldTypeStr = group._getTypeStr(subField);
-						if (subFieldTypeStr === "Object") {
-							var subFieldObjCtx = new Expression.ObjectCtx({isDocumentOk:true});
-							groupExpression = Expression.parseObject(subField, subFieldObjCtx);
-						} else if (subFieldTypeStr === "Array") {
-							throw new Error("15953 aggregating group operators are unary (" + subFieldName + ")");
-						} else {
-							groupExpression = Expression.parseOperand(subField);
+							subElementTypeStr = group._getTypeStr(subElement);
+						if (subElementTypeStr === "Object") {
+							var subElementObjCtx = new Expression.ObjectCtx({isDocumentOk:true});
+							groupExpression = Expression.parseObject(subElement, subElementObjCtx, vps);
+						} else if (subElementTypeStr === "Array") {
+							throw new Error("15953 aggregating group operators are unary (" + subElementName + ")");
+						} else { /* assume its an atomic single operand */
+							groupExpression = Expression.parseOperand(subElement, vps);
 						}
-						group.addAccumulator(groupFieldName,op, groupExpression);
+						group.addAccumulator(groupFieldName, op, groupExpression);
 
-						++subFieldCount;
+						++subElementCount;
 					}
 				}
-				if (subFieldCount != 1) throw new Error("15954 the computed aggregate '" + groupFieldName + "' must specify exactly one operator");
+				if (subElementCount !== 1) throw new Error("15954 the computed aggregate '" + groupFieldName + "' must specify exactly one operator");
 			}
 		}
 	}
 
 	if (!idSet) throw new Error("15955 a group specification must include an _id");
 
+	group._variables = new Variables(idGenerator.getIdCount());
+
 	return group;
 };
 
-proto._getTypeStr = function _getTypeStr(obj) {
-	var typeofStr = typeof obj,
-		typeStr = (typeofStr == "object" && obj !== null) ? obj.constructor.name : typeofStr;
-	return typeStr;
-};
+/**
+ * Populates the GroupDocumentSource by grouping all of the input documents at once.
+ *
+ * @method populate
+ * @param callback {Function} Required. callback(err) when done populating.
+ * @async
+ **/
+proto.populate = function populate(callback) {
+	var numAccumulators = this.accumulatorFactories.length;
+	if(numAccumulators !== this.expressions.length) {
+		callback(new Error("Must have equal number of accumulators and expressions"));
+	}
 
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // Check for interupts ????
-	if(!this.populated) this.populate();
+	var input,
+		self = this;
+	async.whilst(
+		function() {
+			return input !== DocumentSource.EOF;
+		},
+		function(cb) {
+			self.source.getNext(function(err, doc) {
+				if(err) return cb(err);
+				if(doc === DocumentSource.EOF) {
+					input = doc;
+					return cb(); //Need to stop now, no new input
+				}
 
-	//verify(this.currentGroupsKeysIndex < this.groupsKeys.length);
+				input = doc;
+				self._variables.setRoot(input);
 
-	++this.currentGroupsKeysIndex;
-	if (this.currentGroupsKeysIndex >= this.groupsKeys.length) {
-		this.currentDocument = null;
-		return false;
-	}
+				/* get the _id value */
+				var id = self.idExpression.evaluate(self._variables);
 
-	this.currentDocument = this.makeDocument(this.currentGroupsKeysIndex);
-	return true;
-};
+				if(undefined === id) id = null;
+
+				var groupKey = JSON.stringify(id),
+					group = self.groups[JSON.stringify(id)];
+
+				if(!group) {
+					self.groupsKeys.push(groupKey);
+					group = [];
+					self.groups[groupKey] = group;
+					// Add the accumulators
+					for(var afi = 0; afi<self.accumulatorFactories.length; afi++) {
+						group.push(self.accumulatorFactories[afi]());
+					}
+				}
+				//NOTE: Skipped memory usage stuff for case when group already existed
+
+				if(numAccumulators !== group.length) {
+					throw new Error('Group must have one of each accumulator');
+				}
+
+				//NOTE: passing the input to each accumulator
+				for(var gi=0; gi<group.length; gi++) {
+					group[gi].process(self.expressions[gi].evaluate(self._variables /*, doingMerge*/));
+				}
+
+				// We are done with the ROOT document so release it.
+				self._variables.clearRoot();
 
-proto.eof = function eof() {
-	if (!this.populated) this.populate();
-	return this.currentGroupsKeysIndex === this.groupsKeys.length;
+				//NOTE: Skipped the part about sorted files
+
+				return cb();
+			});
+		},
+		function(err) {
+			if(err) return callback(err);
+
+			self.populated = true;
+
+			return callback();
+		}
+	);
 };
 
-proto.getCurrent = function getCurrent() {
-	if (!this.populated) this.populate();
-	return this.currentDocument;
+/**
+ * Get the type of something. Handles objects specially to return their true type; i.e. their constructor
+ *
+ * @method populate
+ * @param obj {Object} The object to get the type of
+ * @return {String} The type of the object as a string
+ * @async
+ **/
+proto._getTypeStr = function _getTypeStr(obj) {
+	var typeofStr = typeof obj,
+		typeStr = (typeofStr == "object" && obj !== null) ? obj.constructor.name : typeofStr;
+	return typeStr;
 };
 
+/**
+ * Get the dependencies of the group
+ *
+ * @method getDependencies
+ * @param deps {Object} The
+ * @return {DocumentSource.getDepsReturn} An enum value specifying that these dependencies are exhaustive
+ * @async
+ **/
 proto.getDependencies = function getDependencies(deps) {
 	var self = this;
 	// add _id
@@ -205,67 +370,53 @@ proto.getDependencies = function getDependencies(deps) {
 	return DocumentSource.GetDepsReturn.EXHAUSTIVE;
 };
 
+/**
+ * Called internally only. Adds an accumulator for each matching group.
+ *
+ * @method addAccumulator
+ * @param fieldName {String} The name of the field where the accumulated value will be placed
+ * @param accumulatorFactory {Accumulator} The constructor for creating accumulators
+ * @param epxression {Expression} The expression to be evaluated on incoming documents before they are accumulated
+ **/
 proto.addAccumulator = function addAccumulator(fieldName, accumulatorFactory, expression) {
 	this.fieldNames.push(fieldName);
 	this.accumulatorFactories.push(accumulatorFactory);
 	this.expressions.push(expression);
 };
 
-proto.populate = function populate() {
-	for (var hasNext = !this.source.eof(); hasNext; hasNext = this.source.advance()) {
-		var group,
-			currentDocument = this.source.getCurrent(),
-			_id = this.idExpression.evaluate(currentDocument);
-
-		if (undefined === _id) _id = null;
+/**
+ * Makes a document with the given id and accumulators
+ *
+ * @method makeDocument
+ * @param fieldName {String} The name of the field where the accumulated value will be placed
+ * @param accums {Array} An array of accumulators
+ * @param epxression {Expression} The expression to be evaluated on incoming documents before they are accumulated
+ **/
+proto.makeDocument = function makeDocument(id, accums /*,mergeableOutput*/) {
+	var out = {};
 
-		var idHash = JSON.stringify(_id); //TODO: USE A REAL HASH.  I didn't have time to take collision into account.
+	/* add the _id field */
+	out._id = id;
 
-		if (idHash in this.groups) {
-			group = this.groups[idHash];
+	/* add the rest of the fields */
+	this.fieldNames.forEach(function(fieldName, i) {
+		var val = accums[i].getValue(/*mergeableOutput*/);
+		if(!val) {
+			out[fieldName] = null;
 		} else {
-			this.groups[idHash] = group = [];
-			this.groupsKeys[this.currentGroupsKeysIndex] = idHash;
-			this.originalGroupsKeys[this.currentGroupsKeysIndex] = (_id && typeof _id === 'object') ? Document.clone(_id) : _id;
-			++this.currentGroupsKeysIndex;
-			for (var ai = 0; ai < this.accumulatorFactories.length; ++ai) {
-				var accumulator = new this.accumulatorFactories[ai]();
-				accumulator.addOperand(this.expressions[ai]);
-				group.push(accumulator);
-			}
-		}
-
-
-		// tickle all the accumulators for the group we found
-		for (var gi = 0; gi < group.length; ++gi) {
-			group[gi].evaluate(currentDocument);
+			out[fieldName] = val;
 		}
+	});
 
-	}
-
-	this.currentGroupsKeysIndex = 0; // Start the group
-	if (this.groupsKeys.length > 0) {
-		this.currentDocument = this.makeDocument(this.currentGroupsKeysIndex);
-	}
-	this.populated = true;
-
+	return out;
 };
 
-proto.makeDocument = function makeDocument(groupKeyIndex) {
-	var groupKey = this.groupsKeys[groupKeyIndex],
-		originalGroupKey = this.originalGroupsKeys[groupKeyIndex],
-		group = this.groups[groupKey],
-		doc = {};
-
-	doc[Document.ID_PROPERTY_NAME] = originalGroupKey;
-
-	for (var i = 0; i < this.fieldNames.length; ++i) {
-		var fieldName = this.fieldNames[i],
-			item = group[i];
-		if (item !== "null" && item !== undefined) {
-			doc[fieldName] = item.getValue();
-		}
-	}
-
-	return doc;
+/**
+ * Sets the id expression for the group
+ *
+ * @method setIdExpression
+ * @param epxression {Expression} The expression to set
+ **/
+proto.setIdExpression = function setIdExpression(expression) {
+	this.idExpression = expression;
 };

lib/pipeline/documentSources/LimitDocumentSource.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var DocumentSource = require('./DocumentSource');
+
 /**
  * A document source limiter
  * @class LimitDocumentSource
@@ -35,60 +37,23 @@ proto.coalesce = function coalesce(nextSource) {
 
 	// if it's not another $limit, we can't coalesce
 	if (!nextLimit) return false;
-	
+
 	// we need to limit by the minimum of the two limits
 	if (nextLimit.limit < this.limit) this.limit = nextLimit.limit;
 
 	return true;
 };
 
-/**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	return this.source.eof() || this.count >= this.limit;
-};
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
 
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	return this.source.getCurrent();
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-	++this.count;
-	if (this.count >= this.limit) {
-		return false;
+	if (++this.count > this.limit) {
+		this.source.dispose();
+		callback(null, DocumentSource.EOF);
+		return DocumentSource.EOF;
 	}
-	this.current = this.source.getCurrent();
-	return this.source.advance();
-};
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	builder.$limit = this.limit;
+	return this.source.getNext(callback);
 };
 
 /**
@@ -106,3 +71,21 @@ klass.createFromJson = function createFromJson(jsonElement, ctx) {
 
 	return nextLimit;
 };
+
+proto.getLimit = function getLimit(newLimit) {
+	return this.limit;
+};
+
+proto.setLimit = function setLimit(newLimit) {
+	this.limit = newLimit;
+};
+
+proto.getDependencies = function(deps) {
+	return DocumentSource.GetDepsReturn.SEE_NEXT;
+};
+
+proto.serialize = function(explain) {
+	var out = {};
+	out[this.getSourceName()] = this.limit;
+	return out;
+};

lib/pipeline/documentSources/MatchDocumentSource.js

@@ -1,8 +1,10 @@
 "use strict";
-var sift = require("sift");	//TODO: DEVIATION FROM MONGO: this was a temporary hack to get this done quickly but it is too inconsistent to keep; need a real port of MatchDocumentSource
+var async = require("async"),
+	matcher = require("../matcher/Matcher2.js"),
+	DocumentSource = require("./DocumentSource");
 
 /**
- * A match document source built off of FilterBaseDocumentSource
+ * A match document source built off of DocumentSource
  *
  * NOTE: THIS IS A DEVIATION FROM THE MONGO IMPLEMENTATION.
  * TODO: internally uses `sift` to fake it, which has bugs, so we need to reimplement this by porting the MongoDB implementation
@@ -19,8 +21,8 @@ var MatchDocumentSource = module.exports = function MatchDocumentSource(query, c
 	if (!query) throw new Error("arg `query` is required");
 	base.call(this, ctx);
 	this.query = query; // save the query, so we can check it for deps later. THIS IS A DEVIATION FROM THE MONGO IMPLEMENTATION
-	this.matcher = sift(query);
-}, klass = MatchDocumentSource, base = require('./FilterBaseDocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	this.matcher = new matcher(query);
+}, klass = MatchDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 klass.matchName = "$match";
 
@@ -28,56 +30,56 @@ proto.getSourceName = function getSourceName(){
 	return klass.matchName;
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	builder[this.getSourceName()] = this.matcher.query;
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		next,
+		test = function test(doc) {
+			return self.matcher.matches(doc);
+		},
+		makeReturn = function makeReturn(doc) {
+			if(doc !== DocumentSource.EOF && test(doc)) { // Passes the match criteria
+				return doc;
+			} else if(doc === DocumentSource.EOF){ // Got EOF
+				return doc;
+			}
+			return undefined; // Didn't match, but not EOF
+		};
+	async.doUntil(
+		function(cb) {
+			self.source.getNext(function(err, doc) {
+				if(err) return callback(err);
+				if (makeReturn(doc)) {
+					next = doc;
+				}
+				return cb();
+			});
+		},
+		function() {
+			var foundDoc = (next === DocumentSource.EOF || next !== undefined);
+			return foundDoc; //keep going until doc is found
+		},
+		function(err) {
+			return callback(err, next);
+		}
+	);
+	return next;
 };
 
-/**
- *  Test the given document against the predicate and report if it should be accepted or not.
- * @param {object} document the document to test
- * @returns {bool} true if the document matches the filter, false otherwise
- **/
-proto.accept = function accept(document) {
-	/**
-	* The matcher only takes BSON documents, so we have to make one.
-	*
-	* LATER
-	* We could optimize this by making a document with only the
-	* fields referenced by the Matcher.  We could do this by looking inside
-	* the Matcher's BSON before it is created, and recording those.  The
-	* easiest implementation might be to hold onto an ExpressionDocument
-	* in here, and give that pDocument to create the created subset of
-	* fields, and then convert that instead.
-	**/
-	return this.matcher.test(document);
+proto.coalesce = function coalesce(nextSource) {
+	if (!(nextSource instanceof MatchDocumentSource))
+		return false;
+
+	this.matcher = new matcher({"$and": [this.getQuery(), nextSource.getQuery()]});
+
+	return true;
 };
 
-/**
- * Create a JSONObj suitable for Matcher construction.
- *
- * This is used after filter analysis has moved as many filters to
- * as early a point as possible in the document processing pipeline.
- * See db/Matcher.h and the associated wiki documentation for the
- * format.  This conversion is used to move back to the low-level
- * find() Cursor mechanism.
- *
- * @param builder the builder to write to
- **/
-proto.toMatcherJson = function toMatcherJson(builder) {
-	var q = this.matcher.query;
-	for(var k in q){
-		builder[k] = q[k];
-	}
+proto.serialize = function(explain) {
+	var out = {};
+	out[this.getSourceName()] = this.getQuery();
+	return out;
 };
 
 klass.uassertNoDisallowedClauses = function uassertNoDisallowedClauses(query) {
@@ -100,3 +102,188 @@ klass.createFromJson = function createFromJson(jsonElement, ctx) {
 	var matcher = new MatchDocumentSource(jsonElement, ctx);
 	return matcher;
 };
+
+proto.getQuery = function getQuery() {
+	return this.matcher._pattern;
+};
+
+/** Returns the portion of the match that can safely be promoted to before a $redact.
+ * If this returns an empty BSONObj, no part of this match may safely be promoted.
+ *
+ * To be safe to promote, removing a field from a document to be matched must not cause
+ * that document to be accepted when it would otherwise be rejected. As an example,
+ * {name: {$ne: "bob smith"}} accepts documents without a name field, which means that
+ * running this filter before a redact that would remove the name field would leak
+ * information. On the other hand, {age: {$gt:5}} is ok because it doesn't accept documents
+ * that have had their age field removed.
+ */
+proto.redactSafePortion = function redactSafePortion() {
+	var self = this;
+
+	// This block contains the functions that make up the implementation of
+	// DocumentSourceMatch::redactSafePortion(). They will only be called after
+	// the Match expression has been successfully parsed so they can assume that
+	// input is well formed.
+
+	var isAllDigits = function(n) {
+		return !isNaN(n);
+	};
+
+	var isFieldnameRedactSafe = function isFieldnameRedactSafe(field) {
+		var dotPos = field.indexOf('.');
+		if (dotPos === -1)
+			return !isAllDigits(field);
+
+		var part = field.slice(0, dotPos),
+			rest = field.slice(dotPos+1, field.length);
+
+		return !isAllDigits(part) && isFieldnameRedactSafe(rest);
+	};
+
+	// Returns the redact-safe portion of an "inner" match expression. This is the layer like
+	// {$gt: 5} which does not include the field name. Returns an empty document if none of the
+	// expression can safely be promoted in front of a $redact.
+	var redactSavePortionDollarOps = function redactSafePortionDollarOps(expr) {
+		var output = {},
+			elem,i,j,k;
+
+		var keys = Object.keys(expr);
+		for (i = 0; i < keys.length; i++) {
+			var field = keys[i],
+				value = expr[field];
+
+			if (field[0] !== '$')
+				continue;
+
+			// Ripped the case apart and did not implement this painful thing:
+			// https://github.com/mongodb/mongo/blob/r2.5.4/src/mongo/db/jsobj.cpp#L286
+			// Somebody should be taken to task for that work of art.
+			if (field === '$type' || field === '$regex' || field === '$options' || field === '$mod') {
+				output[field] = value;
+			} else if (field === '$lte' || field === '$gte' || field === '$lt' || field === '$gt') {
+				if (isTypeRedactSafeInComparison(field))
+					output[field] = value;
+			} else if (field === '$in') {
+				// TODO: value/elem/field/etc may be mixed up and wrong here
+				var allOk = true;
+				for (j = 0; j < Object.keys(value).length; j++) {
+					elem = Object.keys(value)[j];
+					if (!isTypeRedactSafeInComparison(value[elem])) {
+						allOk = false;
+						break;
+					}
+				}
+				if (allOk) {
+					output[field] = value;
+				}
+				break;
+			} else if (field === '$all') {
+				// TODO: value/elem/field/etc may be mixed up and wrong here
+				var matches = [];
+				for (j = 0; j < value.length; j++) {
+					elem = Object.keys(value)[j];
+					if (isTypeRedactSafeInComparison(value[elem]))
+						matches.push(value[elem]);
+				}
+				if (matches.length)
+					output[field] = matches;
+
+			} else if (field === '$elemMatch') {
+				var subIn = value,
+					subOut;
+
+				if (subIn[0] === '$')
+					subOut = redactSafePortionDollarOps(subIn);
+				else
+					subOut = redactSafePortionTopLevel(subIn);
+
+				if (subOut && Object.keys(subOut).length)
+					output[field] = subOut;
+
+				break;
+			} else {
+				// never allowed:
+				// equality, maxDist, near, ne, opSize, nin, exists, within, geoIntersects
+				continue;
+			}
+		}
+
+		return output;
+	};
+
+	var isTypeRedactSafeInComparison = function isTypeRedactSafeInComparison(type) {
+		if (type instanceof Array || (type instanceof Object && type.constructor === Object) || type === null || type === undefined)
+			return false;
+		return true;
+	};
+
+	// Returns the redact-safe portion of an "outer" match expression. This is the layer like
+	// {fieldName: {...}} which does include the field name. Returns an empty document if none of
+	// the expression can safely be promoted in front of a $redact.
+	var redactSafePortionTopLevel = function(topQuery) {
+		var output = {},
+			okClauses = [],
+			keys = topQuery ? Object.keys(topQuery) : [],
+			j, elm, clause;
+
+		for (var i = 0; i < keys.length; i++) {
+			var field = keys[i],
+				value = topQuery[field];
+
+			if (field.length && field[0] === '$') {
+				if (field === '$or') {
+					okClauses = [];
+					for (j = 0; j < Object.keys(value).length; j++) {
+						elm = value[Object.keys(value)[j]];
+						clause = redactSafePortionTopLevel(elm);
+
+						if (!clause || Object.keys(clause).length === 0) {
+							okClauses = [];
+							break;
+						}
+
+						okClauses.push(clause);
+					}
+
+					if (okClauses && okClauses.length) {
+						output.$or = okClauses;
+					}
+				} else if (field === '$and') {
+					okClauses = [];
+					for (j = 0; j < Object.keys(value).length; j++) {
+						elm = value[Object.keys(value)[j]];
+						clause = redactSafePortionTopLevel(elm);
+
+						if (clause && Object.keys(clause).length)
+							okClauses.push(clause);
+					}
+
+					if (okClauses.length)
+						output.$and = okClauses;
+				}
+
+				continue;
+			}
+
+			if (!isFieldnameRedactSafe(field))
+					continue;
+
+			if (value instanceof Array || !value) {
+				continue;
+			} else if (value instanceof Object && value.constructor === Object) {
+				// subobjects (not regex etc)
+				var sub = redactSavePortionDollarOps(value);
+				if (sub && Object.keys(sub).length)
+					output[field] = sub;
+
+				break;
+			} else {
+				output[field] = value;
+			}
+		}
+
+		return output;
+	};
+
+	return redactSafePortionTopLevel(this.getQuery());
+};

lib/pipeline/documentSources/OutDocumentSource.js

@@ -0,0 +1,53 @@
+"use strict";
+
+var DocumentSource = require('./DocumentSource');
+
+/**
+ * @class OutDocumentSource
+ * @namespace mungedb-aggregate.pipeline.documentSources
+ * @module mungedb-aggregate
+ * @constructor
+ * @param [ctx] {ExpressionContext}
+ **/
+var OutDocumentSource = module.exports = function OutDocumentSource(outputNs, ctx){
+	if (arguments.length > 2) throw new Error("up to two arg expected");
+	base.call(this, ctx);
+	// defaults
+	this._done = false;
+	this._outputNs = outputNs;
+	this._collectionName = "";
+}, klass = OutDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+klass.outName = "$out";
+
+proto.getSourceName = function() {
+	return klass.outName;
+};
+
+proto.getNext = DocumentSource.GET_NEXT_PASS_THROUGH;
+
+proto.serialize = function(explain) {
+	var doc = {},
+		srcNm = this.getSourceName();
+	doc[srcNm] = this._collectionName;
+	return doc;
+};
+
+proto.getOutputNs = function() {
+	return this._outputNs;
+};
+
+klass.createFromJson = function(jsonElement, ctx) {
+	if (typeof jsonElement !== 'string')
+		throw new Error('code 16990; $out only supports a string argument, not ' + typeof jsonElement);
+
+	var out = new OutDocumentSource(null, ctx); // TODO: outputNs
+	out._collectionName = jsonElement;
+
+	return out;
+};
+
+proto.getDependencies = function(deps) {
+	deps.needWholeDocument = true;
+	return DocumentSource.GetDepsReturn.EXHAUSTIVE;
+};

lib/pipeline/documentSources/ProjectDocumentSource.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var DocumentSource = require('./DocumentSource');
+
 /**
  * A base class for filter document sources
  * @class ProjectDocumentSource
@@ -16,9 +18,12 @@ var ProjectDocumentSource = module.exports = function ProjectDocumentSource(ctx)
 }, klass = ProjectDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
-var Expression = require('../expressions/Expression');
-var ObjectExpression = require('../expressions/ObjectExpression');
-var Value = require('../Value');
+var Expression = require('../expressions/Expression'),
+	ObjectExpression = require('../expressions/ObjectExpression'),
+	Value = require('../Value'),
+	Variables = require('../expressions/Variables'),
+	VariablesIdGenerator = require('../expressions/VariablesIdGenerator'),
+	VariablesParseState = require('../expressions/VariablesParseState');
 
 klass.projectName = "$project";
 
@@ -30,6 +35,39 @@ proto.getSourceName = function getSourceName() {
 	return klass.projectName;
 };
 
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		out;
+
+	this.source.getNext(function(err, input) {
+		if (err)
+			return callback(null, err);
+
+		if (input === DocumentSource.EOF) {
+			out = input;
+			return callback(null, DocumentSource.EOF);
+		}
+
+		/* create the result document */
+		out = {};
+
+		/**
+		 * Use the ExpressionObject to create the base result.
+		 *
+		 * If we're excluding fields at the top level, leave out the _id if
+		 * it is found, because we took care of it above.
+		 **/
+		self._variables.setRoot(input);
+		self.OE.addToDocument(out, input, self._variables);
+		self._variables.clearRoot();
+
+		return callback(null, out);
+	});
+	return out;
+};
+
 /**
  * Returns the object that was used to construct the ProjectDocumentSource
  * @return {object} the object that was used to construct the ProjectDocumentSource
@@ -38,33 +76,10 @@ proto.getRaw = function getRaw() {
 	return this._raw;
 };
 
-/**
- * Calls base document source eof()
- * @return {bool} The result of base.source.eof()
- **/
-proto.eof = function eof() {
-	return this.source.eof();
-};
-
-/**
- * Calls base document source advance()
- * @return {bool} The result of base.source.advance()
- **/
-proto.advance = function advance() {
-	return this.source.advance();
-};
-
-
-/**
- * Builds a new document(object) that represents this base document
- * @return {object} A document that represents this base document
- **/
-proto.getCurrent = function getCurrent() {
-	var inDocument = this.source.getCurrent();
-	if (!inDocument) throw new Error('inDocument must be an object');
-	var resultDocument = {};
-	this.OE.addToDocument(resultDocument, inDocument, /*root=*/inDocument);
-	return resultDocument;
+proto.serialize = function serialize(explain) {
+	var out = {};
+	out[this.getSourceName()] = this.OE.serialize(explain);
+	return out;
 };
 
 /**
@@ -99,18 +114,24 @@ proto.sourceToJson = function sourceToJson(builder, explain) {
  **/
 klass.createFromJson = function(jsonElement, expCtx) {
 	if (!(jsonElement instanceof Object) || jsonElement.constructor !== Object) throw new Error('Error 15969. Specification must be an object but was ' + typeof jsonElement);
+
 	var objectContext = new Expression.ObjectCtx({
 		isDocumentOk: true,
 		isTopLevel: true,
 		isInclusionOk: true
 	});
-	var project = new ProjectDocumentSource(expCtx);
+
+	var project = new ProjectDocumentSource(expCtx),
+		idGenerator = new VariablesIdGenerator(),
+		vps = new VariablesParseState(idGenerator);
+
 	project._raw = jsonElement;
-	var parsed = Expression.parseObject(jsonElement, objectContext);
+	var parsed = Expression.parseObject(jsonElement, objectContext, vps);
 	var exprObj = parsed;
 	if (!exprObj instanceof ObjectExpression) throw new Error("16402, parseObject() returned wrong type of Expression");
 	if (!exprObj.getFieldCount()) throw new Error("16403, $projection requires at least one output field");
 	project.OE = exprObj;
+	project._variables = new Variables(idGenerator.getIdCount());
 	return project;
 };
 

lib/pipeline/documentSources/RedactDocumentSource.js

@@ -0,0 +1,156 @@
+"use strict";
+
+var async = require("async"),
+	DocumentSource = require("./DocumentSource"),
+	Expression = require("../expressions/Expression"),
+	Variables = require("../expressions/Variables"),
+	VariablesIdGenerator = require("../expressions/VariablesIdGenerator"),
+	VariablesParseState = require("../expressions/VariablesParseState");
+
+/**
+ * A document source skipper
+ * @class RedactDocumentSource
+ * @namespace mungedb-aggregate.pipeline.documentSources
+ * @module mungedb-aggregate
+ * @constructor
+ * @param [ctx] {ExpressionContext}
+ **/
+var RedactDocumentSource = module.exports = function RedactDocumentSource(ctx, expression){
+	if (arguments.length > 2) throw new Error("up to two args expected");
+	base.call(this, ctx);
+	this._expression = expression;
+	this._variables = new Variables();
+	this._currentId = null;
+}, klass = RedactDocumentSource, base = require('./DocumentSource'), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+klass.redactName = "$redact";
+proto.getSourceName = function getSourceName(){
+	return klass.redactName;
+};
+
+var DESCEND_VAL = 'descend',
+	PRUNE_VAL = 'prune',
+	KEEP_VAL = 'keep';
+
+proto.getNext = function getNext(callback) {
+	var self = this,
+		doc;
+	async.whilst(
+		function() {
+			return doc !== DocumentSource.EOF;
+		},
+		function(cb) {
+			self.source.getNext(function(err, input) {
+				doc = input;
+				if (input === DocumentSource.EOF)
+					return cb();
+				self._variables.setRoot(input);
+				self._variables.setValue(self._currentId, input);
+				var result = self.redactObject();
+				if (result !== DocumentSource.EOF)
+					return cb(result); //Using the err argument to pass the result document; this lets us break out without having EOF
+				return cb();
+			});
+		},
+		function(doc) {
+			if (doc)
+				return callback(null, doc);
+			return callback(null, DocumentSource.EOF);
+		}
+	);
+	return doc;
+};
+
+proto.redactValue = function redactValue(input) {
+	// reorder to make JS happy with types
+	if (input instanceof Array) {
+		var newArr,
+			arr = input;
+		for (var i = 0; i < arr.length; i++) {
+			if ((arr[i] instanceof Object && arr[i].constructor === Object) || arr[i] instanceof Array) {
+				var toAdd = this.redactValue(arr[i]);
+				if (toAdd)
+					newArr.push(arr[i]);
+			} else {
+				newArr.push(arr[i]);
+			}
+		}
+		return newArr;
+	} else if (input instanceof Object && input.constructor === Object) {
+		this._variables.setValue(this._currentId, input);
+		var result = this.redactObject();
+		if (result !== DocumentSource.EOF)
+			return result;
+		return null;
+	} else {
+		return input;
+	}
+};
+
+/**
+ * Redacts the current object
+ **/
+proto.redactObject = function redactObject() {
+	var expressionResult = this._expression.evaluate(this._variables);
+
+	if (expressionResult === KEEP_VAL) {
+		return this._variables.getDocument(this._currentId);
+	} else if (expressionResult === PRUNE_VAL) {
+		return DocumentSource.EOF;
+	} else if (expressionResult === DESCEND_VAL) {
+		var input = this._variables.getDocument(this._currentId);
+		var out;
+
+		var inputKeys = Object.keys(input);
+		for (var i = 0; i < inputKeys.length; i++) {
+			var field = inputKeys[i],
+				value = input[field];
+
+			var val = this.redactValue(value);
+			if (val)
+				out[field] = val;
+		}
+
+		return out;
+	} else {
+		throw new Error("17053 $redact's expression should not return anything aside from the variables $$KEEP, $$DESCEND, and $$PRUNE, but returned " + expressionResult);
+	}
+};
+
+proto.optimize = function optimize() {
+	this._expression = this._expression.optimize();
+};
+
+proto.serialize = function serialize(explain) {
+	var doc = {};
+	doc[this.getSourceName()] = this._expression.serialize(explain);
+	return doc;
+};
+
+/**
+ * Creates a new RedactDocumentSource with the input number as the skip
+ *
+ * @param {Number} JsonElement this thing is *called* Json, but it expects a number
+ **/
+klass.createFromJson = function createFromJson(jsonElement, ctx) {
+	if (!jsonElement)
+		throw new Error("#createFromJson requires at least one argument");
+
+	var idGenerator = new VariablesIdGenerator(),
+		vps = new VariablesParseState(idGenerator),
+		currentId = vps.defineVariable("CURRENT"),
+		descendId = vps.defineVariable("DESCEND"),
+		pruneId = vps.defineVariable("PRUNE"),
+		keepId = vps.defineVariable("KEEP");
+
+	var expression = new Expression.parseOperand(jsonElement, vps),
+		source = new RedactDocumentSource(ctx, expression);
+
+	source._currentId = currentId;
+	source._variables = new Variables(idGenerator.getIdCount());
+	source._variables.setValue(descendId, DESCEND_VAL);
+	source._variables.setValue(pruneId, PRUNE_VAL);
+	source._variables.setValue(keepId, KEEP_VAL);
+
+	return source;
+};

lib/pipeline/documentSources/SkipDocumentSource.js

@@ -1,5 +1,8 @@
 "use strict";
 
+var async = require('async'),
+	DocumentSource = require('./DocumentSource');
+
 /**
  * A document source skipper
  * @class SkipDocumentSource
@@ -36,72 +39,44 @@ proto.coalesce = function coalesce(nextSource) {
 	return true;
 };
 
-proto.skipper = function skipper() {
-	if (this.count === 0) {
-		while (!this.source.eof() && this.count++ < this.skip) {
-			this.source.advance();
-		}
-	}
-
-	if (this.source.eof()) {
-		this.current = null;
-		return;
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		next;
+
+	if (this.count < this.skip) {
+
+		async.doWhilst(
+			function(cb) {
+				self.source.getNext(function(err, val) {
+					if(err) return cb(err);
+					self.count++;
+					next = val;
+					return cb();
+				});
+			},
+			function() {
+				return self.count < self.skip || next === DocumentSource.EOF;
+			},
+			function(err) {
+				if (err)
+					return callback(err);
+			}
+		);
 	}
 
-	this.current = this.source.getCurrent();
-};
-
-
-/**
- * Is the source at EOF?
- * @method	eof
- **/
-proto.eof = function eof() {
-	this.skipper();
-	return this.source.eof();
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	this.skipper();
-	return this.source.getCurrent();
+	return this.source.getNext(callback);
 };
 
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-	if (this.eof()) {
-		this.current = null;
-		return false;
-	}
-
-	this.current = this.source.getCurrent();
-	return this.source.advance();
+proto.serialize = function serialize(explain) {
+	var out = {};
+	out[this.getSourceName()] = this.skip;
+	return out;
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
- **/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	builder.$skip = this.skip;
+proto.getSkip = function getSkip() {
+	return this.skip;
 };
 
 /**

lib/pipeline/documentSources/SortDocumentSource.js

@@ -1,5 +1,9 @@
 "use strict";
 
+var async = require("async"),
+	DocumentSource = require("./DocumentSource"),
+	LimitDocumentSource = require("./LimitDocumentSource");
+
 /**
  * A document source sorter
  *
@@ -21,7 +25,6 @@ var SortDocumentSource = module.exports = function SortDocumentSource(ctx){
 	* boolean indicates that this has been done
 	**/
 	this.populated = false;
-	this.current = null;
 	this.docIterator = null; // a number tracking our position in the documents array
 	this.documents = []; // an array of documents
 
@@ -47,6 +50,16 @@ klass.GetDepsReturn = {
 	SEE_NEXT: "SEE_NEXT" // Add the next Source's deps to the set
 };
 
+proto.dispose = function dispose() {
+	this.docIterator = 0;
+	this.documents = [];
+	this.source.dispose();
+};
+
+proto.getLimit = function getLimit() {
+	return this.limitSrc ? this.limitSrc.getLimit() : -1;
+};
+
 proto.getDependencies = function getDependencies(deps) {
 	for(var i = 0; i < this.vSortKey.length; ++i) {
 		this.vSortKey[i].addDependencies(deps);
@@ -54,64 +67,78 @@ proto.getDependencies = function getDependencies(deps) {
 	return klass.GetDepsReturn.SEE_NEXT;
 };
 
-/**
- * Is the source at EOF?
- * @method	eof
- * @return {bool} return if we have hit the end of input
- **/
-proto.eof = function eof() {
-	if (!this.populated) this.populate();
-	return (this.docIterator == this.documents.length);
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
- **/
-proto.getCurrent = function getCurrent() {
-	if (!this.populated) this.populate();
-	return this.current;
+proto.coalesce = function coalesce(nextSource) {
+	if (!this.limitSrc) {
+		if (nextSource instanceof LimitDocumentSource) {
+			this.limitSrc = nextSource;
+			return nextSource;
+		}
+		return false;
+	} else {
+		return this.limitSrc.coalesce(nextSource);
+	}
 };
 
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
- **/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-
-	if (!this.populated) this.populate();
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		out;
+	async.series(
+		[
+			function(next) {
+				if (!self.populated)
+					self.populate(function(err) {
+						return next(err);
+					});
+				else
+					next();
+			},
+			function(next) {
+				if (self.docIterator >= self.documents.length) {
+					out = DocumentSource.EOF;
+					return next(null, DocumentSource.EOF);
+				}
+
+				var output = self.documents[self.docIterator++];
+				if (!output || output === DocumentSource.EOF) {
+					out = DocumentSource.EOF;
+					return next(null, DocumentSource.EOF);
+				}
+
+				out = output;
+				return next(null, output);
+			}
+		],
+		function(err, results) {
+			return callback(err, out);
+		}
+	);
 
-	if (this.docIterator == this.documents.length) throw new Error("This should never happen");
-	++this.docIterator;
+	return out;
+};
 
-	if (this.docIterator == this.documents.length) {
-		this.current = null;
-		return false;
+proto.serializeToArray = function serializeToArray(array, explain) {
+	var doc = {};
+	if (explain) {
+		doc.sortKey = this.serializeSortKey();
+		doc.mergePresorted = this._mergePresorted;
+		doc.limit = this.limitSrc ? this.limitSrc.getLimit() : undefined;
+		array.push(doc);
+	} else {
+		var inner = this.serializeSortKey();
+		if (this._mergePresorted)
+			inner.$mergePresorted = true;
+		doc[this.getSourceName()] = inner;
+		array.push(doc);
+
+		if (this.limitSrc)
+			this.limitSrc.serializeToArray(array);
 	}
-	this.current = this.documents[this.docIterator];
-	return true;
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
-**/
-proto.sourceToJson = function sourceToJson(builder, explain) {
-	var insides = {};
-	this.sortKeyToJson(insides, false);
-	builder[this.getSourceName()] = insides;
+proto.serialize = function serialize(explain) {
+	throw new Error("should call serializeToArray instead");
 };
 
 /**
@@ -129,34 +156,45 @@ proto.addKey = function addKey(fieldPath, ascending) {
 	if (ascending === true || ascending === false) {
 		this.vAscending.push(ascending);
 	} else {
+		// This doesn't appear to be an error in real mongo?
 		throw new Error("ascending must be true or false");
 	}
 };
 
-proto.populate = function populate() {
+proto.populate = function populate(callback) {
 	/* make sure we've got a sort key */
-	if (this.vSortKey.length === null) throw new Error("This should never happen");
+	if (!this.vSortKey.length) throw new Error("no sort key for " + this.getSourceName());
 
-	/* pull everything from the underlying source */
-	for(var hasNext = !this.source.eof(); hasNext; hasNext = this.source.advance()) {
-		var doc = this.source.getCurrent();
-		this.documents.push(doc);
-	}
-
-	this.vSortKeyFPEs = this.vSortKey.map(function(aSortKey){
-		return new FieldPathExpression(aSortKey.getFieldPath(false));
-	});
+	// Skipping stuff about mergeCursors and commandShards
 
+	/* pull everything from the underlying source */
+	var self = this,
+		next;
+	async.doWhilst(
+		function (cb) {
+			self.source.getNext(function(err, doc) {
+				next = doc;
+
+				// Don't add EOF; it doesn't sort well.
+				if (doc !== DocumentSource.EOF)
+					self.documents.push(doc);
+				return cb();
+			});
+		},
+		function() {
+			return next !== DocumentSource.EOF;
+		},
+		function(err) {
 	/* sort the list */
-	this.documents.sort(SortDocumentSource.prototype.compare.bind(this));
+			self.documents.sort(SortDocumentSource.prototype.compare.bind(self));
 
 	/* start the sort iterator */
-	this.docIterator = 0;
+			self.docIterator = 0;
 
-	if (this.docIterator < this.documents.length) {
-		this.current = this.documents[this.docIterator];
+			self.populated = true;
+			return callback();
 	}
-	this.populated = true;
+	);
 };
 
 /**
@@ -177,7 +215,7 @@ proto.compare = function compare(pL,pR) {
 	var n = this.vSortKey.length;
 	for(var i = 0; i < n; ++i) {
 		/* evaluate the sort keys */
-		var pathExpr = this.vSortKeyFPEs[i];
+		var pathExpr = new FieldPathExpression(this.vSortKey[i].getFieldPath(false));
 		var left = pathExpr.evaluate(pL), right = pathExpr.evaluate(pR);
 
 		/*
@@ -201,19 +239,16 @@ proto.compare = function compare(pL,pR) {
 
 /**
 * Write out an object whose contents are the sort key.
-*
-* @param {Object} builder initialized object builder.
-* @param {bool} fieldPrefix specify whether or not to include the field
 **/
-proto.sortKeyToJson = function sortKeyToJson(builder, usePrefix) {
-	// add the key fields
+proto.serializeSortKey = function sortKeyToJson() {
+	var keyObj = {};
+
 	var n = this.vSortKey.length;
-	for(var i = 0; i < n; ++i) {
-		// create the "field name"
-		var ss = this.vSortKey[i].getFieldPath(usePrefix); // renamed write to get
-		// push a named integer based on the sort order
-		builder[ss] = this.vAscending[i] > 0 ? 1 : -1;
+	for (var i = 0; i < n; i++) {
+		var fieldPath = this.vSortKey[i].getFieldPath();
+		keyObj[fieldPath] = this.vAscending[i] ? 1 : -1;
 	}
+	return keyObj;
 };
 
 /**

lib/pipeline/documentSources/UnwindDocumentSource.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var async = require("async");
+
 /**
  * A document source unwinder
  * @class UnwindDocumentSource
@@ -66,6 +68,21 @@ klass.Unwinder = (function(){
 		this._unwindArrayIteratorCurrent = this._unwindArrayIterator.splice(0,1)[0];
 	};
 
+	/**
+	 * getNext
+	 *
+	 * This is just wrapping the old functions because they are somewhat different
+	 * than the original mongo implementation, but should get updated to follow the current API.
+	 **/
+	proto.getNext = function getNext() {
+		if (this.eof())
+			return DocumentSource.EOF;
+
+		var output = this.getCurrent();
+		this.advance();
+		return output;
+	};
+
 	/**
 	 * eof
 	 * @returns	{Boolean}	true if done unwinding the last document passed to resetDocument().
@@ -166,40 +183,6 @@ klass.Unwinder = (function(){
 	return klass;
 })();
 
-/**
- * Lazily construct the _unwinder and initialize the iterator state of this DocumentSource.
- * To be called by all members that depend on the iterator state.
- **/
-proto.lazyInit = function lazyInit(){
-	if (!this._unwinder) {
-		if (!this._unwindPath){
-			throw new Error("unwind path does not exist!");
-		}
-		this._unwinder = new klass.Unwinder(this._unwindPath);
-		if (!this.source.eof()) {
-			// Set up the first source document for unwinding.
-			this._unwinder.resetDocument(this.source.getCurrent());
-		}
-		this.mayAdvanceSource();
-	}
-};
-
-/**
- * If the _unwinder is exhausted and the source may be advanced, advance the source and
- * reset the _unwinder's source document.
- **/
-proto.mayAdvanceSource = function mayAdvanceSource(){
-	while(this._unwinder.eof()) {
-		// The _unwinder is exhausted.
-
-		if (this.source.eof()) return; // The source is exhausted.
-		if (!this.source.advance()) return; // The source is exhausted.
-
-		// Reset the _unwinder with source's next document.
-		this._unwinder.resetDocument(this.source.getCurrent());
-	}
-};
-
 /**
  * Specify the field to unwind.
 **/
@@ -209,6 +192,7 @@ proto.unwindPath = function unwindPath(fieldPath){
 
 	// Record the unwind path.
 	this._unwindPath = new FieldPath(fieldPath);
+	this._unwinder = new klass.Unwinder(this._unwindPath);
 };
 
 klass.unwindName = "$unwind";
@@ -231,55 +215,47 @@ proto.getDependencies = function getDependencies(deps) {
 	return DocumentSource.GetDepsReturn.SEE_NEXT;
 };
 
+proto.getNext = function getNext(callback) {
+	if (!callback) throw new Error(this.getSourceName() + ' #getNext() requires callback');
+
+	var self = this,
+		out = this._unwinder.getNext(),
+		exhausted = false;
+
+	async.until(
+		function() {
+			if(out === DocumentSource.EOF && exhausted) return true;	// Really is EOF, not just an empty unwinder
+			else if(out !== DocumentSource.EOF) return true; // Return whatever we got that wasn't EOF
+			return false;
+		},
+		function(cb) {
+			self.source.getNext(function(err, doc) {
+				if(err) return cb(err);
+				out = doc;
+				if(out === DocumentSource.EOF) { // Our source is out of documents, we're done
+					exhausted = true;
+					return cb();
+				} else {
+					self._unwinder.resetDocument(doc);
+					out = self._unwinder.getNext();
+					return cb();
+				}
+			});
+		},
+		function(err) {
+			if(err) return callback(err);
+			return callback(null, out);
+		}
+	);
 
-/**
- * Is the source at EOF?
- * @method	eof
-**/
-proto.eof = function eof() {
-	this.lazyInit();
-	return this._unwinder.eof();
-};
-
-/**
- * some implementations do the equivalent of verify(!eof()) so check eof() first
- * @method	getCurrent
- * @returns	{Document}	the current Document without advancing
-**/
-proto.getCurrent = function getCurrent() {
-	this.lazyInit();
-	return this._unwinder.getCurrent();
-};
-
-/**
- * Advance the state of the DocumentSource so that it will return the next Document.
- * The default implementation returns false, after checking for interrupts.
- * Derived classes can call the default implementation in their own implementations in order to check for interrupts.
- *
- * @method	advance
- * @returns	{Boolean}	whether there is another document to fetch, i.e., whether or not getCurrent() will succeed.  This default implementation always returns false.
-**/
-proto.advance = function advance() {
-	base.prototype.advance.call(this); // check for interrupts
-	this.lazyInit();
-	this._unwinder.advance();
-	this.mayAdvanceSource();
-	return !this._unwinder.eof();
+	return out; //For sync mode
 };
 
-/**
- * Create an object that represents the document source.  The object
- * will have a single field whose name is the source's name.  This
- * will be used by the default implementation of addToJsonArray()
- * to add this object to a pipeline being represented in JSON.
- *
- * @method	sourceToJson
- * @param	{Object} builder	JSONObjBuilder: a blank object builder to write to
- * @param	{Boolean}	explain	create explain output
-**/
-proto.sourceToJson = function sourceToJson(builder, explain) {
+proto.serialize = function serialize(explain) {
 	if (!this._unwindPath) throw new Error("unwind path does not exist!");
-	builder[this.getSourceName()] = this._unwindPath.getPath(true);
+	var doc = {};
+	doc[this.getSourceName()] = this._unwindPath.getPath(true);
+	return doc;
 };
 
 /**

lib/pipeline/documentSources/index.js

@@ -2,12 +2,14 @@
 module.exports = {
 	CursorDocumentSource: require("./CursorDocumentSource.js"),
 	DocumentSource: require("./DocumentSource.js"),
-	FilterBaseDocumentSource: require("./FilterBaseDocumentSource.js"),
 	GroupDocumentSource: require("./GroupDocumentSource.js"),
 	LimitDocumentSource: require("./LimitDocumentSource.js"),
 	MatchDocumentSource: require("./MatchDocumentSource.js"),
 	ProjectDocumentSource: require("./ProjectDocumentSource.js"),
 	SkipDocumentSource: require("./SkipDocumentSource.js"),
 	SortDocumentSource: require("./SortDocumentSource.js"),
-	UnwindDocumentSource: require("./UnwindDocumentSource.js")
+	UnwindDocumentSource: require("./UnwindDocumentSource.js"),
+	RedactDocumentSource: require("./RedactDocumentSource.js"),
+	GeoNearDocumentSource: require("./GeoNearDocumentSource.js"),
+	OutDocumentSource: require("./OutDocumentSource.js")
 };

lib/pipeline/expressions/AddExpression.js

@@ -1,37 +1,35 @@
 "use strict";
 
-/** 
- * Create an expression that finds the sum of n operands. 
+/**
+ * Create an expression that finds the sum of n operands.
  * @class AddExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var AddExpression = module.exports = function AddExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+//	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = AddExpression, NaryExpression = require("./NaryExpression"), base = NaryExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = AddExpression, base = require("./VariadicExpressionT")(AddExpression), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
+klass.opName = "$add";
 proto.getOpName = function getOpName(){
-	return "$add";
-};
-
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
+	return klass.opName;
 };
 
 /**
  * Takes an array of one or more numbers and adds them together, returning the sum.
  * @method @evaluate
  **/
-proto.evaluate = function evaluate(doc) {
+proto.evaluateInternal = function evaluateInternal(vars) {
 	var total = 0;
 	for (var i = 0, n = this.operands.length; i < n; ++i) {
-		var value = this.operands[i].evaluate(doc);
+		var value = this.operands[i].evaluateInternal(vars);
 		if (value instanceof Date) throw new Error("$add does not support dates; code 16415");
 		if (typeof value == "string") throw new Error("$add does not support strings; code 16416");
 		total += Value.coerceToDouble(value);
@@ -39,3 +37,7 @@ proto.evaluate = function evaluate(doc) {
 	if (typeof total != "number") throw new Error("$add resulted in a non-numeric type; code 16417");
 	return total;
 };
+
+
+/** Register Expression */
+Expression.registerExpression(klass.opName,base.parse);

lib/pipeline/expressions/AllElementsTrueExpression.js

@@ -0,0 +1,48 @@
+"use strict";
+
+/**
+ * Create an expression that returns true exists in all elements.
+ * @class AllElementsTrueExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var AllElementsTrueExpression = module.exports = function AllElementsTrueExpression() {
+	base.call(this);
+},
+	klass = AllElementsTrueExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	CoerceToBoolExpression = require("./CoerceToBoolExpression"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$allElementsTrue";
+};
+
+/**
+ * Takes an array of one or more numbers and returns true if all.
+ * @method @evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var value = evaluateInternal(vars);
+	if (!vars instanceof Array) throw new Error("$allElementsTrue requires an array");
+
+	for (var i = 0, n = this.operands.length; i < n; ++i) {
+		var checkValue = this.operands[i].evaluateInternal(vars);
+		if (!checkValue.coerceToBool()) return false;
+	}
+	return true;
+};
+
+/** Register Expression */
+Expression.registerExpression("$allElementsTrue", base.parse);

lib/pipeline/expressions/AndExpression.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/** 
+/**
  * Create an expression that finds the conjunction of n operands. The
  * conjunction uses short-circuit logic; the expressions are evaluated in the
  * order they were added to the conjunction, and the evaluation stops and
@@ -11,33 +11,31 @@
  * @module mungedb-aggregate
  * @constructor
  **/
-var AndExpression = module.exports = function AndExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var AndExpression = module.exports = function AndExpression() {
+//	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = AndExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = AndExpression, base = require("./VariadicExpressionT")(AndExpression), proto = klass.prototype = Object.create(base.prototype, {constructor: {value: klass}});
 
 // DEPENDENCIES
 var Value = require("../Value"),
 	ConstantExpression = require("./ConstantExpression"),
-	CoerceToBoolExpression = require("./CoerceToBoolExpression");
+	CoerceToBoolExpression = require("./CoerceToBoolExpression"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
-	return "$and";
-};
-
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
+klass.opName = "$and";
+proto.getOpName = function getOpName() {
+	return klass.opName;
 };
 
 /**
  * Takes an array one or more values and returns true if all of the values in the array are true. Otherwise $and returns false.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc) {
+proto.evaluateInternal = function evaluateInternal(vars) {
 	for (var i = 0, n = this.operands.length; i < n; ++i) {
-		var value = this.operands[i].evaluate(doc);
-		if (!Value.coerceToBool(value)) return false;
+		var value = this.operands[i].evaluateInternal(vars);
+		if (!Value.coerceToBool()) return false;
 	}
 	return true;
 };
@@ -71,4 +69,7 @@ proto.optimize = function optimize() {
 	return expr;
 };
 
-//TODO:	proto.toMatcherBson
+/** Register Expression */
+Expression.registerExpression(klass.opName, base.parse);
+
+//TODO: proto.toMatcherBson

lib/pipeline/expressions/AnyElementTrueExpression.js

@@ -0,0 +1,50 @@
+"use strict";
+
+/**
+ * Create an expression that returns true exists in any element.
+ * @class AnyElementTrueExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var AnyElementTrueExpression = module.exports = function AnyElementTrueExpression(){
+	base.call(this);
+},
+	klass = AnyElementTrueExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype,{
+		constructor:{
+			value:klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName(){
+	return "$anyElementTrue";
+};
+
+/**
+ * Takes an array of one or more numbers and returns true if any.
+ * @method @evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var arr = this.operands[0].evaluateInternal(vars);
+	if (!(arr instanceof Array)) {
+		throw new Error("uassert 17041: $anyElementTrue's " +
+						"argument must be an array, but is " +
+						typeof arr);
+	}
+	for (var i=0, n=arr.length; i<n; ++i) {
+		if (Value.coerceToBool(arr[i]))
+			return true;
+	}
+	return false;
+};
+
+/** Register Expression */
+Expression.registerExpression("$anyElementTrue",base.parse);

lib/pipeline/expressions/CoerceToBoolExpression.js

@@ -1,7 +1,7 @@
 "use strict";
 
-/** 
- * internal expression for coercing things to booleans 
+/**
+ * internal expression for coercing things to booleans
  * @class CoerceToBoolExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
@@ -17,16 +17,11 @@ var CoerceToBoolExpression = module.exports = function CoerceToBoolExpression(ex
 var Value = require("../Value"),
 	AndExpression = require("./AndExpression"),
 	OrExpression = require("./OrExpression"),
-	NotExpression = require("./NotExpression");
-
-// PROTOTYPE MEMBERS
-proto.evaluate = function evaluate(doc){
-	var result = this.expression.evaluate(doc);
-	return Value.coerceToBool(result);
-};
+	NotExpression = require("./NotExpression"),
+	Expression = require("./Expression");
 
 proto.optimize = function optimize() {
-	this.expression = this.expression.optimize();	// optimize the operand
+	this.expression = this.expression.optimize();   // optimize the operand
 
 	// if the operand already produces a boolean, then we don't need this
 	// LATER - Expression to support a "typeof" query?
@@ -43,10 +38,25 @@ proto.addDependencies = function addDependencies(deps, path) {
 	return this.expression.addDependencies(deps);
 };
 
+// PROTOTYPE MEMBERS
+proto.evaluateInternal = function evaluateInternal(vars){
+	var result = this.expression.evaluateInternal(vars);
+	return Value.coerceToBool(result);
+};
+
+proto.serialize = function serialize(explain) {
+	if ( explain ) {
+		return {$coerceToBool:[this.expression.toJSON()]};
+	}
+	else {
+		return {$and:[this.expression.toJSON()]};
+	}
+};
+
 proto.toJSON = function toJSON() {
 	// Serializing as an $and expression which will become a CoerceToBool
 	return {$and:[this.expression.toJSON()]};
 };
 
-//TODO:	proto.addToBsonObj   --- may be required for $project to work
-//TODO:	proto.addToBsonArray
+//TODO: proto.addToBsonObj   --- may be required for $project to work
+//TODO: proto.addToBsonArray

lib/pipeline/expressions/CompareExpression.js

@@ -1,108 +1,113 @@
 "use strict";
 
 /**
- * Generic comparison expression that gets used for $eq, $ne, $lt, $lte, $gt, $gte, and $cmp. 
+ * Generic comparison expression that gets used for $eq, $ne, $lt, $lte, $gt, $gte, and $cmp.
  * @class CompareExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var CompareExpression = module.exports = function CompareExpression(cmpOp) {
-	if (arguments.length !== 1) throw new Error("args expected: cmpOp");
-	this.cmpOp = cmpOp;
-	base.call(this);
-}, klass = CompareExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+    this.cmpOp = cmpOp;
+    base.call(this);
+}, klass = CompareExpression,
+    FixedArityExpression = require("./FixedArityExpressionT")(klass, 2),
+	base = FixedArityExpression,
+    proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+    });
 
 // DEPENDENCIES
-var Value = require("../Value"),
-	Expression = require("./Expression"),
-	ConstantExpression = require("./ConstantExpression"),
-	FieldPathExpression = require("./FieldPathExpression"),
-	FieldRangeExpression = require("./FieldRangeExpression");
+var Value = require("../Value");
+var Expression = require("./Expression");
+var ConstantExpression = require("./ConstantExpression");
+var FieldPathExpression = require("./FieldPathExpression");
+var FieldRangeExpression = require("./FieldRangeExpression");
+var NaryExpression = require("./NaryExpression");
 
 // NESTED CLASSES
 /**
  * Lookup table for truth value returns
  *
- * @param truthValues	truth value for -1, 0, 1
- * @param reverse		reverse comparison operator
- * @param name			string name
+ * @param truthValues   truth value for -1, 0, 1
+ * @param reverse               reverse comparison operator
+ * @param name                  string name
  **/
-var CmpLookup = (function(){	// emulating a struct
+var CmpLookup = (function() { // emulating a struct
 	// CONSTRUCTOR
 	var klass = function CmpLookup(truthValues, reverse, name) {
-		if(arguments.length !== 3) throw new Error("args expected: truthValues, reverse, name");
+		if (arguments.length !== 3) throw new Error("args expected: truthValues, reverse, name");
 		this.truthValues = truthValues;
 		this.reverse = reverse;
 		this.name = name;
-	}, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	}, base = Object,
+		proto = klass.prototype = Object.create(base.prototype, {
+			constructor: {
+				value: klass
+			}
+		});
 	return klass;
 })();
 
+// verify we need this below
 // PRIVATE STATIC MEMBERS
 /**
  * a table of cmp type lookups to truth values
  * @private
  **/
-var cmpLookupMap = [	//NOTE: converted from this Array to a Dict/Object below using CmpLookup#name as the key
-	//              -1      0      1      reverse             name     (taking advantage of the fact that our 'enums' are strings below)
-	new CmpLookup([false, true, false], Expression.CmpOp.EQ, Expression.CmpOp.EQ),
-	new CmpLookup([true, false, true], Expression.CmpOp.NE, Expression.CmpOp.NE),
-	new CmpLookup([false, false, true], Expression.CmpOp.LT, Expression.CmpOp.GT),
-	new CmpLookup([false, true, true], Expression.CmpOp.LTE, Expression.CmpOp.GTE),
-	new CmpLookup([true, false, false], Expression.CmpOp.GT, Expression.CmpOp.LT),
-	new CmpLookup([true, true, false], Expression.CmpOp.GTE, Expression.CmpOp.LTE),
-	new CmpLookup([false, false, false], Expression.CmpOp.CMP, Expression.CmpOp.CMP)
-].reduce(function(r,o){r[o.name]=o;return r;},{});
+var cmpLookupMap = [ //NOTE: converted from this Array to a Dict/Object below using CmpLookup#name as the key
+    //              -1      0      1      reverse             name     (taking advantage of the fact that our 'enums' are strings below)
+    new CmpLookup([false, true, false], CompareExpression.EQ, CompareExpression.EQ),
+    new CmpLookup([true, false, true], CompareExpression.NE, CompareExpression.NE),
+    new CmpLookup([false, false, true], CompareExpression.LT, CompareExpression.GT),
+    new CmpLookup([false, true, true], CompareExpression.LTE, CompareExpression.GTE),
+    new CmpLookup([true, false, false], CompareExpression.GT, CompareExpression.LT),
+    new CmpLookup([true, true, false], CompareExpression.GTE, CompareExpression.LTE),
+    new CmpLookup([false, false, false], CompareExpression.CMP, CompareExpression.CMP)
+].reduce(function(r, o) {
+	r[o.name] = o;
+	return r;
+}, {});
 
 
-// PROTOTYPE MEMBERS
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
+klass.parse = function parse(bsonExpr, vps, op) {
+    var expr = new CompareExpression(op);
+    var args = NaryExpression.parseArguments(bsonExpr, vps);
+    expr.validateArguments(args);
+    expr.vpOperand = args;
+    return expr;
 
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc),
-		right = this.operands[1].evaluate(doc),
-		cmp = Expression.signum(Value.compare(left, right));
-	if (this.cmpOp == Expression.CmpOp.CMP) return cmp;
-	return cmpLookupMap[this.cmpOp].truthValues[cmp + 1] || false;
 };
 
-proto.optimize = function optimize(){
-	var expr = base.prototype.optimize.call(this); // first optimize the comparison operands
-	if (!(expr instanceof CompareExpression)) return expr; // if no longer a comparison, there's nothing more we can do.
-
-	// check to see if optimizing comparison operator is supported	// CMP and NE cannot use ExpressionFieldRange which is what this optimization uses
-	var newOp = this.cmpOp;
-	if (newOp == Expression.CmpOp.CMP || newOp == Expression.CmpOp.NE) return expr;
-
-	// There's one localized optimization we recognize:  a comparison between a field and a constant.  If we recognize that pattern, replace it with an ExpressionFieldRange.
-	// When looking for this pattern, note that the operands could appear in any order.  If we need to reverse the sense of the comparison to put it into the required canonical form, do so.
-	var leftExpr = this.operands[0],
-		rightExpr = this.operands[1];
-	var fieldPathExpr, constantExpr;
-	if (leftExpr instanceof FieldPathExpression) {
-		fieldPathExpr = leftExpr;
-		if (!(rightExpr instanceof ConstantExpression)) return expr; // there's nothing more we can do
-		constantExpr = rightExpr;
-	} else {
-		// if the first operand wasn't a path, see if it's a constant
-		if (!(leftExpr instanceof ConstantExpression)) return expr; // there's nothing more we can do
-		constantExpr = leftExpr;
-
-		// the left operand was a constant; see if the right is a path
-		if (!(rightExpr instanceof FieldPathExpression)) return expr; // there's nothing more we can do
-		fieldPathExpr = rightExpr;
-
-		// these were not in canonical order, so reverse the sense
-		newOp = cmpLookupMap[newOp].reverse;
-	}
-	return new FieldRangeExpression(fieldPathExpr, newOp, constantExpr.getValue());
+// PROTOTYPE MEMBERS
+proto.evaluateInternal = function evaluateInternal(vars) {
+	//debugger;
+    var left = this.operands[0].evaluateInternal(vars),
+        right = this.operands[1].evaluateInternal(vars),
+        cmp = Expression.signum(Value.compare(left, right));
+    if (this.cmpOp == Expression.CmpOp.CMP) return cmp;
+    return cmpLookupMap[this.cmpOp].truthValues[cmp + 1] || false;
 };
 
-proto.getOpName = function getOpName(){
+klass.EQ = "$eq";
+klass.NE = "$ne";
+klass.GT = "$gt";
+klass.GTE = "$gte";
+klass.LT = "$lt";
+klass.LTE = "$lte";
+klass.CMP = "$cmp";
+
+proto.getOpName = function getOpName() {
 	return this.cmpOp;
 };
+
+/** Register Expression */
+Expression.registerExpression("$eq", klass.parse);
+Expression.registerExpression("$ne", klass.parse);
+Expression.registerExpression("$gt", klass.parse);
+Expression.registerExpression("$gte", klass.parse);
+Expression.registerExpression("$lt", klass.parse);
+Expression.registerExpression("$lte", klass.parse);
+Expression.registerExpression("$cmp", klass.parse);

lib/pipeline/expressions/ConcatExpression.js

@@ -1,6 +1,8 @@
 "use strict";
 
-/** 
+var Expression = require("./Expression");
+
+/**
  * Creates an expression that concatenates a set of string operands.
  * @class ConcatExpression
  * @namespace mungedb-aggregate.pipeline.expressions
@@ -10,31 +12,30 @@
 var ConcatExpression = module.exports = function ConcatExpression(){
 	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = ConcatExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = ConcatExpression, base = require("./VariadicExpressionT")(ConcatExpression), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
 var Value = require("../Value");
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
+klass.opName = "$concat";
 proto.getOpName = function getOpName(){
-	return "$concat";
-};
-
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
+	return klass.opName;
 };
 
 /**
  * Concats a string of values together.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc) {
-	var result = "";
-	for (var i = 0, n = this.operands.length; i < n; ++i) {
-		var val = this.operands[i].evaluate(doc);
-		if (val === null) return null; // if any operand is null, return null for all
-		if (typeof(val) != "string") throw new Error("$concat only supports strings, not " + typeof(val) + "; code 16702");
-		result = result + Value.coerceToString(val);
-	}
-	return result;
+proto.evaluateInternal = function evaluateInternal(vars) {
+    return this.operands.map(function(x) {
+		var y = x.evaluateInternal(vars);
+		if(typeof(y) !== "string") {
+	    	throw new Error("$concat only supports strings - 16702");
+		}
+	return y;
+    }).join("");
 };
+
+Expression.registerExpression(klass.opName, base.parse);

lib/pipeline/expressions/CondExpression.js

@@ -1,37 +1,113 @@
 "use strict";
 
 /**
- * $cond expression;  @see evaluate 
- * @class AndExpression
+ * $cond expression;  @see evaluate
+ * @class CondExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var CondExpression = module.exports = function CondExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
-	base.call(this);
-}, klass = CondExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+var CondExpression = module.exports = function CondExpression(vars) {
+		if (arguments.length !== 0) throw new Error("zero args expected");
+    base.call(this);
+}, klass = CondExpression,
+	base = require("./FixedArityExpressionT")(klass, 3),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+    Expression = require("./Expression"),
+	FixedArityExpressionT = require("./FixedArityExpressionT");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
-	return "$cond";
+klass.opName = "$cond";
+proto.getOpName = function getOpName() {
+    return klass.opName;
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(3);
-	base.prototype.addOperand.call(this, expr);
+/**
+ *
+ * @param expr	- I expect this to be the RHS of $cond:{...} or $cond:[,,,]
+ * @param vps
+ * @returns {*}
+ */
+klass.parse = function parse(expr, vps) {
+	// There may only be one argument - an array of 3 items, or a hash containing 3 keys.
+    //this.checkArgLimit(3);
+
+    // if not an object, return;
+	// todo I don't understand why we'd do this.  shouldn't expr be {}, [], or wrong?
+    if (typeof(expr) !== Object || )
+		return FixedArityExpressionT.parse(expr, vps);
+
+	// ...or expr could be the entirety of $cond:{...} or $cond:[,,,].
+	if(!(klass.opName in expr)) {
+		throw new Error("Invalid expression. Expected to see '"+klass.opName+"'");
+	}
+
+    var ret = new CondExpression();
+
+	// If this is an Object and not an array, verify all the bits are specified.
+	// If this is an Object that is an array, verify there are three bits.
+	// (My issue here is that we got to this parse function when we parsed the $cond:{...} item, and we're calling
+	// parseOperand (again) without altering the input.)
+//    var args = Expression.parseOperand(expr, vps);
+
+	var args = expr[getOpName()];
+
+	if (typeof args !== 'object') throw new Error("this should not happen");
+	if (args instanceof Array) {
+		// it's the array form. Convert it to the object form.
+		if (args.length !== 3) throw new Error("$cond requires exactly three arguments");
+		args = {if: args[0], then: args[1], else: args[2]};
+	}
+
+	// One way or the other, args is now in object form.
+	Object.keys(args).forEach(function(arg) {
+		if (arg === 'if') {
+			ret.operands[0] = Expression.parseOperand(args['if'], vps);
+		}
+		else if (arg === 'then') {
+			ret.operands[1] = Expression.parseOperand(args['then'], vps);
+		}
+		else if (arg === 'else') {
+			ret.operands[2] = Expression.parseOperand(args['else'], vps);
+		}
+		else {
+			throw new Error("Unrecognized parameter to $cond: '" + arg + "'; code 17083");
+		}
+	});
+
+    if (!ret.operands[0]) throw new Error("Missing 'if' parameter to $cond; code 17080");
+    if (!ret.operands[1]) throw new Error("Missing 'then' parameter to $cond; code 17081");
+    if (!ret.operands[2]) throw new Error("Missing 'else' parameter to $cond; code 17082");
+
+    return ret;
 };
 
-/** 
- * Use the $cond operator with the following syntax:  { $cond: [ <boolean-expression>, <true-case>, <false-case> ] } 
+/**
+ * Use the $cond operator with the following syntax:
+ * { $cond: { if: <boolean-expression>, then: <true-case>, else: <false-case-> } }
+ * -or-
+ * { $cond: [ <boolean-expression>, <true-case>, <false-case> ] }
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(3);
-	var pCond = this.operands[0].evaluate(doc),
-		idx = Value.coerceToBool(pCond) ? 1 : 2;
-	return this.operands[idx].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+		var pCond1 = this.operands[0].evaluateInternal(vars);
+
+		this.idx = 0;
+		if (pCond1.coerceToBool()) {
+			this.idx = 1;
+		} else {
+			this.idx = 2;
+		}
+
+		return this.operands[this.idx].evaluateInternal(vars);
 };
+
+/** Register Expression */
+Expression.registerExpression(klass.opName, klass.parse);

lib/pipeline/expressions/ConstantExpression.js

@@ -1,30 +1,33 @@
 "use strict";
 
-/** 
- * Internal expression for constant values 
+var Value = require("../Value"),
+    Expression = require("./Expression");
+
+/**
+ * Internal expression for constant values
  * @class ConstantExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
- **/
+ */
 var ConstantExpression = module.exports = function ConstantExpression(value){
-	if (arguments.length !== 1) throw new Error("args expected: value");
-	this.value = value;	//TODO: actually make read-only in terms of JS?
-	base.call(this);
-}, klass = ConstantExpression, base = require("./Expression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+    if (arguments.length !== 1) throw new Error(klass.name + ": args expected: value");
+    this.value = value;
+    base.call(this);
+}, klass = ConstantExpression, base = require("./FixedArityExpressionT")(klass,1), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
-	return "$const";
+klass.parse = function parse(exprElement, vps) {
+	return new ConstantExpression(exprElement);
 };
 
-/**
- * Get the constant value represented by this Expression.
- * @method getValue
- * @returns the value
- **/
-proto.getValue = function getValue(){	//TODO: convert this to an instance field rather than a property
-	return this.value;
+klass.create = function create(value) {
+	var constExpr = new ConstantExpression(value);
+	return constExpr;
+};
+
+proto.optimize = function optimize() {
+	// nothing to do
+	return this;
 };
 
 proto.addDependencies = function addDependencies(deps, path) {
@@ -34,18 +37,24 @@ proto.addDependencies = function addDependencies(deps, path) {
 /**
  * Get the constant value represented by this Expression.
  * @method evaluate
- **/
-proto.evaluate = function evaluate(doc){
+ */
+proto.evaluateInternal = function evaluateInternal(vars) {
 	return this.value;
 };
 
-proto.optimize = function optimize() {
-	return this; // nothing to do
-};
+/// Helper function to easily wrap constants with $const.
+function serializeConstant(val) {
+    return {$const: val};
+}
 
-proto.toJSON = function(isExpressionRequired){
-	return isExpressionRequired ? {$const: this.value} : this.value;
+proto.serialize = function serialize(explain) {
+	return serializeConstant(this.value);
 };
 
-//TODO:	proto.addToBsonObj   --- may be required for $project to work -- my hope is that we can implement toJSON methods all around and use that instead
-//TODO:	proto.addToBsonArray
+Expression.registerExpression("$const", klass.parse);
+
+Expression.registerExpression("$literal", klass.parse); // alias
+
+proto.getOpName = function getOpName() {
+	return "$const";
+};

lib/pipeline/expressions/DayOfMonthExpression.js

@@ -7,27 +7,34 @@
  * @module mungedb-aggregate
  * @constructor
  **/
-var DayOfMonthExpression = module.exports = function DayOfMonthExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var DayOfMonthExpression = module.exports = function DayOfMonthExpression() {
 	base.call(this);
-}, klass = DayOfMonthExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = DayOfMonthExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
+
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$dayOfMonth";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the day of the month as a number between 1 and 31.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCDate();
 };
+
+/** Register Expression */
+Expression.registerExpression("$dayOfMonth", base.parse);

lib/pipeline/expressions/DayOfWeekExpression.js

@@ -8,26 +8,32 @@
  * @constructor
  **/
 var DayOfWeekExpression = module.exports = function DayOfWeekExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = DayOfWeekExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = DayOfWeekExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor:{
+			value:klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$dayOfWeek";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the day of the week as a number between 1 (Sunday) and 7 (Saturday.)
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars){
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCDay()+1;
 };
+
+/** Register Expression */
+Expression.registerExpression("$dayOfWeek",base.parse);

lib/pipeline/expressions/DayOfYearExpression.js

@@ -8,34 +8,40 @@
  * @constructor
  **/
 var DayOfYearExpression = module.exports = function DayOfYearExpression(){
-	if(arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = DayOfYearExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = DayOfYearExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor:{
+			value:klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
-	return "$dayOfYear";
-};
-
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
+    return "$dayOfYear";
 };
 
 /**
  * Takes a date and returns the day of the year as a number between 1 and 366.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars){
 	//NOTE: the below silliness is to deal with the leap year scenario when we should be returning 366
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
-	return klass.getDateDayOfYear(date);
+    var date = this.operands[0].evaluateInternal(vars);
+    return klass.getDateDayOfYear(date);
 };
 
 // STATIC METHODS
 klass.getDateDayOfYear = function getDateDayOfYear(d){
-	var y11 = new Date(d.getUTCFullYear(), 0, 1),	// same year, first month, first day; time omitted
-		ymd = new Date(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()+1);	// same y,m,d; time omitted, add 1 because days start at 1
-	return Math.ceil((ymd - y11) / 86400000);	//NOTE: 86400000 ms is 1 day
+    var y11 = new Date(d.getUTCFullYear(), 0, 1),       // same year, first month, first day; time omitted
+	ymd = new Date(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()+1);  // same y,m,d; time omitted, add 1 because days start at 1
+    return Math.ceil((ymd - y11) / 86400000);   //NOTE: 86400000 ms is 1 day
 };
+
+/** Register Expression */
+Expression.registerExpression("$dayOfYear",base.parse);

lib/pipeline/expressions/DivideExpression.js

@@ -1,42 +1,46 @@
 "use strict";
 
-/** 
- * A $divide pipeline expression. 
- * @see evaluate 
+/**
+ * A $divide pipeline expression.
+ * @see evaluateInternal
  * @class DivideExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var DivideExpression = module.exports = function DivideExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
-	base.call(this);
-}, klass = DivideExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+    base.call(this);
+}, klass = DivideExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 2),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor:{
+			value:klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){	//TODO: try to move this to a static and/or instance field instead of a getter function
+proto.getOpName = function getOpName(){ //TODO: try to move this to a static and/or instance field instead of a getter function
 	return "$divide";
 };
 
-proto.addOperand = function addOperand(expr){
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes an array that contains a pair of numbers and returns the value of the first number divided by the second number.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc),
-		right = this.operands[1].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var left = this.operands[0].evaluateInternal(vars),
+		right = this.operands[1].evaluateInternal(vars);
 	if (!(left instanceof Date) && (!right instanceof Date)) throw new Error("$divide does not support dates; code 16373");
 	right = Value.coerceToDouble(right);
 	if (right === 0) return undefined;
 	left = Value.coerceToDouble(left);
 	return left / right;
 };
+
+/** Register Expression */
+Expression.registerExpression("$divide",base.parse);

lib/pipeline/expressions/Expression.js

@@ -5,28 +5,30 @@
  *
  * NOTE: An object expression can take any of the following forms:
  *
- *	f0: {f1: ..., f2: ..., f3: ...}
- *	f0: {$operator:[operand1, operand2, ...]}
+ *      f0: {f1: ..., f2: ..., f3: ...}
+ *      f0: {$operator:[operand1, operand2, ...]}
  *
  * @class Expression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
- **/
-var Expression = module.exports = function Expression(){
+ */
+var Expression = module.exports = function Expression() {
 	if (arguments.length !== 0) throw new Error("zero args expected");
 }, klass = Expression, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// DEPENDENCIES
-var Document = require("../Document");
 
-// NESTED CLASSES
+var Value = require("../Value"),
+	Document = require("../Document"),
+	Variables = require("./Variables");
+
+
 /**
  * Reference to the `mungedb-aggregate.pipeline.expressions.Expression.ObjectCtx` class
  * @static
  * @property ObjectCtx
- **/
-var ObjectCtx = Expression.ObjectCtx = (function(){
+ */
+var ObjectCtx = Expression.ObjectCtx = (function() {
 	// CONSTRUCTOR
 	/**
 	 * Utility class for parseObject() below. isDocumentOk indicates that it is OK to use a Document in the current context.
@@ -38,375 +40,297 @@ var ObjectCtx = Expression.ObjectCtx = (function(){
 	 * @module mungedb-aggregate
 	 * @constructor
 	 * @param opts
-	 *	@param [opts.isDocumentOk]	{Boolean}
-	 *	@param [opts.isTopLevel]	{Boolean}
-	 *	@param [opts.isInclusionOk]	{Boolean}
-	 **/
-	var klass = function ObjectCtx(opts /*= {isDocumentOk:..., isTopLevel:..., isInclusionOk:...}*/){
-		if(!(opts instanceof Object && opts.constructor == Object)) throw new Error("opts is required and must be an Object containing named args");
+	 *      @param [opts.isDocumentOk]      {Boolean}
+	 *      @param [opts.isTopLevel]        {Boolean}
+	 *      @param [opts.isInclusionOk]     {Boolean}
+	 */
+	var klass = function ObjectCtx(opts /*= {isDocumentOk:..., isTopLevel:..., isInclusionOk:...}*/ ) {
+		if (!(opts instanceof Object && opts.constructor === Object)) throw new Error("opts is required and must be an Object containing named args");
 		for (var k in opts) { // assign all given opts to self so long as they were part of klass.prototype as undefined properties
 			if (opts.hasOwnProperty(k) && proto.hasOwnProperty(k) && proto[k] === undefined) this[k] = opts[k];
 		}
-	}, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+	}, base = Object,
+		proto = klass.prototype = Object.create(base.prototype, {
+			constructor: {
+				value: klass
+			}
+		});
 
 	// PROTOTYPE MEMBERS
 	proto.isDocumentOk =
-	proto.isTopLevel =
-	proto.isInclusionOk = undefined;
+		proto.isTopLevel =
+		proto.isInclusionOk = undefined;
 
 	return klass;
 })();
 
+
+//
+// Diagram of relationship between parse functions when parsing a $op:
+//
+// { someFieldOrArrayIndex: { $op: [ARGS] } }
+//                             ^ parseExpression on inner $op BSONElement
+//                          ^ parseObject on BSONObject
+//             ^ parseOperand on outer BSONElement wrapping the $op Object
+//
+
 /**
- * Reference to the `mungedb-aggregate.pipeline.expressions.Expression.OpDesc` class
+ * Parses a JSON Object that could represent a functional expression or a Document expression.
+ * @method parseObject
  * @static
- * @property OpDesc
- **/
-var OpDesc = Expression.OpDesc = (function(){
-	// CONSTRUCTOR
-	/**
-	 * Decribes how and when to create an Op instance
-	 *
-	 * @class OpDesc
-	 * @namespace mungedb-aggregate.pipeline.expressions.Expression
-	 * @module mungedb-aggregate
-	 * @constructor
-	 * @param name
-	 * @param factory
-	 * @param flags
-	 * @param argCount
-	 **/
-	var klass = function OpDesc(name, factory, flags, argCount){
-		var firstArg = arguments[0];
-		if (firstArg instanceof Object && firstArg.constructor == Object) { //TODO: using this?
-			var opts = firstArg;
-			for (var k in opts) { // assign all given opts to self so long as they were part of klass.prototype as undefined properties
-				if (opts.hasOwnProperty(k) && proto.hasOwnProperty(k) && proto[k] === undefined) this[k] = opts[k];
-			}
-		} else {
-			this.name = name;
-			this.factory = factory;
-			this.flags = flags || 0;
-			this.argCount = argCount || 0;
-		}
-	}, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+ * @param obj   the element representing the object
+ * @param ctx   a MiniCtx representing the options above
+ * @param vps	Variables Parse State
+ * @returns the parsed Expression
+ */
+klass.parseObject = function parseObject(obj, ctx, vps) {
+	if (!(ctx instanceof ObjectCtx)) throw new Error("ctx must be ObjectCtx");
+	/*
+	  An object expression can take any of the following forms:
 
-	// STATIC MEMBERS
-	klass.FIXED_COUNT = 1;
-	klass.OBJECT_ARG = 2;
+	  f0: {f1: ..., f2: ..., f3: ...}
+	  f0: {$operator:[operand1, operand2, ...]}
+	*/
 
-	// PROTOTYPE MEMBERS
-	proto.name =
-	proto.factory =
-	proto.flags =
-	proto.argCount = undefined;
+	var expression, // the result
+		expressionObject, // the alt result
+		UNKNOWN = 0,
+		NOTOPERATOR = 1,
+		OPERATOR = 2,
+		kind = UNKNOWN;
 
-	/**
-	 * internal `OpDesc#name` comparer
-	 * @method cmp
-	 * @param that the other `OpDesc` instance
-	 **/
-	proto.cmp = function cmp(that) {
-		return this.name < that.name ? -1 : this.name > that.name ? 1 : 0;
-	};
+	if (obj === undefined || obj === null || (obj instanceof Object && Object.keys(obj).length === 0)) return new ObjectExpression();
+	var fieldNames = Object.keys(obj);
+	for (var fieldCount = 0, n = fieldNames.length; fieldCount < n; ++fieldCount) {
+		var fieldName = fieldNames[fieldCount];
 
-	return klass;
-})();
-// END OF NESTED CLASSES
-/**
- * @class Expression
- * @namespace mungedb-aggregate.pipeline.expressions
- * @module mungedb-aggregate
- **/
+		if (fieldName[0] === "$") {
+			if (fieldCount !== 0)
+				throw new Error("the operator must be the only field in a pipeline object (at '" + fieldName + "'.; uassert code 15983");
 
-var kinds = {
-	UNKNOWN: "UNKNOWN",
-	OPERATOR: "OPERATOR",
-	NOT_OPERATOR: "NOT_OPERATOR"
-};
+			if (ctx.isTopLevel)
+				throw new Error("$expressions are not allowed at the top-level of $project; uassert code 16404");
 
+			// we've determined this "object" is an operator expression
+			kind = OPERATOR;
 
-// STATIC MEMBERS
-/**
- * Enumeration of comparison operators.  These are shared between a few expression implementations, so they are factored out here.
- *
- * @static
- * @property CmpOp
- **/
-klass.CmpOp = {
-	EQ: "$eq",		// return true for a == b, false otherwise
-	NE: "$ne",		// return true for a != b, false otherwise
-	GT: "$gt",		// return true for a > b, false otherwise
-	GTE: "$gte",	// return true for a >= b, false otherwise
-	LT: "$lt",		// return true for a < b, false otherwise
-	LTE: "$lte",	// return true for a <= b, false otherwise
-	CMP: "$cmp"		// return -1, 0, 1 for a < b, a == b, a > b
-};
+			expression = Expression.parseExpression(fieldName, obj[fieldName], vps); //NOTE: DEVIATION FROM MONGO: c++ code uses 2 arguments. See #parseExpression
+		} else {
+			if (kind === OPERATOR)
+				throw new Error("this object is already an operator expression, and can't be used as a document expression (at '" + fieldName + "'.; uassert code 15990");
 
-// DEPENDENCIES (later in this file as compared to others to ensure that the required statics are setup first)
-var FieldPathExpression = require("./FieldPathExpression"),
-	ObjectExpression = require("./ObjectExpression"),
-	ConstantExpression = require("./ConstantExpression"),
-	CompareExpression = require("./CompareExpression");
+			if (!ctx.isTopLevel && fieldName.indexOf(".") !== -1)
+				throw new Error("dotted field names are only allowed at the top level; uassert code 16405");
 
-// DEFERRED DEPENDENCIES
-/**
- * Expressions, as exposed to users
- *
- * @static
- * @property opMap
- **/
-setTimeout(function(){ // Even though `opMap` is deferred, force it to load early rather than later to prevent even *more* potential silliness
-	Object.defineProperty(klass, "opMap", {value:klass.opMap});
-}, 0);
-Object.defineProperty(klass, "opMap", {	//NOTE: deferred requires using a getter to allow circular requires (to maintain the ported API)
-	configurable: true,
-	get: function getOpMapOnce() {
-		return Object.defineProperty(klass, "opMap", {
-			value: [	//NOTE: rather than OpTable because it gets converted to a dict via OpDesc#name in the Array#reduce() below
-				new OpDesc("$add", require("./AddExpression"), 0),
-				new OpDesc("$and", require("./AndExpression"), 0),
-				new OpDesc("$cmp", CompareExpression.bind(null, Expression.CmpOp.CMP), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$concat", require("./ConcatExpression"), 0),
-				new OpDesc("$cond", require("./CondExpression"), OpDesc.FIXED_COUNT, 3),
-		//		$const handled specially in parseExpression
-				new OpDesc("$dayOfMonth", require("./DayOfMonthExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$dayOfWeek", require("./DayOfWeekExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$dayOfYear", require("./DayOfYearExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$divide", require("./DivideExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$eq", CompareExpression.bind(null, Expression.CmpOp.EQ), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$gt", CompareExpression.bind(null, Expression.CmpOp.GT), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$gte", CompareExpression.bind(null, Expression.CmpOp.GTE), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$hour", require("./HourExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$ifNull", require("./IfNullExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$lt", CompareExpression.bind(null, Expression.CmpOp.LT), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$lte", CompareExpression.bind(null, Expression.CmpOp.LTE), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$minute", require("./MinuteExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$mod", require("./ModExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$month", require("./MonthExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$multiply", require("./MultiplyExpression"), 0),
-				new OpDesc("$ne", CompareExpression.bind(null, Expression.CmpOp.NE), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$not", require("./NotExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$or", require("./OrExpression"), 0),
-				new OpDesc("$second", require("./SecondExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$strcasecmp", require("./StrcasecmpExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$substr", require("./SubstrExpression"), OpDesc.FIXED_COUNT, 3),
-				new OpDesc("$subtract", require("./SubtractExpression"), OpDesc.FIXED_COUNT, 2),
-				new OpDesc("$toLower", require("./ToLowerExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$toUpper", require("./ToUpperExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$week", require("./WeekExpression"), OpDesc.FIXED_COUNT, 1),
-				new OpDesc("$year", require("./YearExpression"), OpDesc.FIXED_COUNT, 1)
-			].reduce(function(r,o){r[o.name]=o; return r;}, {})
-		}).opMap;
-	}
-});
+			// if it's our first time, create the document expression
+			if (expression === undefined) {
+				if (!ctx.isDocumentOk) throw new Error("Assertion failure");
+				// CW TODO error: document not allowed in this context
 
-/**
- * Parse an Object.  The object could represent a functional expression or a Document expression.
- *
- * An object expression can take any of the following forms:
- *
- *	f0: {f1: ..., f2: ..., f3: ...}
- *	f0: {$operator:[operand1, operand2, ...]}
- *
- * @static
- * @method parseObject
- * @param obj	the element representing the object
- * @param ctx	a MiniCtx representing the options above
- * @returns the parsed Expression
- **/
-klass.parseObject = function parseObject(obj, ctx){
-	if(!(ctx instanceof ObjectCtx)) throw new Error("ctx must be ObjectCtx");
-	var kind = kinds.UNKNOWN,
-		expr, // the result
-		exprObj; // the alt result
-	if (obj === undefined) return new ObjectExpression();
-	var fieldNames = Object.keys(obj);
-	for (var fc = 0, n = fieldNames.length; fc < n; ++fc) {
-		var fn = fieldNames[fc];
-		if (fn[0] === "$") {
-			if (fc !== 0) throw new Error("the operator must be the only field in a pipeline object (at '" + fn + "'.; code 16410");
-			if(ctx.isTopLevel) throw new Error("$expressions are not allowed at the top-level of $project; code 16404");
-			kind = kinds.OPERATOR;	//we've determined this "object" is an operator expression
-			expr = Expression.parseExpression(fn, obj[fn]);
-		} else {
-			if (kind === kinds.OPERATOR) throw new Error("this object is already an operator expression, and can't be used as a document expression (at '" + fn + "'.; code 15990");
-			if (!ctx.isTopLevel && fn.indexOf(".") != -1) throw new Error("dotted field names are only allowed at the top level; code 16405");
-			if (expr === undefined) { // if it's our first time, create the document expression
-				if (!ctx.isDocumentOk) throw new Error("document not allowed in this context"); // CW TODO error: document not allowed in this context
-				expr = exprObj = new ObjectExpression();
-				kind = kinds.NOT_OPERATOR;	//this "object" is not an operator expression
+				expressionObject = ctx.isTopLevel ? ObjectExpression.createRoot() : ObjectExpression.create();
+				expression = expressionObject;
+
+				// this "object" is not an operator expression
+				kind = NOTOPERATOR;
 			}
-			var fv = obj[fn];
-			switch (typeof(fv)) {
-			case "object":
-				// it's a nested document
-				var subCtx = new ObjectCtx({
-					isDocumentOk: ctx.isDocumentOk,
-					isInclusionOk: ctx.isInclusionOk
-				});
-				exprObj.addField(fn, Expression.parseObject(fv, subCtx));
-				break;
-			case "string":
-				// it's a renamed field		// CW TODO could also be a constant
-				var pathExpr = new FieldPathExpression(Expression.removeFieldPrefix(fv));
-				exprObj.addField(fn, pathExpr);
-				break;
-			case "boolean":
-			case "number":
-				// it's an inclusion specification
-				if (fv) {
-					if (!ctx.isInclusionOk) throw new Error("field inclusion is not allowed inside of $expressions; code 16420");
-					exprObj.includePath(fn);
-				} else {
-					if (!(ctx.isTopLevel && fn == Document.ID_PROPERTY_NAME)) throw new Error("The top-level " + Document.ID_PROPERTY_NAME + " field is the only field currently supported for exclusion; code 16406");
-					exprObj.excludeId = true;
-				}
-				break;
-			default:
-				throw new Error("disallowed field type " + (fv ? fv.constructor.name + ":" : "") + typeof(fv) + " in object expression (at '" + fn + "')");
+
+			var fieldValue = obj[fieldName];
+			switch (typeof(fieldValue)) {
+				case "object":
+					// it's a nested document
+					var subCtx = new ObjectCtx({
+						isDocumentOk: ctx.isDocumentOk,
+						isInclusionOk: ctx.isInclusionOk
+					});
+
+					expressionObject.addField(fieldName, Expression.parseObject(fieldValue, subCtx, vps));
+
+					break;
+				case "string":
+					// it's a renamed field
+					// CW TODO could also be a constant
+					expressionObject.addField(fieldName, FieldPathExpression.parse(fieldValue, vps));
+					break;
+				case "boolean":
+				case "number":
+					// it's an inclusion specification
+					if (fieldValue) {
+						if (!ctx.isInclusionOk)
+							throw new Error("field inclusion is not allowed inside of $expressions; uassert code 16420");
+						expressionObject.includePath(fieldName);
+					} else {
+						if (!(ctx.isTopLevel && fieldName === Document.ID_PROPERTY_NAME))
+							throw new Error("The top-level " + Document.ID_PROPERTY_NAME + " field is the only field currently supported for exclusion; uassert code 16406");
+						expressionObject.excludeId = true;
+					}
+					break;
+				default:
+					throw new Error("disallowed field type " + Value.getType(fieldValue) + " in object expression (at '" + fieldName + "') uassert code 15992");
 			}
 		}
 	}
-	return expr;
+
+	return expression;
 };
 
-/**
- * Parse a BSONElement Object which has already been determined to be functional expression.
- *
- * @static
- * @method parseExpression
- * @param opName	the name of the (prefix) operator
- * @param obj	the BSONElement to parse
- * @returns the parsed Expression
- **/
-klass.parseExpression = function parseExpression(opName, obj) {
-	// look for the specified operator
-	if (opName === "$const") return new ConstantExpression(obj); //TODO: createFromBsonElement was here, not needed since this isn't BSON?
-	var op = klass.opMap[opName];
-	if (!(op instanceof OpDesc)) throw new Error("invalid operator " + opName + "; code 15999");
 
-	// make the expression node
-	var IExpression = op.factory,	//TODO: should this get renamed from `factory` to `ctor` or something?
-		expr = new IExpression();
-
-	// add the operands to the expression node
-	if (op.flags & OpDesc.FIXED_COUNT && op.argCount > 1 && !(obj instanceof Array)) throw new Error("the " + op.name + " operator requires an array of " + op.argCount + " operands; code 16019");
-	var operand; // used below
-	if (obj.constructor === Object) { // the operator must be unary and accept an object argument
-		if (!(op.flags & OpDesc.OBJECT_ARG)) throw new Error("the " + op.name + " operator does not accept an object as an operand");
-		operand = Expression.parseObject(obj, new ObjectCtx({isDocumentOk: 1}));
-		expr.addOperand(operand);
-	} else if (obj instanceof Array) { // multiple operands - an n-ary operator
-		if (op.flags & OpDesc.FIXED_COUNT && op.argCount !== obj.length) throw new Error("the " + op.name + " operator requires " + op.argCount + " operand(s); code 16020");
-		for (var i = 0, n = obj.length; i < n; ++i) {
-			operand = Expression.parseOperand(obj[i]);
-			expr.addOperand(operand);
-		}
-	} else { //assume it's an atomic operand
-		if (op.flags & OpDesc.FIXED_COUNT && op.argCount != 1) throw new Error("the " + op.name + " operator requires an array of " + op.argCount + " operands; code 16022");
-		operand = Expression.parseOperand(obj);
-		expr.addOperand(operand);
-	}
+klass.expressionParserMap = {};
 
-	return expr;
-};
 
 /**
- * Parse a BSONElement which is an operand in an Expression.
- *
- * @static
- * @param pBsonElement the expected operand's BSONElement
- * @returns the parsed operand, as an Expression
- **/
-klass.parseOperand = function parseOperand(obj){
-	var t = typeof(obj);
-	if (t === "string" && obj[0] == "$") { //if we got here, this is a field path expression
-		var path = Expression.removeFieldPrefix(obj);
-		return new FieldPathExpression(path);
+ * Registers an ExpressionParser so it can be called from parseExpression and friends.
+ * As an example, if your expression looks like {"$foo": [1,2,3]} you would add this line:
+ * REGISTER_EXPRESSION("$foo", ExpressionFoo::parse);
+ */
+klass.registerExpression = function registerExpression(key, parserFunc) {
+	if (key in klass.expressionParserMap) {
+		throw new Error("Duplicate expression (" + key + ") detected; massert code 17064");
 	}
-	else if (t === "object" && obj && obj.constructor === Object) return Expression.parseObject(obj, new ObjectCtx({isDocumentOk: true}));
-	else return new ConstantExpression(obj);
+	klass.expressionParserMap[key] = parserFunc;
+	return 1;
 };
 
-/**
- * Produce a field path string with the field prefix removed.
- * Throws an error if the field prefix is not present.
- *
- * @static
- * @param prefixedField the prefixed field
- * @returns the field path with the prefix removed
- **/
-klass.removeFieldPrefix = function removeFieldPrefix(prefixedField) {
-	if (prefixedField.indexOf("\0") != -1) throw new Error("field path must not contain embedded null characters; code 16419");
-	if (prefixedField[0] !== "$") throw new Error("field path references must be prefixed with a '$' ('" + prefixedField + "'); code 15982");
-	return prefixedField.substr(1);
-};
 
+//NOTE: DEVIATION FROM MONGO: the c++ version has 2 arguments, not 3.	//TODO: could easily fix this inconsistency
 /**
- * returns the signe of a number
- *
+ * Parses a BSONElement which has already been determined to be functional expression.
  * @static
- * @method signum
- * @returns the sign of a number; -1, 1, or 0
- **/
-klass.signum = function signum(i) {
-	if (i < 0) return -1;
-	if (i > 0) return 1;
-	return 0;
+ * @method parseExpression
+ * @param exprElement should be the only element inside the expression object.
+ *    That is the field name should be the $op for the expression.
+ * @param vps the variable parse state
+ * @returns the parsed Expression
+ */
+klass.parseExpression = function parseExpression(exprElementKey, exprElementValue, vps) {
+	var opName = exprElementKey,
+		op = Expression.expressionParserMap[opName];
+	if (!op) throw new Error("invalid operator : " + exprElementKey + "; uassert code 15999");
+
+	// make the expression node
+	return op(exprElementValue, vps);
 };
 
 
-// PROTOTYPE MEMBERS
 /**
- * Evaluate the Expression using the given document as input.
+ * Parses a BSONElement which is an operand in an Expression.
  *
- * @method evaluate
- * @returns the computed value
- **/
-proto.evaluate = function evaluate(obj) {
-	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");
+ * This is the most generic parser and can parse ExpressionFieldPath, a literal, or a $op.
+ * If it is a $op, exprElement should be the outer element whose value is an Object
+ * containing the $op.
+ *
+ * @method parseOperand
+ * @static
+ * @param exprElement should be the only element inside the expression object.
+ *    That is the field name should be the $op for the expression.
+ * @param vps the variable parse state
+ * @returns the parsed operand, as an Expression
+ */
+klass.parseOperand = function parseOperand(exprElement, vps) {
+	var t = typeof(exprElement);
+	if (t === "string" && exprElement[0] === "$") {
+		//if we got here, this is a field path expression
+	    return FieldPathExpression.parse(exprElement, vps);
+	} else if (t === "object" && exprElement && exprElement.constructor === Object) {
+		var oCtx = new ObjectCtx({
+			isDocumentOk: true
+		});
+		return Expression.parseObject(exprElement, oCtx, vps);
+	} else {
+		return ConstantExpression.parse(exprElement, vps);
+	}
 };
 
+
 /**
  * Optimize the Expression.
  *
  * This provides an opportunity to do constant folding, or to collapse nested
- *  operators that have the same precedence, such as $add, $and, or $or.
+ * operators that have the same precedence, such as $add, $and, or $or.
  *
  * The Expression should be replaced with the return value, which may or may
- *  not be the same object.  In the case of constant folding, a computed
- *  expression may be replaced by a constant.
+ * not be the same object.  In the case of constant folding, a computed
+ * expression may be replaced by a constant.
  *
  * @method optimize
  * @returns the optimized Expression
- **/
+ */
 proto.optimize = function optimize() {
-	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");
+	return this;
 };
 
+
 /**
- * Add this expression's field dependencies to the set Expressions are trees, so this is often recursive.
- *
- * Top-level ExpressionObject gets pointer to empty vector.
- * If any other Expression is an ancestor, or in other cases where {a:1} inclusion objects aren't allowed, they get NULL.
+ * Add this expression's field dependencies to the set.
+ * Expressions are trees, so this is often recursive.
  *
  * @method addDependencies
- * @param deps	output parameter
- * @param path	path to self if all ancestors are ExpressionObjects.
- **/
+ * @param deps Fully qualified paths to depended-on fields are added to this set.
+ *             Empty string means need full document.
+ * @param path path to self if all ancestors are ExpressionObjects.
+ *             Top-level ExpressionObject gets pointer to empty vector.
+ *             If any other Expression is an ancestor, or in other cases
+ *             where {a:1} inclusion objects aren't allowed, they get
+ *             NULL.
+ */
 proto.addDependencies = function addDependencies(deps, path) {
 	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");
 };
 
+
 /**
  * simple expressions are just inclusion exclusion as supported by ExpressionObject
- * @method getIsSimple
- **/
-proto.getIsSimple = function getIsSimple() {
+ * @method isSimple
+ */
+proto.isSimple = function isSimple() {
 	return false;
 };
 
-proto.toMatcherBson = function toMatcherBson(){
-	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");	//verify(false && "Expression::toMatcherBson()");
+/**
+ * Serialize the Expression tree recursively.
+ * If explain is false, returns a Value parsable by parseOperand().
+ * @method serialize
+ */
+proto.serialize = function serialize(explain) {
+	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");
+};
+
+/**
+ * Evaluate expression with specified inputs and return result.
+ *
+ * While vars is non-const, if properly constructed, subexpressions modifications to it
+ * should not effect outer expressions due to unique variable Ids.
+ *
+ * @method evaluate
+ * @param vars
+ */
+proto.evaluate = function evaluate(vars) {
+	if (!(vars instanceof Variables)) vars = new Variables(0, vars); /// Evaluate expression with specified inputs and return result. (only used by tests)
+	return this.evaluateInternal(vars);
+};
+
+/**
+ * Produce a field path string with the field prefix removed.
+ * Throws an error if the field prefix is not present.
+ * @method removeFieldPrefix
+ * @static
+ * @param prefixedField the prefixed field
+ * @returns the field path with the prefix removed
+ */
+klass.removeFieldPrefix = function removeFieldPrefix(prefixedField) {
+	if (prefixedField.indexOf("\0") !== -1) throw new Error("field path must not contain embedded null characters; uassert code 16419");
+	if (prefixedField[0] !== "$") throw new Error("field path references must be prefixed with a '$' ('" + prefixedField + "'); uassert code 15982");
+	return prefixedField.substr(1);
+};
+
+/**
+ * Evaluate the subclass Expression using the given Variables as context and return result.
+ * @method evaluate
+ * @returns the computed value
+ */
+proto.evaluateInternal = function evaluateInternal(vars) {
+	throw new Error("WAS NOT IMPLEMENTED BY INHERITOR!");
 };
+
+var ObjectExpression = require("./ObjectExpression"),
+	FieldPathExpression = require("./FieldPathExpression"),
+	ConstantExpression = require("./ConstantExpression");

lib/pipeline/expressions/FieldPathExpression.js

@@ -1,93 +1,160 @@
 "use strict";
 
+var Expression = require("./Expression"),
+    Variables = require("./Variables"),
+    Value = require("../Value"),
+    FieldPath = require("../FieldPath");
+
 /**
- * Create a field path expression. Evaluation will extract the value associated with the given field path from the source document.
+ * Create a field path expression.
+ *
+ * Evaluation will extract the value associated with the given field
+ * path from the source document.
+ *
  * @class FieldPathExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @extends mungedb-aggregate.pipeline.expressions.Expression
  * @constructor
- * @param {String} fieldPath the field path string, without any leading document indicator
+ * @param {String} theFieldPath the field path string, without any leading document indicator
+ */
+var FieldPathExpression = module.exports = function FieldPathExpression(theFieldPath, variable) {
+    if (arguments.length != 2) throw new Error(klass.name + ": expected args: theFieldPath[, variable]");
+    this._fieldPath = new FieldPath(theFieldPath);
+    this._variable = variable;
+}, klass = FieldPathExpression, base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ * Create a field path expression using old semantics (rooted off of CURRENT).
+ *
+ * // NOTE: this method is deprecated and only used by tests
+ * // TODO remove this method in favor of parse()
+ *
+ * Evaluation will extract the value associated with the given field
+ * path from the source document.
+ *
+ * @param fieldPath the field path string, without any leading document
+ * indicator
+ * @returns the newly created field path expression
  **/
-var FieldPathExpression = module.exports = function FieldPathExpression(path){
-	if (arguments.length !== 1) throw new Error("args expected: path");
-	this.path = new FieldPath(path);
-}, klass = FieldPathExpression, base = require("./Expression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+klass.create = function create(fieldPath) {
+    return new FieldPathExpression("CURRENT." + fieldPath, Variables.ROOT_ID);
+};
+
+// this is the new version that supports every syntax
+/**
+ * Like create(), but works with the raw string from the user with the "$" prefixes.
+ * @param raw raw string fieldpath
+ * @param vps variablesParseState
+ * @returns a new FieldPathExpression
+ */
+klass.parse = function parse(raw, vps) {
+    if (raw[0] !== "$") throw new Error("FieldPath: '" + raw + "' doesn't start with a $; uassert code 16873");
+    if (raw.length < 2) throw new Error("'$' by itself is not a valid FieldPath; uassert code 16872"); // need at least "$" and either "$" or a field name
+    if (raw[1] === "$") {
+        var fieldPath = raw.substr(2), // strip off $$
+            varName = fieldPath.substr(0, fieldPath.indexOf("."));
+        Variables.uassertValidNameForUserRead(varName);
+        return new FieldPathExpression(raw.slice(2), vps.getVariableName(varName));
+    } else {
+        return new FieldPathExpression("CURRENT." + raw.substr(1), vps.getVariable("CURRENT"));
+    }
+};
 
-// DEPENDENCIES
-var FieldPath = require("../FieldPath");
+proto.optimize = function optimize() {
+    // nothing can be done for these
+    return this;
+};
 
-// PROTOTYPE MEMBERS
-proto.evaluate = function evaluate(obj){
-	return this._evaluatePath(obj, 0, this.path.fields.length);
+proto.addDependencies = function addDependencies(deps) {
+    if (this._variable === Variables.ROOT_ID) {
+        if (this._fieldPath.fieldNames.length === 1) {
+            deps.needWholeDocument = true; // need full doc if just "$$ROOT"
+        } else {
+            deps.fields[this._fieldPath.tail().getPath(false)] = 1;
+        }
+    }
 };
 
 /**
- * Internal implementation of evaluate(), used recursively.
+ * Helper for evaluatePath to handle Array case
+ */
+proto._evaluatePathArray = function _evaluatePathArray(index, input) {
+    if (!(input instanceof Array)) throw new Error("must be array; dassert");
+
+    // Check for remaining path in each element of array
+    var result = [];
+    for (var i = 0, l = input.length; i < l; i++) {
+        if (!(input[i] instanceof Object))
+            continue;
+
+        var nested = this._evaluatePath(index, input[i]);
+        if (nested !== undefined)
+            result.push(nested);
+    }
+    return result;
+};
+
+/**
+ * Internal implementation of evaluateInternal(), used recursively.
  *
- * The internal implementation doesn't just use a loop because of the
- * possibility that we need to skip over an array.  If the path is "a.b.c",
- * and a is an array, then we fan out from there, and traverse "b.c" for each
- * element of a:[...].  This requires that a be an array of objects in order
- * to navigate more deeply.
+ * The internal implementation doesn't just use a loop because of
+ * the possibility that we need to skip over an array.  If the path
+ * is "a.b.c", and a is an array, then we fan out from there, and
+ * traverse "b.c" for each element of a:[...].  This requires that
+ * a be an array of objects in order to navigate more deeply.
  *
  * @param index current path field index to extract
- * @param pathLength maximum number of fields on field path
- * @param pDocument current document traversed to (not the top-level one)
+ * @param input current document traversed to (not the top-level one)
  * @returns the field found; could be an array
- **/
-proto._evaluatePath = function _evaluatePath(obj, i, len){
-	var fieldName = this.path.fields[i],
-		field = obj[fieldName]; // It is possible we won't have an obj (document) and we need to not fail if that is the case
-
-	// if the field doesn't exist, quit with an undefined value
-	if (field === undefined) return undefined;
-
-	// if we've hit the end of the path, stop
-	if (++i >= len) return field;
-
-	// We're diving deeper.  If the value was null, return null
-	if(field === null) return undefined;
-
-	if (field.constructor === Object) {
-		return this._evaluatePath(field, i, len);
-	} else if (Array.isArray(field)) {
-		var results = [];
-		for (var i2 = 0, l2 = field.length; i2 < l2; i2++) {
-			var subObj = field[i2],
-				subObjType = typeof(subObj);
-			if (subObjType === "undefined" || subObj === null) {
-				results.push(subObj);
-			} else if (subObj.constructor === Object) {
-				results.push(this._evaluatePath(subObj, i, len));
-			} else {
-				throw new Error("the element '" + fieldName + "' along the dotted path '" + this.path.getPath() + "' is not an object, and cannot be navigated.; code 16014");
-			}
-		}
-		return results;
-	}
-	return undefined;
-};
+ */
+proto._evaluatePath = function _evaluatePath(index, input) {
+    // Note this function is very hot so it is important that is is well optimized.
+    // In particular, all return paths should support RVO.
 
-proto.optimize = function(){
-	return this;
-};
+    // if we've hit the end of the path, stop
+    if (index == this._fieldPath.fieldNames.length - 1)
+        return input[this._fieldPath.fieldNames[index]];
 
-proto.addDependencies = function addDependencies(deps){
-	deps[this.path.getPath()] = 1;
-	return deps;
+    // Try to dive deeper
+    var val = input[this._fieldPath.fieldNames[index]];
+    if (val instanceof Object && val.constructor === Object) {
+        return this._evaluatePath(index + 1, val);
+    } else if (val instanceof Array) {
+        return this._evaluatePathArray(index + 1, val);
+    } else {
+        return undefined;
+    }
 };
 
-// renamed write to get because there are no streams
-proto.getFieldPath = function getFieldPath(usePrefix){
-	return this.path.getPath(usePrefix);
-};
+proto.evaluateInternal = function evaluateInternal(vars) {
+    if (this._fieldPath.fieldNames.length === 1) // get the whole variable
+        return vars.getValue(this._variable);
 
-proto.toJSON = function toJSON(){
-	return this.path.getPath(true);
+    if (this._variable === Variables.ROOT_ID) {
+        // ROOT is always a document so use optimized code path
+        return this._evaluatePath(1, vars.getRoot());
+    }
+
+    var val = vars.getValue(this._variable);
+    if (val instanceof Object && val.constructor === Object) {
+        return this._evaluatePath(1, val);
+    } else if(val instanceof Array) {
+        return this._evaluatePathArray(1,val);
+    } else {
+        return undefined;
+    }
 };
 
-//TODO: proto.addToBsonObj = ...?
-//TODO: proto.addToBsonArray = ...?
+proto.serialize = function serialize(){
+    if(this._fieldPath.fieldNames[0] === "CURRENT" && this._fieldPath.fieldNames.length > 1) {
+        // use short form for "$$CURRENT.foo" but not just "$$CURRENT"
+        return "$" + this._fieldPath.tail().getPath(false);
+    } else {
+        return "$$" + this._fieldPath.getPath(false);
+    }
+};
 
-//proto.writeFieldPath = ...?   use #getFieldPath instead
+proto.getFieldPath = function getFieldPath(){
+    return this._fieldPath;
+};

lib/pipeline/expressions/FieldRangeExpression.js

@@ -6,9 +6,9 @@
  * Field ranges are meant to match up with classic Matcher semantics, and therefore are conjunctions.
  *
  * For example, these appear in mongo shell predicates in one of these forms:
- *	{ a : C } -> (a == C) // degenerate "point" range
- *	{ a : { $lt : C } } -> (a < C) // open range
- *	{ a : { $gt : C1, $lte : C2 } } -> ((a > C1) && (a <= C2)) // closed
+ *      { a : C } -> (a == C) // degenerate "point" range
+ *      { a : { $lt : C } } -> (a < C) // open range
+ *      { a : { $gt : C1, $lte : C2 } } -> ((a > C1) && (a <= C2)) // closed
  *
  * When initially created, a field range only includes one end of the range.  Additional points may be added via intersect().
  *
@@ -141,6 +141,8 @@ var Range = (function(){
 proto.evaluate = function evaluate(obj){
 	if(this.range === undefined) return false;
 	var value = this.pathExpr.evaluate(obj);
+	if(value instanceof Array)
+		throw new Error('FieldRangeExpression cannot evaluate an array.');
 	return this.range.contains(value);
 };
 

lib/pipeline/expressions/FixedArityExpressionT.js

@@ -0,0 +1,36 @@
+"use strict";
+
+/**
+ * A factory and base class for expressions that take a fixed number of arguments
+ * @class FixedArityExpressionT
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+
+var FixedArityExpressionT = module.exports = function FixedArityExpressionT(SubClass, nArgs) {
+
+	var FixedArityExpression = function FixedArityExpression() {
+		if (arguments.length !== 0) throw new Error(klass.name + "<" + SubClass.name + ">: zero args expected");
+		base.call(this);
+	}, klass = FixedArityExpression, base = require("./NaryBaseExpressionT")(SubClass), proto = klass.prototype = Object.create(base.prototype, {constructor: {value: klass}});
+
+	/**
+	 * Check that the number of args is what we expected
+	 * @method validateArguments
+	 * @param args Array The array of arguments to the expression
+	 * @throws
+	 **/
+	proto.validateArguments = function validateArguments(args) {
+		if(args.length !== nArgs) {
+			throw new Error("Expression " + this.getOpName() + " takes exactly " +
+				nArgs + " arguments. " + args.length + " were passed in.");
+		}
+	};
+
+	klass.parse = base.parse; 						// NOTE: Need to explicitly
+	klass.parseArguments = base.parseArguments;		// bubble static members in
+													// our inheritance chain
+	return FixedArityExpression;
+};
+

lib/pipeline/expressions/HourExpression.js

@@ -1,34 +1,41 @@
 "use strict";
 
-/** 
- * An $hour pipeline expression. 
- * @see evaluate 
+/**
+ * An $hour pipeline expression.
+ * @see evaluateInternal
  * @class HourExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var HourExpression = module.exports = function HourExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = HourExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = HourExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor:{
+			value:klass
+		}
+	});
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$hour";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
+// DEPENDENCIES
+var Expression = require("./Expression");
 
-/** 
- * Takes a date and returns the hour between 0 and 23. 
- * @method evaluate
+/**
+ * Takes a date and returns the hour between 0 and 23.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars){
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCHours();
 };
+
+
+/** Register Expression */
+Expression.registerExpression("$hour",base.parse);

lib/pipeline/expressions/IfNullExpression.js

@@ -8,29 +8,38 @@
  * @module mungedb-aggregate
  * @constructor
  **/
-var IfNullExpression = module.exports = function IfNullExpression(){
+var IfNullExpression = module.exports = function IfNullExpression() {
 	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = IfNullExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = IfNullExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 2),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$ifNull";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
+// virtuals from ExpressionNary
 
 /**
  * Use the $ifNull operator with the following syntax: { $ifNull: [ <expression>, <replacement-if-null> ] }
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc);
-	if(left !== undefined && left !== null) return left;
-	var right = this.operands[1].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var left = this.operands[0].evaluateInternal(vars);
+	if (left !== undefined && left !== null) return left;
+	var right = this.operands[1].evaluateInternal(vars);
 	return right;
 };
+
+/** Register Expression */
+Expression.registerExpression("$ifNull", base.parse);

lib/pipeline/expressions/LetExpression.js

@@ -0,0 +1,119 @@
+"use strict";
+
+var LetExpression = module.exports = function LetExpression(vars, subExpression){
+	//if (arguments.length !== 2) throw new Error("Two args expected");
+	this._variables = vars;
+	this._subExpression = subExpression;
+}, klass = LetExpression, Expression = require("./Expression"), base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+Expression.registerExpression("$let", LetExpression.parse);
+
+// DEPENDENCIES
+var Variables = require("./Variables"),
+	VariablesParseState = require("./VariablesParseState");
+
+// PROTOTYPE MEMBERS
+
+
+proto.parse = function parse(expr, vpsIn){
+	if(!("$let" in expr)) {
+		throw new Error("Tried to create a $let with something other than let. Looks like your parse map went all funny.");
+	}
+
+	if(typeof(expr.$let) !== 'object' || (expr.$let instanceof Array)) {
+		throw new Error("$let only supports an object as its argument: 16874");
+	}
+
+	var args = expr.$let,
+		varsElem = args.vars,
+		inElem = args['in']; // args.in; ??
+
+	//NOTE: DEVIATION FROM MONGO: 1. These if statements are in a loop in the c++ version,
+	// 2. 'vars' and 'in' are each mandatory here. in the c++ code you only need one of the two.
+	// 3. Below, we croak if there are more than 2 arguments.  The original does not have this limitation, specifically.
+	// Upon further review, I think our code is more accurate.  The c++ code will accept if there are multiple 'in'
+	// or 'var' values. The previous ones will be overwritten by newer ones.
+	//
+	// Final note - I think this code is fine.
+	//
+	if(!varsElem) {
+		throw new Error("Missing 'vars' parameter to $let: 16876");
+	}
+	if(!inElem) {
+		throw new Error("Missing 'in' parameter to $let: 16877");
+	}
+
+	// Should this be !== 2?  Why would we have fewer than 2 arguments?  Why do we even care what the length of the
+	// array is? It may be an optimization of sorts. But what we're really wanting here is, 'If any keys are not "in"
+	// or "vars" then we need to bugcheck.'
+	if(Object.keys(args).length > 2) {
+		var bogus = Object.keys(args).filter(function(x) {return !(x === 'in' || x === 'vars');});
+		throw new Error("Unrecognized parameter to $let: " + bogus.join(",") + "- 16875");
+	}
+
+	var vpsSub = new VariablesParseState(vpsIn),
+		vars = {};
+
+	for(var varName in varsElem) {
+		Variables.uassertValidNameForUserWrite(varName);
+		var id = vpsSub.defineVariable(varName);
+
+		vars[id] = {};
+		vars[id][varName] = Expression.parseOperand(varsElem, vpsIn);
+	}
+
+	var subExpression = Expression.parseOperand(inElem, vpsSub);
+	return new LetExpression(vars, subExpression);
+};
+
+proto.optimize = function optimize() {
+	if(this._variables.empty()) {
+		return this._subExpression.optimize();
+	}
+
+	for(var id in this._variables){
+		for(var name in this._variables[id]) {
+			//NOTE: DEVIATION FROM MONGO: This is actually ok. The c++ code does this with a single map. The js structure
+			// is nested objects.
+			this._variables[id][name] = this._variables[id][name].optimize();
+		}
+	}
+
+	this._subExpression = this._subExpression.optimize();
+
+	return this;
+};
+
+proto.serialize = function serialize(explain) {
+	var vars = {};
+	for(var id in this._variables) {
+		for(var name in this._variables[id]) {
+			vars[name] = this._variables[id][name];
+		}
+	}
+
+	return {$let: {vars:vars, 'in':this._subExpression.serialize(explain)}};
+};
+
+proto.evaluateInternal = function evaluateInternal(vars) {
+	for(var id in this._variables) {
+		for(var name in this._variables[id]) {
+			vars.setValue(id, this._variables[id][name]);
+		}
+	}
+
+	return this._subExpression.evaluateInternal(vars);
+};
+
+proto.addDependencies = function addDependencies(deps, path){
+	for(var id in this._variables) {
+		for(var name in this._variables[id]) {
+			this._variables[id][name].addDependencies(deps);
+		}
+	}
+	this._subExpression.addDependencies(deps, path);
+	return deps; //NOTE: DEVIATION FROM MONGO: The c++ version does not return a value. We seem to use the returned value
+					// (or something from a different method named
+					// addDependencies) in many places.
+
+};

lib/pipeline/expressions/MapExpression.js

@@ -0,0 +1,107 @@
+"use strict";
+
+var MapExpression = module.exports = function MapExpression(varName, varId, input, each){
+	if (arguments.length !== 4) throw new Error("Four args expected");
+	this._varName = varName;
+	this._varId = varId;
+	this._input = input;
+	this._each = each;
+}, klass = MapExpression, Expression = require("./Expression"), base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var Variables = require("./Variables"),
+	VariablesParseState = require("./VariablesParseState");
+
+// PROTOTYPE MEMBERS
+
+
+klass.parse = function parse(expr, vpsIn){
+	if(!("$map" in expr)) {
+		throw new Error("Tried to create a $let with something other than let. Looks like your parse map went all funny.");
+	}
+
+	if(typeof(expr.$map) !== 'object' || (expr.$map instanceof Array)) {
+		throw new Error("$map only supports an object as it's argument:16878");
+	}
+
+	var args = expr.$map,
+		inputElem = args.input,
+		inElem = args['in'],
+		asElem = args.as;
+
+	if(!inputElem) {
+		throw new Error("Missing 'input' parameter to $map: 16880");
+	}
+	if(!asElem) {
+		throw new Error("Missing 'as' parameter to $map: 16881");
+	}
+	if(!inElem) {
+		throw new Error("Missing 'in' parameter to $let: 16882");
+	}
+
+
+	if(Object.keys(args).length > 3) {
+		var bogus = Object.keys(args).filter(function(x) {return !(x === 'in' || x === 'as' || x === 'input');});
+		throw new Error("Unrecognized parameter to $map: " + bogus.join(",") + "- 16879");
+	}
+
+	var input = Expression.parseOperand(inputElem, vpsIn);
+
+	var vpsSub = new VariablesParseState(vpsIn),
+		varName = asElem;
+
+	Variables.uassertValidNameForUserWrite(varName);
+	var varId = vpsSub.defineVariable(varName);
+
+	var invert = Expression.parseOperand(inElem, vpsSub);
+
+	return new MapExpression(varName, varId, input, invert);
+};
+
+
+proto.optimize = function optimize() {
+	this._input = this._input.optimize();
+	this._each = this._each.optimize();
+	return this;
+};
+
+proto.serialize = function serialize(explain) {
+	return {$map: {input:this._input.serialize(explain),
+				   as: this._varName,
+				   'in': this._each.serialize(explain)}};
+};
+
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var inputVal = this._input.evaluateInternal(vars);
+	if( inputVal === null) {
+		return null;
+	}
+
+	if(!(inputVal instanceof Array)) {
+		throw new Error("Input to $map must be an Array, not a ____ 16883");
+	}
+
+	if(inputVal.length === 0) {
+		return [];
+	}
+
+	// Diverge from Mongo source here, as Javascript has a builtin map operator.
+	return inputVal.map(function(x) {
+	   vars.setValue(this._varId, x);
+	   var toInsert = this._each.evaluateInternal(vars);
+	   if(toInsert === undefined) {
+		   toInsert = null;
+	   }
+
+	   return toInsert;
+   });
+};
+
+proto.addDependencies = function addDependencies(deps, path){
+	this._input.addDependencies(deps, path);
+	this._each.addDependencies(deps, path);
+	return deps;
+};
+
+
+Expression.registerExpression("$map", klass.parse);

lib/pipeline/expressions/MillisecondExpression.js

@@ -0,0 +1,42 @@
+"use strict";
+
+/**
+ * An $millisecond pipeline expression.
+ * @see evaluateInternal
+ * @class MillisecondExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var MillisecondExpression = module.exports = function MillisecondExpression() {
+	base.call(this);
+}, klass = MillisecondExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$millisecond";
+};
+
+/**
+ * Takes a date and returns the millisecond between 0 and 999, but can be 1000 to account for leap milliseconds.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
+	return date.getUTCMilliseconds(); //TODO: incorrect for last millisecond of leap year, need to fix...
+	// currently leap milliseconds are unsupported in v8
+	// http://code.google.com/p/v8/issues/detail?id=1944
+};
+
+/** Register Expression */
+Expression.registerExpression("$millisecond", base.parse);

lib/pipeline/expressions/MinuteExpression.js

@@ -1,34 +1,40 @@
 "use strict";
 
-/** 
- * An $minute pipeline expression. 
- * @see evaluate 
+/**
+ * An $minute pipeline expression.
+ * @see evaluateInternal
  * @class MinuteExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var MinuteExpression = module.exports = function MinuteExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var MinuteExpression = module.exports = function MinuteExpression() {
 	base.call(this);
-}, klass = MinuteExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = MinuteExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$minute";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Takes a date and returns the minute between 0 and 59. 
- * @method evaluate
+/**
+ * Takes a date and returns the minute between 0 and 59.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCMinutes();
 };
+
+/** Register Expression */
+Expression.registerExpression("$minute", base.parse);

lib/pipeline/expressions/ModExpression.js

@@ -1,49 +1,54 @@
 "use strict";
 
-/** 
- * An $mod pipeline expression. 
- * @see evaluate 
+/**
+ * An $mod pipeline expression.
+ * @see evaluate
  * @class ModExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var ModExpression = module.exports = function ModExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var ModExpression = module.exports = function ModExpression() {
 	base.call(this);
-}, klass = ModExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = ModExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 2),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$mod";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Takes an array that contains a pair of numbers and returns the remainder of the first number divided by the second number. 
+/**
+ * Takes an array that contains a pair of numbers and returns the remainder of the first number divided by the second number.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars) {
 	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc),
-		right = this.operands[1].evaluate(doc);
-	if(left instanceof Date || right instanceof Date) throw new Error("$mod does not support dates; code 16374");
+	var left = this.operands[0].evaluateInternal(vars),
+		right = this.operands[1].evaluateInternal(vars);
+	if (left instanceof Date || right instanceof Date) throw new Error("$mod does not support dates; code 16374");
 
 	// pass along jstNULLs and Undefineds
-	if(left === undefined || left === null) return left;
-	if(right === undefined || right === null) return right;
+	if (left === undefined || left === null) return left;
+	if (right === undefined || right === null) return right;
 
 	// ensure we aren't modding by 0
 	right = Value.coerceToDouble(right);
-	if(right === 0) return undefined;
+	if (right === 0) return undefined;
 
 	left = Value.coerceToDouble(left);
 	return left % right;
 };
+
+/** Register Expression */
+Expression.registerExpression("$mod", base.parse);

lib/pipeline/expressions/MonthExpression.js

@@ -1,34 +1,40 @@
 "use strict";
 
-/** 
- * A $month pipeline expression. 
- * @see evaluate 
+/**
+ * A $month pipeline expression.
+ * @see evaluate
  * @class MonthExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var MonthExpression = module.exports = function MonthExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var MonthExpression = module.exports = function MonthExpression() {
 	base.call(this);
-}, klass = MonthExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
-	
+}, klass = MonthExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
+
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$month";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the month as a number between 1 and 12.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
-	return date.getUTCMonth() + 1;
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
+	return date.getUTCMonth();
 };
+
+/** Register Expression */
+Expression.registerExpression("$month", base.parse);

lib/pipeline/expressions/MultiplyExpression.js

@@ -1,34 +1,36 @@
 "use strict";
 
-/** 
- * A $multiply pipeline expression. 
- * @see evaluate 
+/**
+ * A $multiply pipeline expression.
+ * @see evaluateInternal
  * @class MultiplyExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var MultiplyExpression = module.exports = function MultiplyExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+	//if (arguments.length !== 0) throw new Error("Zero args expected");
 	base.call(this);
-}, klass = MultiplyExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = MultiplyExpression, base = require("./VariadicExpressionT")(MultiplyExpression), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+ Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
+klass.opName = "$multiply";
 proto.getOpName = function getOpName(){
-	return "$multiply";
+	return klass.opName;
 };
 
-/** 
- * Takes an array of one or more numbers and multiples them, returning the resulting product. 
- * @method evaluate
+/**
+ * Takes an array of one or more numbers and multiples them, returning the resulting product.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars){
 	var product = 1;
 	for(var i = 0, n = this.operands.length; i < n; ++i){
-		var value = this.operands[i].evaluate(doc);
+		var value = this.operands[i].evaluateInternal(vars);
 		if(value instanceof Date) throw new Error("$multiply does not support dates; code 16375");
 		product *= Value.coerceToDouble(value);
 	}
@@ -36,6 +38,5 @@ proto.evaluate = function evaluate(doc){
 	return product;
 };
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
-};
+/** Register Expression */
+Expression.registerExpression(klass.opName, base.parse);

lib/pipeline/expressions/NaryBaseExpressionT.js

@@ -0,0 +1,30 @@
+"use strict";
+
+/**
+ * Inherit from ExpressionVariadic or ExpressionFixedArity instead of directly from this class.
+ * @class NaryBaseExpressionT
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @extends mungedb-aggregate.pipeline.expressions.NaryExpression
+ * @constructor
+ */
+var NaryBaseExpressionT = module.exports = function NaryBaseExpressionT(SubClass) {
+
+	var NaryBaseExpression = function NaryBaseExpression() {
+		if (arguments.length !== 0) throw new Error(klass.name + "<" + SubClass.name + ">: zero args expected");
+		base.call(this);
+	}, klass = NaryBaseExpression, NaryExpression = require("./NaryExpression"), base = NaryExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+	klass.parse = function(objExpr, vps) {
+		var expr = new SubClass(),
+			args = NaryExpression.parseArguments(objExpr, vps);
+		expr.validateArguments(args);
+		expr.operands = args;
+		return expr;
+	};
+
+	klass.parseArguments = base.parseArguments;		// NOTE: Need to explicitly
+													// bubble static members in
+													// our inheritance chain
+	return NaryBaseExpression;
+};

lib/pipeline/expressions/NaryExpression.js

@@ -7,132 +7,146 @@
  * @module mungedb-aggregate
  * @extends mungedb-aggregate.pipeline.expressions.Expression
  * @constructor
- **/
-var NaryExpression = module.exports = function NaryExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+ */
+var NaryExpression = module.exports = function NaryExpression() {
+	if (arguments.length !== 0) throw new Error("Zero args expected");
 	this.operands = [];
 	base.call(this);
-}, klass = NaryExpression, base = require("./Expression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = NaryExpression, Expression = require("./Expression"), base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
-// DEPENDENCIES
-var ConstantExpression = require("./ConstantExpression");
+var Variables = require("./Variables"),
+	ConstantExpression = require("./ConstantExpression");
 
-// PROTOTYPE MEMBERS
-proto.evaluate = undefined; // evaluate(doc){ ... defined by inheritor ... }
+proto.optimize = function optimize() {
+	var n = this.operands.length;
 
-proto.getOpName = function getOpName(doc){
-	throw new Error("NOT IMPLEMENTED BY INHERITOR");
-};
+	// optimize sub-expressions and count constants
+	var constCount = 0;
+	for (var i = 0; i < n; ++i) {
+		var optimized = this.operands[i].optimize();
+
+		// substitute the optimized expression
+		this.operands[i] = optimized;
 
-proto.optimize = function optimize(){
-	var constsFound = 0,
-		stringsFound = 0;
-	for (var i = 0, l = this.operands.length; i < l; i++) {
-		var optimizedExpr = this.operands[i].optimize();
-		if (optimizedExpr instanceof ConstantExpression) {
-			constsFound++;
-			if (typeof(optimizedExpr.value) == "string") stringsFound++;
+		// check to see if the result was a constant
+		if (optimized instanceof ConstantExpression) {
+			constCount++;
 		}
-		this.operands[i] = optimizedExpr;
 	}
-	// If all the operands are constant, we can replace this expression with a constant.  We can find the value by evaluating this expression over a NULL Document because evaluating the ExpressionConstant never refers to the argument Document.
-	if (constsFound === l) return new ConstantExpression(this.evaluate());
-	// If there are any strings, we can't re-arrange anything, so stop now.     LATER:  we could concatenate adjacent strings as a special case.
-	if (stringsFound) return this;
-	// If there's no more than one constant, then we can't do any constant folding, so don't bother going any further.
-	if (constsFound <= 1) return this;
-	// If the operator isn't commutative or associative, there's nothing more we can do.  We test that by seeing if we can get a factory; if we can, we can use it to construct a temporary expression which we'll evaluate to collapse as many constants as we can down to a single one.
-	var IExpression = this.getFactory();
-	if (!(IExpression instanceof Function)) return this;
-	// Create a new Expression that will be the replacement for this one.  We actually create two:  one to hold constant expressions, and one to hold non-constants.
-	// Once we've got these, we evaluate the constant expression to produce a single value, as above.  We then add this operand to the end of the non-constant expression, and return that.
-	var expr = new IExpression(),
-		constExpr = new IExpression();
-	for (i = 0; i < l; ++i) {
-		var operandExpr = this.operands[i];
-		if (operandExpr instanceof ConstantExpression) {
-			constExpr.addOperand(operandExpr);
+
+	// If all the operands are constant, we can replace this expression with a constant. Using
+	// an empty Variables since it will never be accessed.
+	if (constCount === n) {
+		var emptyVars = new Variables(),
+			result = this.evaluateInternal(emptyVars),
+			replacement = ConstantExpression.create(result);
+		return replacement;
+	}
+
+	// Remaining optimizations are only for associative and commutative expressions.
+	if(!this.isAssociativeAndCommutative()) {
+		return this;
+	}
+
+	// Process vpOperand to split it into constant and nonconstant vectors.
+	// This can leave vpOperand in an invalid state that is cleaned up after the loop.
+	var constExprs = [],
+		nonConstExprs = [];
+	for (i = 0; i < this.operands.length; ++i) { // NOTE: vpOperand grows in loop
+		var expr = this.operands[i];
+		if (expr instanceof ConstantExpression) {
+			constExprs.push(expr);
 		} else {
-			// If the child operand is the same type as this, then we can extract its operands and inline them here because we already know this is commutative and associative because it has a factory.  We can detect sameness of the child operator by checking for equality of the factory
-			// Note we don't have to do this recursively, because we called optimize() on all the children first thing in this call to optimize().
-			if (!(operandExpr instanceof NaryExpression)) {
-				expr.addOperand(operandExpr);
+			// If the child operand is the same type as this, then we can
+			// extract its operands and inline them here because we know
+			// this is commutative and associative.  We detect sameness of
+			// the child operator by checking for equality of the opNames
+			var nary = expr instanceof NaryExpression ? expr : undefined;
+			if (!nary || nary.getOpName() !== this.getOpName) {
+				nonConstExprs.push(expr);
 			} else {
-				if (operandExpr.getFactory() !== IExpression) {
-					expr.addOperand(operandExpr);
-				} else { // same factory, so flatten
-					for (var i2 = 0, n2 = operandExpr.operands.length; i2 < n2; ++i2) {
-						var childOperandExpr = operandExpr.operands[i2];
-						if (childOperandExpr instanceof ConstantExpression) {
-							constExpr.addOperand(childOperandExpr);
-						} else {
-							expr.addOperand(childOperandExpr);
-						}
-					}
-				}
+				// same expression, so flatten by adding to vpOperand which
+				// will be processed later in this loop.
+				Array.prototype.push.apply(this.operands, nary.operands);
 			}
 		}
 	}
 
-	if (constExpr.operands.length === 1) { // If there was only one constant, add it to the end of the expression operand vector.
-		expr.addOperand(constExpr.operands[0]);
-	} else if (constExpr.operands.length > 1) { // If there was more than one constant, collapse all the constants together before adding the result to the end of the expression operand vector.
-		var pResult = constExpr.evaluate();
-		expr.addOperand(new ConstantExpression(pResult));
+	// collapse all constant expressions (if any)
+	var constValue;
+	if (constExprs.length > 0) {
+		this.operands = constExprs;
+		var emptyVars2 = new Variables();
+		constValue = this.evaluateInternal(emptyVars2);
+	}
+
+	// now set the final expression list with constant (if any) at the end
+	this.operands = nonConstExprs;
+	if (constExprs.length > 0) {
+		this.operands.push(ConstantExpression.create(constValue));
 	}
 
-	return expr;
+	return this;
 };
 
-proto.addDependencies = function addDependencies(deps){
-	for(var i = 0, l = this.operands.length; i < l; ++i)
+proto.addDependencies = function addDependencies(deps, path) {
+	for (var i = 0, l = this.operands.length; i < l; ++i) {
 		this.operands[i].addDependencies(deps);
-	return deps;
+	}
 };
 
 /**
  * Add an operand to the n-ary expression.
  * @method addOperand
- * @param pExpression the expression to add
- **/
+ * @param expr the expression to add
+ */
 proto.addOperand = function addOperand(expr) {
 	this.operands.push(expr);
 };
 
-proto.getFactory = function getFactory() {
-	return undefined;
-};
+proto.serialize = function serialize(explain) {
+	var nOperand = this.operands.length,
+		array = [];
+	// build up the array
+	for (var i = 0; i < nOperand; i++) {
+		array.push(this.operands[i].serialize(explain));
+	}
 
-proto.toJSON = function toJSON() {
-	var o = {};
-	o[this.getOpName()] = this.operands.map(function(operand){
-		return operand.toJSON();
-	});
-	return o;
+	var obj = {};
+	obj[this.getOpName()] = array;
+	return obj;
 };
 
-//TODO:	proto.toBson  ?   DONE NOW???
-//TODO:	proto.addToBsonObj  ?
-//TODO: proto.addToBsonArray  ?
+proto.isAssociativeAndCommutative = function isAssociativeAndCommutative() {
+	return false;
+};
 
 /**
- * Checks the current size of vpOperand; if the size equal to or greater than maxArgs, fires a user assertion indicating that this operator cannot have this many arguments.
- * The equal is there because this is intended to be used in addOperand() to check for the limit *before* adding the requested argument.
- *
- * @method checkArgLimit
- * @param maxArgs the maximum number of arguments the operator accepts
- **/
-proto.checkArgLimit = function checkArgLimit(maxArgs) {
-	if (this.operands.length >= maxArgs) throw new Error(this.getOpName() + " only takes " + maxArgs + " operand" + (maxArgs == 1 ? "" : "s") + "; code 15993");
+ * Get the name of the operator.
+ * @method getOpName
+ * @returns the name of the operator; this string belongs to the class
+ *  implementation, and should not be deleted
+ *  and should not
+ */
+proto.getOpName = function getOpName() {
+	throw new Error("NOT IMPLEMENTED BY INHERITOR");
 };
 
 /**
- * Checks the current size of vpOperand; if the size is not equal to reqArgs, fires a user assertion indicating that this must have exactly reqArgs arguments.
- * This is meant to be used in evaluate(), *before* the evaluation takes place.
- *
- * @method checkArgCount
- * @param reqArgs the number of arguments this operator requires
- **/
-proto.checkArgCount = function checkArgCount(reqArgs) {
-	if (this.operands.length !== reqArgs) throw new Error(this.getOpName() + ":  insufficient operands; " + reqArgs + " required, only got " + this.operands.length + "; code 15997");
-};
+ * Allow subclasses the opportunity to validate arguments at parse time.
+ * @method validateArguments
+ * @param {[type]} args [description]
+ */
+proto.validateArguments = function(args) {};
+
+klass.parseArguments = function(exprElement, vps) {
+	var out = [];
+	if (exprElement instanceof Array) {
+		for (var ii = 0; ii < exprElement.length; ii++) {
+			out.push(Expression.parseOperand(exprElement[ii], vps));
+		}
+	} else {
+		out.push(Expression.parseOperand(exprElement, vps));
+	}
+	return out;
+};

lib/pipeline/expressions/NotExpression.js

@@ -1,37 +1,42 @@
 "use strict";
 
-/** 
- * A $not pipeline expression. 
- * @see evaluate 
+/**
+ * A $not pipeline expression.
+ * @see evaluateInternal
  * @class NotExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var NotExpression = module.exports = function NotExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var NotExpression = module.exports = function NotExpression() {
+		if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = NotExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = NotExpression,
+	base = require("./FixedArityExpressionT")(klass, 1),
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
-	return "$not";
+klass.opName = "$not";
+proto.getOpName = function getOpName() {
+	return klass.opName;
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Returns the boolean opposite value passed to it. When passed a true value, $not returns false; when passed a false value, $not returns true. 
- * @method evaluate
+/**
+ * Returns the boolean opposite value passed to it. When passed a true value, $not returns false; when passed a false value, $not returns true.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var op = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var op = this.operands[0].evaluateInternal(vars);
 	return !Value.coerceToBool(op);
 };
+
+/** Register Expression */
+Expression.registerExpression(klass.opName, base.parse);

lib/pipeline/expressions/ObjectExpression.js

@@ -1,20 +1,29 @@
 "use strict";
 
 /**
- * Create an empty expression.  Until fields are added, this will evaluate to an empty document (object).
+ * Create an empty expression.  Until fields are added, this will evaluateInternal to an empty document (object).
  * @class ObjectExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @extends mungedb-aggregate.pipeline.expressions.Expression
  * @constructor
  **/
-var ObjectExpression = module.exports = function ObjectExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var ObjectExpression = module.exports = function ObjectExpression(atRoot){
+	if (arguments.length !== 1) throw new Error("one arg expected");
 	this.excludeId = false;	/// <Boolean> for if _id is to be excluded
+	this.atRoot = atRoot;
 	this._expressions = {};	/// <Object<Expression>> mapping from fieldname to Expression to generate the value NULL expression means include from source document
 	this._order = []; /// <Array<String>> this is used to maintain order for generated fields not in the source document
 }, klass = ObjectExpression, Expression = require("./Expression"), base = Expression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
+klass.create = function create() {
+	return new ObjectExpression(false);
+};
+
+klass.createRoot = function createRoot() {
+	return new ObjectExpression(true);
+};
+
 // DEPENDENCIES
 var Document = require("../Document"),
 	FieldPath = require("../FieldPath");
@@ -41,20 +50,20 @@ proto._order = [];
 // PROTOTYPE MEMBERS
 
 /**
- * evaluate(), but return a Document instead of a Value-wrapped Document.
+ * evaluateInternal(), but return a Document instead of a Value-wrapped Document.
  * @method evaluateDocument
- * @param pDocument the input Document
+ * @param currentDoc the input Document
  * @returns the result document
  **/
-proto.evaluateDocument = function evaluateDocument(doc) {
+proto.evaluateDocument = function evaluateDocument(vars) {
 	// create and populate the result
-	var pResult = {};
-	this.addToDocument(pResult, pResult, doc); // No inclusion field matching.
+	var pResult = {_id:0};
+	this.addToDocument(pResult, pResult, vars); // No inclusion field matching.
 	return pResult;
 };
 
-proto.evaluate = function evaluate(doc) { //TODO: collapse with #evaluateDocument()?
-	return this.evaluateDocument(doc);
+proto.evaluateInternal = function evaluateInternal(vars) { //TODO: collapse with #evaluateDocument()?
+	return this.evaluateDocument(vars);
 };
 
 proto.optimize = function optimize(){
@@ -65,21 +74,22 @@ proto.optimize = function optimize(){
 	return this;
 };
 
-proto.getIsSimple = function getIsSimple(){
+proto.isSimple = function isSimple(){
 	for (var key in this._expressions) {
 		var expr = this._expressions[key];
-		if (expr !== undefined && expr !== null && !expr.getIsSimple()) return false;
+		if (expr !== undefined && expr !== null && !expr.isSimple()) return false;
 	}
 	return true;
 };
 
 proto.addDependencies = function addDependencies(deps, path){
-	var depsSet = {};
 	var pathStr = "";
 	if (path instanceof Array) {
 		if (path.length === 0) {
 			// we are in the top level of a projection so _id is implicit
-			if (!this.excludeId) depsSet[Document.ID_PROPERTY_NAME] = 1;
+			if (!this.excludeId) {
+							deps[Document.ID_PROPERTY_NAME] = 1;
+						}
 		} else {
 			pathStr = new FieldPath(path).getPath() + ".";
 		}
@@ -94,38 +104,35 @@ proto.addDependencies = function addDependencies(deps, path){
 			if (path instanceof Array) path.pop();
 		} else { // inclusion
 			if (path === undefined || path === null) throw new Error("inclusion not supported in objects nested in $expressions; uassert code 16407");
-			depsSet[pathStr + key] = 1;
+			deps[pathStr + key] = 1;
 		}
 	}
-	//Array.prototype.push.apply(deps, Object.getOwnPropertyNames(depsSet));
-	for(key in depsSet) {
-		deps[key] = 1;
-	}
+
 	return deps;	// NOTE: added to munge as a convenience
 };
 
 /**
- * evaluate(), but add the evaluated fields to a given document instead of creating a new one.
+ * evaluateInternal(), but add the evaluated fields to a given document instead of creating a new one.
  * @method addToDocument
  * @param pResult the Document to add the evaluated expressions to
- * @param pDocument the input Document for this level
- * @param rootDoc the root of the whole input document
+ * @param currentDoc the input Document for this level
+ * @param vars the root of the whole input document
  **/
-proto.addToDocument = function addToDocument(pResult, pDocument, rootDoc){
-	var atRoot = (pDocument === rootDoc);
+proto.addToDocument = function addToDocument(out, currentDoc, vars){
+
 
 	var doneFields = {};	// This is used to mark fields we've done so that we can add the ones we haven't
 
-	for(var fieldName in pDocument){
-		if (!pDocument.hasOwnProperty(fieldName)) continue;
-		var fieldValue = pDocument[fieldName];
+	for(var fieldName in currentDoc){
+		if (!currentDoc.hasOwnProperty(fieldName)) continue;
+		var fieldValue = currentDoc[fieldName];
 
 		// This field is not supposed to be in the output (unless it is _id)
 		if (!this._expressions.hasOwnProperty(fieldName)) {
-			if (!this.excludeId && atRoot && fieldName == Document.ID_PROPERTY_NAME) {
+			if (!this.excludeId && this.atRoot && fieldName == Document.ID_PROPERTY_NAME) {
 				// _id from the root doc is always included (until exclusion is supported)
 				// not updating doneFields since "_id" isn't in _expressions
-				pResult[fieldName] = fieldValue;
+				out[fieldName] = fieldValue;
 			}
 			continue;
 		}
@@ -136,41 +143,41 @@ proto.addToDocument = function addToDocument(pResult, pDocument, rootDoc){
 		// This means pull the matching field from the input document
 		var expr = this._expressions[fieldName];
 		if (!(expr instanceof Expression)) {
-			pResult[fieldName] = fieldValue;
+			out[fieldName] = fieldValue;
 			continue;
 		}
 
 		// Check if this expression replaces the whole field
 		if (!(fieldValue instanceof Object) || (fieldValue.constructor !== Object && fieldValue.constructor !== Array) || !(expr instanceof ObjectExpression)) {
-			var pValue = expr.evaluate(rootDoc);
+			var pValue = expr.evaluateInternal(vars);
 
 			// don't add field if nothing was found in the subobject
 			if (expr instanceof ObjectExpression && pValue instanceof Object && Object.getOwnPropertyNames(pValue).length === 0) continue;
 
 			// Don't add non-existent values (note:  different from NULL); this is consistent with existing selection syntax which doesn't force the appearnance of non-existent fields.
 			// TODO make missing distinct from Undefined
-			if (pValue !== undefined) pResult[fieldName] = pValue;
+			if (pValue !== undefined) out[fieldName] = pValue;
 			continue;
 		}
 
 		// Check on the type of the input value.  If it's an object, just walk down into that recursively, and add it to the result.
 		if (fieldValue instanceof Object && fieldValue.constructor === Object) {
-			pResult[fieldName] = expr.addToDocument({}, fieldValue, rootDoc);	//TODO: pretty sure this is broken;
+			out[fieldName] = expr.addToDocument({}, fieldValue, vars);	//TODO: pretty sure this is broken;
 		} else if (fieldValue instanceof Object && fieldValue.constructor === Array) {
 			// If it's an array, we have to do the same thing, but to each array element.  Then, add the array of results to the current document.
 			var result = [];
 			for(var fvi = 0, fvl = fieldValue.length; fvi < fvl; fvi++){
 				var subValue = fieldValue[fvi];
 				if (subValue.constructor !== Object) continue;	// can't look for a subfield in a non-object value.
-				result.push(expr.addToDocument({}, subValue, rootDoc));
+				result.push(expr.addToDocument({}, subValue, vars));
 			}
-			pResult[fieldName] = result;
+			out[fieldName] = result;
 		} else {
 			throw new Error("should never happen");	//verify( false );
 		}
 	}
 
-	if (Object.getOwnPropertyNames(doneFields).length == Object.getOwnPropertyNames(this._expressions).length) return pResult;	//NOTE: munge returns result as a convenience
+	if (Object.getOwnPropertyNames(doneFields).length == Object.getOwnPropertyNames(this._expressions).length) return out;	//NOTE: munge returns result as a convenience
 
 	// add any remaining fields we haven't already taken care of
 	for(var i = 0, l = this._order.length; i < l; i++){
@@ -183,18 +190,18 @@ proto.addToDocument = function addToDocument(pResult, pDocument, rootDoc){
 		// this is a missing inclusion field
 		if (!expr2) continue;
 
-		var value = expr2.evaluate(rootDoc);
+		var value = expr2.evaluateInternal(vars);
 
 		// Don't add non-existent values (note:  different from NULL); this is consistent with existing selection syntax which doesn't force the appearnance of non-existent fields.
-		if (value === undefined) continue;
+		if (value === undefined || (typeof(value) == 'object' && value !== null && Object.keys(value).length === 0)) continue;
 
 		// don't add field if nothing was found in the subobject
-		if (expr2 instanceof ObjectExpression && value && value instanceof Object && Object.getOwnPropertyNames(value).length === 0) continue;
+		if (expr2 instanceof ObjectExpression && value && value instanceof Object && Object.getOwnPropertyNames(value) == {} ) continue;
 
-		pResult[fieldName2] = value;
+		out[fieldName2] = value;
 	}
 
-	return pResult;	//NOTE: munge returns result as a convenience
+	return out;	//NOTE: munge returns result as a convenience
 };
 
 /**
@@ -206,15 +213,27 @@ proto.getSizeHint = function getSizeHint(){
 	return Object.getOwnPropertyNames(this._expressions).length + (this.excludeId ? 0 : 1);
 };
 
+
+proto.evaluateDocument = function evaluateDocument(vars) {
+	var out = {};
+	this.addToDocument(out, {}, vars);
+	return out;
+};
+
+proto.evaluateInternal = function evaluateInternal(vars) {
+	return this.evaluateDocument(vars);
+};
+
+
 /**
  * Add a field to the document expression.
  * @method addField
  * @param fieldPath the path the evaluated expression will have in the result Document
- * @param pExpression the expression to evaluate obtain this field's Value in the result Document
+ * @param pExpression the expression to evaluateInternal obtain this field's Value in the result Document
  **/
 proto.addField = function addField(fieldPath, pExpression){
 	if(!(fieldPath instanceof FieldPath)) fieldPath = new FieldPath(fieldPath);
-	var fieldPart = fieldPath.fields[0],
+	var fieldPart = fieldPath.getFieldName(0),
 		haveExpr = this._expressions.hasOwnProperty(fieldPart),
 		subObj = this._expressions[fieldPart];	// inserts if !haveExpr //NOTE: not in munge & JS it doesn't, handled manually below
 
@@ -245,7 +264,7 @@ proto.addField = function addField(fieldPath, pExpression){
 		return;
 	}
 
-	if (!haveExpr) subObj = this._expressions[fieldPart] = new ObjectExpression();
+	if (!haveExpr) subObj = this._expressions[fieldPart] = new ObjectExpression(false);
 
 	subObj.addField(fieldPath.tail(), pExpression);
 };
@@ -259,9 +278,33 @@ proto.addField = function addField(fieldPath, pExpression){
  * @param fieldPath the name of the field to be included
  **/
 proto.includePath = function includePath(path){
-	this.addField(path, undefined);
+	this.addField(path, null);
 };
 
+
+proto.serialize = function serialize(explain) {
+	var valBuilder = {};
+
+	if(this._excludeId) {
+		valBuilder._id = false;
+	}
+
+	for(var ii = 0; ii < this._order.length; ii ++) {
+		var fieldName = this._order[ii],
+			expr = this._expressions[fieldName];
+
+		if(expr === undefined || expr === null) {
+			valBuilder[fieldName] = {$const:expr};
+		} else {
+			valBuilder[fieldName] = expr.serialize(explain);
+		}
+
+	}
+	return valBuilder;
+};
+
+
+
 /**
  * Get a count of the added fields.
  * @method getFieldCount

lib/pipeline/expressions/OrExpression.js

@@ -1,35 +1,37 @@
 "use strict";
 
-/** 
- * An $or pipeline expression. 
- * @see evaluate 
+/**
+ * An $or pipeline expression.
+ * @see evaluateInternal
  * @class OrExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var OrExpression = module.exports = function OrExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+//	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = OrExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = OrExpression, base = require("./VariadicExpressionT")(OrExpression), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
 
 // DEPENDENCIES
 var Value = require("../Value"),
 	ConstantExpression = require("./ConstantExpression"),
-	CoerceToBoolExpression = require("./CoerceToBoolExpression");
+	CoerceToBoolExpression = require("./CoerceToBoolExpression"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
+klass.opName = "$or";
 proto.getOpName = function getOpName(){
-	return "$or";
+	return klass.opName;
 };
 
-/** 
- * Takes an array of one or more values and returns true if any of the values in the array are true. Otherwise $or returns false. 
- * @method evaluate
+/**
+ * Takes an array of one or more values and returns true if any of the values in the array are true. Otherwise $or returns false.
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
+proto.evaluateInternal = function evaluateInternal(vars){
 	for(var i = 0, n = this.operands.length; i < n; ++i){
-		var value = this.operands[i].evaluate(doc);
+		var value = this.operands[i].evaluateInternal(vars);
 		if (Value.coerceToBool(value)) return true;
 	}
 	return false;
@@ -50,7 +52,7 @@ proto.optimize = function optimize() {
 	if (!(pLast instanceof ConstantExpression)) return pE;
 
 	// Evaluate and coerce the last argument to a boolean.  If it's true, then we can replace this entire expression.
-	var last = Value.coerceToBool(pLast.evaluate());
+	var last = Value.coerceToBool();
 	if (last) return new ConstantExpression(true);
 
 	// If we got here, the final operand was false, so we don't need it anymore.
@@ -62,6 +64,5 @@ proto.optimize = function optimize() {
 	return pE;
 };
 
-proto.getFactory = function getFactory(){
-	return klass;	// using the ctor rather than a separate .create() method
-};
+/** Register Expression */
+Expression.registerExpression(klass.opName, base.parse);

lib/pipeline/expressions/SecondExpression.js

@@ -1,36 +1,42 @@
 "use strict";
 
-/** 
- * An $second pipeline expression. 
- * @see evaluate 
+/**
+ * An $second pipeline expression.
+ * @see evaluateInternal
  * @class SecondExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var SecondExpression = module.exports = function SecondExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var SecondExpression = module.exports = function SecondExpression() {
 	base.call(this);
-}, klass = SecondExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = SecondExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$second";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the second between 0 and 59, but can be 60 to account for leap seconds.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
-	return date.getUTCSeconds();	//TODO: incorrect for last second of leap year, need to fix...
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
+	return date.getUTCSeconds(); //TODO: incorrect for last second of leap year, need to fix...
 	// currently leap seconds are unsupported in v8
 	// http://code.google.com/p/v8/issues/detail?id=1944
 };
+
+/** Register Expression */
+Expression.registerExpression("$second", base.parse);

lib/pipeline/expressions/SetDifferenceExpression.js

@@ -0,0 +1,52 @@
+"use strict";
+
+/**
+ * A $setdifference pipeline expression.
+ * @see evaluateInternal
+ * @class SetDifferenceExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetDifferenceExpression = module.exports = function SetDifferenceExpression() {
+	base.call(this);
+}, klass = SetDifferenceExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 2),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setdifference";
+};
+
+/**
+ * Takes 2 arrays. Assigns the second array to the first array.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var array1 = this.operands[0].evaluateInternal(vars),
+		array2 = this.operands[1].evaluateInternal(vars);
+	if (array1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an array");
+	if (array2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an array");
+
+	var returnVec = [];
+
+	array1.forEach(function(key) {
+		if (-1 === array2.indexOf(key)) {
+			returnVec.push(key);
+		}
+	}, this);
+	return returnVec;
+};
+
+/** Register Expression */
+Expression.registerExpression("$setdifference", base.parse);

lib/pipeline/expressions/SetEqualsExpression.js

@@ -0,0 +1,38 @@
+"use strict";
+
+/**
+ * A $setequals pipeline expression.
+ * @see evaluateInternal
+ * @class SetEqualsExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetEqualsExpression = module.exports = function SetEqualsExpression() {
+	base.call(this);
+}, klass = SetEqualsExpression, base = require("./NaryBaseExpressionT")(SetEqualsExpression), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setequals";
+};
+
+/**
+ * Takes 2 arrays. Assigns the second array to the first array.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var array1 = this.operands[0].evaluateInternal(vars),
+		array2 = this.operands[1].evaluateInternal(vars);
+	if (array1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an array");
+	if (array2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an array");
+	array1 = array2;
+	return array1;
+};
+
+/** Register Expression */
+Expression.registerExpression("$setequals", base.parse);

lib/pipeline/expressions/SetIntersectionExpression.js

@@ -0,0 +1,40 @@
+"use strict";
+
+/**
+ * A $setintersection pipeline expression.
+ * @see evaluateInternal
+ * @class SetIntersectionExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetIntersectionExpression = module.exports = function SetIntersectionExpression() {
+	base.call(this);
+}, klass = SetIntersectionExpression, base = require("./NaryBaseExpressionT")(SetIntersectionExpression), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setIntersection";
+};
+
+/**
+ * Takes 2 objects. Returns the intersects of the objects.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var object1 = this.operands[0].evaluateInternal(vars),
+		object2 = this.operands[1].evaluateInternal(vars);
+	if (object1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an object");
+	if (object2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an object");
+
+	var result = object1.filter(function(n) {
+		return object2.indexOf(n) > -1;
+	});
+};
+
+/** Register Expression */
+Expression.registerExpression("$setIntersection", base.parse);

lib/pipeline/expressions/SetIsSubsetExpression.js

@@ -0,0 +1,88 @@
+"use strict";
+
+/**
+ * A $setissubset pipeline expression.
+ * @see evaluateInternal
+ * @class SetIsSubsetExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetIsSubsetExpression = module.exports = function SetIsSubsetExpression() {
+	if (arguments.length !== 2) throw new Error("two args expected");
+	base.call(this);
+}, klass = SetIsSubsetExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 2),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setissubset";
+};
+
+proto.optimize = function optimize(cachedRhsSet, operands) {
+
+// This optimize needs to be done, eventually
+
+// // perfore basic optimizations
+//     intrusive_ptr<Expression> optimized = ExpressionNary::optimize();
+
+//     // if ExpressionNary::optimize() created a new value, return it directly
+//     if (optimized.get() != this)
+//         return optimized;
+
+//     if (ExpressionConstant* ec = dynamic_cast<ExpressionConstant*>(vpOperand[1].get())) {
+//         const Value rhs = ec->getValue();
+//         uassert(17311, str::stream() << "both operands of $setIsSubset must be arrays. Second "
+//                                      << "argument is of type: " << typeName(rhs.getType()),
+//                 rhs.getType() == Array);
+
+//         return new Optimized(arrayToSet(rhs), vpOperand);
+//     }
+
+//     return optimized;
+
+};
+
+/**
+ * Takes 2 arrays. Assigns the second array to the first array.
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var array1 = this.operands[0].evaluateInternal(vars),
+		array2 = this.operands[1].evaluateInternal(vars);
+	if (array1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an array");
+	if (array2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an array");
+
+	var sizeOfArray1 = array1.length;
+	var sizeOfArray2 = array2.length;
+	var outerLoop = 0;
+	var innerLoop = 0;
+	for (outerLoop = 0; outerLoop < sizeOfArray1; outerLoop++) {
+		for (innerLoop = 0; innerLoop < sizeOfArray2; innerLoop++) {
+			if (array2[outerLoop] == array1[innerLoop])
+				break;
+		}
+
+		/* If the above inner loop was not broken at all then
+		 array2[i] is not present in array1[] */
+		if (innerLoop == sizeOfArray2)
+			return false;
+	}
+
+	/* If we reach here then all elements of array2[]
+	 are present in array1[] */
+	return true;
+};
+
+/** Register Expression */
+Expression.registerExpression("$setissubset", base.parse);

lib/pipeline/expressions/SetUnionExpression.js

@@ -0,0 +1,46 @@
+"use strict";
+
+/**
+ * A $setunion pipeline expression.
+ * @see evaluateInternal
+ * @class SetUnionExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SetUnionExpression = module.exports = function SetUnionExpression() {
+	base.call(this);
+}, klass = SetUnionExpression, base = require("./NaryBaseExpressionT")(SetUnionExpression), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$setUnion";
+};
+
+/**
+ * Takes 2 objects. Unions the objects
+ * @method evaluateInternal
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var object1 = this.operands[0].evaluateInternal(vars),
+		object2 = this.operands[1].evaluateInternal(vars);
+	if (object1 instanceof Array) throw new Error(this.getOpName() + ": object 1 must be an object");
+	if (object2 instanceof Array) throw new Error(this.getOpName() + ": object 2 must be an object");
+
+	var object3 = {};
+	for (var attrname1 in object1) {
+		object3[attrname1] = object1[attrname1];
+	}
+	for (var attrname2 in object2) {
+		object3[attrname2] = object2[attrname2];
+	}
+
+	return object3;
+};
+
+/** Register Expression */
+Expression.registerExpression("$setUnion", base.parse);

lib/pipeline/expressions/SizeExpression.js

@@ -0,0 +1,41 @@
+"use strict";
+
+/**
+ * A $size pipeline expression.
+ * @see evaluateInternal
+ * @class SizeExpression
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var SizeExpression = module.exports = function SizeExpression() {
+	base.call(this);
+}, klass = SizeExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
+
+// DEPENDENCIES
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return "$size";
+};
+
+/**
+ * Takes an array and return the size.
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var array = this.operands[0].evaluateInternal(vars);
+	if (array instanceof Date) throw new Error("$size does not support dates; code 16376");
+	return array.length;
+};
+
+/** Register Expression */
+Expression.registerExpression("$size", base.parse);

lib/pipeline/expressions/StrcasecmpExpression.js

@@ -1,42 +1,46 @@
 "use strict";
 
-/** 
+/**
  * A $strcasecmp pipeline expression.
- * @see evaluate 
+ * @see evaluate
  * @class StrcasecmpExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var StrcasecmpExpression = module.exports = function StrcasecmpExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var StrcasecmpExpression = module.exports = function StrcasecmpExpression() {
 	base.call(this);
-}, klass = StrcasecmpExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = StrcasecmpExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 2),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
 var Value = require("../Value"),
-	NaryExpression = require("./NaryExpression");
+	NaryExpression = require("./NaryExpression"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$strcasecmp";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Takes in two strings. Returns a number. $strcasecmp is positive if the first string is “greater than” the second and negative if the first string is “less than” the second. $strcasecmp returns 0 if the strings are identical. 
+/**
+ * Takes in two strings. Returns a number. $strcasecmp is positive if the first string is “greater than” the second and negative if the first string is “less than” the second. $strcasecmp returns 0 if the strings are identical.
  * @method evaluate
  **/
-proto.evaluate = function evaluate(doc){
-	this.checkArgCount(2);
-	var val1 = this.operands[0].evaluate(doc),
-		val2 = this.operands[1].evaluate(doc),
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var val1 = this.operands[0].evaluateInternal(vars),
+		val2 = this.operands[1].evaluateInternal(vars),
 		str1 = Value.coerceToString(val1).toUpperCase(),
 		str2 = Value.coerceToString(val2).toUpperCase(),
 		cmp = Value.compare(str1, str2);
 	return cmp;
 };
+
+/** Register Expression */
+Expression.registerExpression("$strcasecmp", base.parse);

lib/pipeline/expressions/SubstrExpression.js

@@ -2,39 +2,40 @@
 
 /**
  * A $substr pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class SubstrExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var SubstrExpression = module.exports = function SubstrExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var SubstrExpression = module.exports = function SubstrExpression() {
 	base.call(this);
-}, klass = SubstrExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = SubstrExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 3),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$substr";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(3);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a string and two numbers. The first number represents the number of bytes in the string to skip, and the second number specifies the number of bytes to return from the string.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(3);
-	var val = this.operands[0].evaluate(doc),
-		idx = this.operands[1].evaluate(doc),
-		len = this.operands[2].evaluate(doc),
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var val = this.operands[0].evaluateInternal(vars),
+		idx = this.operands[1].evaluateInternal(vars),
+		len = this.operands[2].evaluateInternal(vars),
 		str = Value.coerceToString(val);
 	if (typeof(idx) != "number") throw new Error(this.getOpName() + ": starting index must be a numeric type; code 16034");
 	if (typeof(len) != "number") throw new Error(this.getOpName() + ": length must be a numeric type; code 16035");
@@ -43,3 +44,6 @@ proto.evaluate = function evaluate(doc) {
 	len = (len === -1 ? undefined : len);
 	return str.substr(idx, len);
 };
+
+/** Register Expression */
+Expression.registerExpression("$substr", base.parse);

lib/pipeline/expressions/SubtractExpression.js

@@ -1,37 +1,42 @@
 "use strict";
 
-/** 
+/**
  * A $subtract pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class SubtractExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var SubtractExpression = module.exports = function SubtractExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = SubtractExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = SubtractExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 2),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
 	return "$subtract";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(2);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
-* Takes an array that contains a pair of numbers and subtracts the second from the first, returning their difference. 
+/**
+* Takes an array that contains a pair of numbers and subtracts the second from the first, returning their difference.
 **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(2);
-	var left = this.operands[0].evaluate(doc),
-		right = this.operands[1].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var left = this.operands[0].evaluateInternal(vars),
+		right = this.operands[1].evaluateInternal(vars);
+	if (left instanceof Date || right instanceof Date) throw new Error("$subtract does not support dates; code 16376");
 	return left - right;
 };
+
+/** Register Expression */
+Expression.registerExpression("$subtract", base.parse);

lib/pipeline/expressions/ToLowerExpression.js

@@ -1,37 +1,36 @@
 "use strict";
-	
-/** 
+
+/**
  * A $toLower pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class ToLowerExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
 var ToLowerExpression = module.exports = function ToLowerExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
 	base.call(this);
-}, klass = ToLowerExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = ToLowerExpression, base = require("./FixedArityExpressionT")(ToLowerExpression, 1), proto = klass.prototype = Object.create(base.prototype, {constructor: {value: klass}});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
+
+klass.opName = "$toLower";
 
 // PROTOTYPE MEMBERS
 proto.getOpName = function getOpName(){
-	return "$toLower";
+	return klass.opName;
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
-* Takes a single string and converts that string to lowercase, returning the result. All uppercase letters become lowercase. 
+/**
+* Takes a single string and converts that string to lowercase, returning the result. All uppercase letters become lowercase.
 **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(1);
-	var val = this.operands[0].evaluate(doc),
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var val = this.operands[0].evaluateInternal(vars),
 		str = Value.coerceToString(val);
 	return str.toLowerCase();
 };
+
+/** Register Expression */
+Expression.registerExpression(klass.opName, base.parse);

lib/pipeline/expressions/ToUpperExpression.js

@@ -1,37 +1,36 @@
 "use strict";
 
-/** 
+/**
  * A $toUpper pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class ToUpperExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var ToUpperExpression = module.exports = function ToUpperExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var ToUpperExpression = module.exports = function ToUpperExpression() {
 	base.call(this);
-}, klass = ToUpperExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = ToUpperExpression, base = require("./FixedArityExpressionT")(ToUpperExpression, 1), proto = klass.prototype = Object.create(base.prototype, {constructor: {value: klass }});
 
 // DEPENDENCIES
-var Value = require("../Value");
+var Value = require("../Value"),
+	Expression = require("./Expression");
 
-// PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
-	return "$toUpper";
-};
+klass.opName = "$toUpper";
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
+// PROTOTYPE MEMBERS
+proto.getOpName = function getOpName() {
+	return klass.opName;
 };
 
-/** 
-* Takes a single string and converts that string to lowercase, returning the result. All uppercase letters become lowercase. 
-**/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(1);
-	var val = this.operands[0].evaluate(doc),
+/**
+ * Takes a single string and converts that string to lowercase, returning the result. All uppercase letters become lowercase.
+ **/
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var val = this.operands[0].evaluateInternal(vars),
 		str = Value.coerceToString(val);
 	return str.toUpperCase();
 };
+
+/** Register Expression */
+Expression.registerExpression(klass.opName, base.parse);

lib/pipeline/expressions/Variables.js

@@ -0,0 +1,126 @@
+"use strict";
+
+/**
+ * Class that stores/tracks variables
+ * @class Variables
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var Variables = module.exports = function Variables(numVars, root){
+	if(numVars) {
+		if(typeof numVars !== 'number') {
+			throw new Error('numVars must be a number');
+		}
+	}
+	this._root = root || {};
+	this._rest = numVars ? [] : undefined; //An array of `Value`s
+	this._numVars = numVars;
+}, klass = Variables,
+	base = Object,
+	proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+klass.ROOT_ID = -1;
+
+// PROTOTYPE MEMBERS
+
+/**
+ * Sets the root variable
+ * @method setRoot
+ * @parameter root {Document} The root variable
+ **/
+proto.setRoot = function setRoot(root){
+	if(!(root instanceof Object && root.constructor.name === 'Object')) { //NOTE: Type checking cause c++ does this for you
+		throw new Error('root must be an Object');
+	}
+	this._root = root;
+};
+
+/**
+ * Clears the root variable
+ * @method clearRoot
+ **/
+proto.clearRoot = function clearRoot(){
+	this._root = {};
+};
+
+/**
+ * Gets the root variable
+ * @method getRoot
+ * @return {Document} the root variable
+ **/
+proto.getRoot = function getRoot(){
+	return this._root;
+};
+
+/**
+ * Inserts a value with the given id
+ * @method setValue
+ * @param id {Number} The index where the value is stored in the _rest Array
+ * @param value {Value} The value to store
+ **/
+proto.setValue = function setValue(id, value) {
+	//NOTE: Some added type enforcement cause c++ does this for you
+	if(typeof id !== 'number') {
+		throw new Error('id must be a Number');
+	}
+
+	if(id === klass.ROOT_ID) {
+		throw new Error("mError 17199: can't use Variables#setValue to set ROOT");
+	}
+	if(id >= this._numVars) { // a > comparator would be off-by-one; i.e. if we have 5 vars, the max id would be 4
+		throw new Error("You have more variables than _numVars");
+	}
+
+	this._rest[id] = value;
+};
+
+/**
+ * Get the value at the given id
+ * @method getValue
+ * @param id {Number} The index where the value was stored
+ * @return {Value} The value
+ **/
+proto.getValue = function getValue(id) {
+	//NOTE: Some added type enforcement cause c++ does this for you
+	if(typeof id !== 'number') {
+		throw new Error('id must be a Number');
+	}
+
+	if(id === klass.ROOT_ID) {
+		return this._root;
+	}
+	if(id >= this._numVars) { // a > comparator would be off-by-one; i.e. if we have 5 vars, the max id would be 4
+		throw new Error("Cannot get value; id was greater than _numVars");
+	}
+
+	return this._rest[id];
+};
+
+
+/**
+ * Get the value for id if it's a document
+ * @method getDocument
+ * @param id {Number} The index where the document was stored
+ * @return {Object} The document
+ **/
+proto.getDocument = function getDocument(id) {
+	//NOTE: Some added type enforcement cause c++ does this for you
+	if(typeof id !== 'number') {
+		throw new Error('id must be a Number');
+	}
+
+	if(id === klass.ROOT_ID) {
+		return this._root;
+	}
+	if(id >= this._numVars) { // a > comparator would be off-by-one; i.e. if we have 5 vars, the max id would be 4
+		throw new Error("Cannot get value; id was greater than _numVars");
+	}
+
+	var value = this._rest[id];
+	if(typeof value === 'object' && value.constructor.name === 'Object') {
+		return value;
+	}
+	return {};
+};

lib/pipeline/expressions/VariablesIdGenerator.js

@@ -0,0 +1,31 @@
+"use strict";
+
+/** 
+ * Class generates unused ids
+ * @class VariablesIdGenerator
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var VariablesIdGenerator = module.exports = function VariablesIdGenerator(){
+	this._nextId = 0;
+}, klass = VariablesIdGenerator, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ * Gets the next unused id
+ * @method generateId
+ * @return {Number} The unused id
+ **/
+proto.generateId = function generateId() {
+	return this._nextId++;
+};
+
+/**
+ * Gets the number of used ids
+ * @method getIdCount
+ * @return {Number} The number of used ids
+ **/
+proto.getIdCount = function getIdCount() {
+	return this._nextId;
+};
+

lib/pipeline/expressions/VariablesParseState.js

@@ -0,0 +1,55 @@
+"use strict";
+
+/** 
+ * Class generates unused ids
+ * @class VariablesParseState
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+var Variables = require('./Variables'),
+	VariablesIdGenerator = require('./VariablesIdGenerator');
+
+var VariablesParseState = module.exports = function VariablesParseState(idGenerator){
+	if(!idGenerator || idGenerator.constructor !== VariablesIdGenerator) {
+		throw new Error("idGenerator is required and must be of type VariablesIdGenerator");
+	}
+	this._idGenerator = idGenerator;
+	this._variables = {}; //Note: The c++ type was StringMap<Variables::Id>
+}, klass = VariablesParseState, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ * Assigns a named variable a unique Id. This differs from all other variables, even
+ * others with the same name.
+ *
+ * The special variables ROOT and CURRENT are always implicitly defined with CURRENT
+ * equivalent to ROOT. If CURRENT is explicitly defined by a call to this function, it
+ * breaks that equivalence.
+ *
+ * NOTE: Name validation is responsibility of caller.
+ **/
+proto.defineVariable = function generateId(name) {
+	// caller should have validated before hand by using Variables::uassertValidNameForUserWrite
+	if(name === 'ROOT') {
+		throw new Error("mError 17275: Can't redefine ROOT");
+	}
+	var id = this._idGenerator.generateId();
+	this._variables[name] = id;
+	return id;
+};
+
+/**
+ * Returns the current Id for a variable. uasserts if the variable isn't defined.
+ * @method getVariable
+ * @param name {String} The name of the variable
+ **/
+proto.getVariable = function getIdCount(name) {
+	var found = this._variables[name];
+	if(typeof found === 'number') return found;
+	if(name !== "ROOT" && name !== "CURRENT") {
+		throw new Error("uError 17276: Use of undefined variable " + name);
+	}
+
+	return Variables.ROOT_ID;
+};
+

lib/pipeline/expressions/VariadicExpressionT.js

@@ -0,0 +1,22 @@
+"use strict";
+
+/**
+ * A factory and base class for all expressions that are variadic (AKA they accept any number of arguments)
+ * @class VariadicExpressionT
+ * @namespace mungedb-aggregate.pipeline.expressions
+ * @module mungedb-aggregate
+ * @constructor
+ **/
+
+var VariadicExpressionT = module.exports = function VariadicExpressionT(SubClass) {
+
+	var VariadicExpression = function VariadicExpression() {
+		if (arguments.length !== 0) throw new Error(klass.name + "<" + SubClass.name + ">: zero args expected");
+		base.call(this);
+	}, klass = VariadicExpression, base = require("./NaryBaseExpressionT")(SubClass), proto = klass.prototype = Object.create(base.prototype, {constructor: {value: klass}});
+
+	klass.parse = base.parse; 						// NOTE: Need to explicitly
+	klass.parseArguments = base.parseArguments;		// bubble static members in
+													// our inheritance chain
+	return VariadicExpression;
+};

lib/pipeline/expressions/WeekExpression.js

@@ -1,46 +1,50 @@
 "use strict";
 
-/** 
+/**
  * A $week pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class WeekExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var WeekExpression = module.exports = function WeekExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var WeekExpression = module.exports = function WeekExpression() {
 	base.call(this);
-}, klass = WeekExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = WeekExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
 var Value = require("../Value"),
-	DayOfYearExpression = require("./DayOfYearExpression");
+	DayOfYearExpression = require("./DayOfYearExpression"),
+	Expression = require("./Expression");
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$week";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
-/** 
- * Takes a date and returns the week of the year as a number between 0 and 53. 
- * Weeks begin on Sundays, and week 1 begins with the first Sunday of the year. 
- * Days preceding the first Sunday of the year are in week 0. 
+/**
+ * Takes a date and returns the week of the year as a number between 0 and 53.
+ * Weeks begin on Sundays, and week 1 begins with the first Sunday of the year.
+ * Days preceding the first Sunday of the year are in week 0.
  * This behavior is the same as the “%U” operator to the strftime standard library function.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc),
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars),
 		dayOfWeek = date.getUTCDay(),
 		dayOfYear = DayOfYearExpression.getDateDayOfYear(date),
-		prevSundayDayOfYear = dayOfYear - dayOfWeek,	// may be negative
-		nextSundayDayOfYear = prevSundayDayOfYear + 7;	// must be positive
+		prevSundayDayOfYear = dayOfYear - dayOfWeek, // may be negative
+		nextSundayDayOfYear = prevSundayDayOfYear + 7; // must be positive
 	// Return the zero based index of the week of the next sunday, equal to the one based index of the week of the previous sunday, which is to be returned.
 	return (nextSundayDayOfYear / 7) | 0; // also, the `| 0` here truncates this so that we return an integer
 };
+
+/** Register Expression */
+Expression.registerExpression("$week", base.parse);

lib/pipeline/expressions/YearExpression.js

@@ -1,38 +1,43 @@
 "use strict";
 
-/** 
+/**
  * A $year pipeline expression.
- * @see evaluate 
+ * @see evaluateInternal
  * @class YearExpression
  * @namespace mungedb-aggregate.pipeline.expressions
  * @module mungedb-aggregate
  * @constructor
  **/
-var YearExpression = module.exports = function YearExpression(){
-	if (arguments.length !== 0) throw new Error("zero args expected");
+var YearExpression = module.exports = function YearExpression() {
 	base.call(this);
-}, klass = YearExpression, base = require("./NaryExpression"), proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+}, klass = YearExpression,
+	FixedArityExpression = require("./FixedArityExpressionT")(klass, 1),
+	base = FixedArityExpression,
+	proto = klass.prototype = Object.create(base.prototype, {
+		constructor: {
+			value: klass
+		}
+	});
 
 // DEPENDENCIES
 var Value = require("../Value"),
-	DayOfYearExpression = require("./DayOfYearExpression");
+	DayOfYearExpression = require("./DayOfYearExpression"),
+	Expression = require("./Expression");
+
 
 // PROTOTYPE MEMBERS
-proto.getOpName = function getOpName(){
+proto.getOpName = function getOpName() {
 	return "$year";
 };
 
-proto.addOperand = function addOperand(expr) {
-	this.checkArgLimit(1);
-	base.prototype.addOperand.call(this, expr);
-};
-
 /**
  * Takes a date and returns the full year.
- * @method evaluate
+ * @method evaluateInternal
  **/
-proto.evaluate = function evaluate(doc) {
-	this.checkArgCount(1);
-	var date = this.operands[0].evaluate(doc);
+proto.evaluateInternal = function evaluateInternal(vars) {
+	var date = this.operands[0].evaluateInternal(vars);
 	return date.getUTCFullYear();
 };
+
+/** Register Expression */
+Expression.registerExpression("$year", base.parse);

lib/pipeline/expressions/index.js

@@ -19,7 +19,7 @@ module.exports = {
 	ModExpression: require("./ModExpression.js"),
 	MonthExpression: require("./MonthExpression.js"),
 	MultiplyExpression: require("./MultiplyExpression.js"),
-	NaryExpression: require("./NaryExpression.js"),
+	NaryBaseExpressionT: require("./NaryBaseExpressionT.js"),
 	NotExpression: require("./NotExpression.js"),
 	ObjectExpression: require("./ObjectExpression.js"),
 	OrExpression: require("./OrExpression.js"),

lib/pipeline/matcher/AllElemMatchOp.js

@@ -0,0 +1,229 @@
+"use strict";
+
+var MatchExpression = require('./MatchExpression');
+
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var AllElemMatchOp = module.exports = function AllElemMatchOp(){
+	base.call(this);
+	this._matchType = 'ALL';
+	this._elementPath = new ElementPath();
+	this._list = [];
+}, klass = AllElemMatchOp, base =  MatchExpression , proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes,
+	ElementPath = require('./ElementPath.js');
+
+// File: expression_array.h lines: 175-175
+//         ElementPath _elementPath;
+
+proto._elementPath = undefined;
+
+
+// File: expression_array.h lines: 176-176
+//         std::vector< const ArrayMatchingMatchExpression* > _list;
+
+proto._list = undefined;
+
+
+// File: expression_array.h lines: 174-174
+//         StringData _path;
+
+proto._path = undefined;
+
+/**
+ *
+ * This method checks the input array and determines if each item inside matches
+ * @method _allMatch
+ * @param anArray
+ *
+ */
+proto._allMatch = function _allMatch( anArray ){ //  const BSONObj& anArray
+	// File: expression_array.cpp lines: 208-215
+	if(this._list.length === 0) { return false; }
+
+	for (var i = 0; i < this._list.length; i++) {
+		if( ! this._list[i].matchesArray( anArray, null ) ) { return false; }
+	}
+
+	return true;
+};
+
+
+/**
+ *
+ * This method adds a new expression to the internal array of expression
+ * @method add
+ * @param expr
+ *
+ */
+proto.add = function add( expr ){//  const ArrayMatchingMatchExpression* expr
+	// File: expression_array.cpp lines: 184-186
+	if (!expr) throw new Error("AllElemMatchOp:add#68 failed to verify expr");
+	this._list.push(expr);
+};
+
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString( level ){ //   StringBuilder& debug, int level
+	// File: expression_array.cpp lines: 219-224
+	console.debug(this._debugAddSpace(level) + this._path + " AllElemMatchOp: " + this._path + '\n');
+	for (var i = 0; i < this._list.length; i++) {
+		this._list[i].debugString(level +1);
+	}
+};
+
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent( other ){//  const MatchExpression* other
+// File: expression_array.cpp lines: 227-242
+	if (this.matchType() != other.matchType()) {
+		return false;
+	}
+
+	if( this._path != other._path ) {
+		return false;
+	}
+
+	if( this._list.length != other._list.length ) {
+		return false;
+	}
+	for (var i = 0; i < this._list.length; i++) {
+		if ( !this._list[i].equivalent( other._list[i] ) ) {
+			return false;
+		}
+	}
+	return true;
+};
+
+
+
+/**
+ *
+ * gets the specified item from the list
+ * @method getChild
+ * @param i
+ *
+ */
+proto.getChild = function getChild( i ){ //  size_t i
+// File: expression_array.h lines: 167-166
+	return this._list[i];
+};
+
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ *
+ */
+proto.init = function init( path ){ //  const StringData& path
+// File: expression_array.cpp lines: 177-181
+	this._path = path;
+	var s = this._elementPath.init( this._path );
+	this._elementPath.setTraverseLeafArray( false );
+	return s;
+};
+
+/**
+ *
+ * matches checks the input doc against the internal path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc, details){
+	// File: expression_array.cpp lines: 189-198
+	var self = this,
+		checker = function(element) {
+			if (!(element instanceof Array))
+				return false;
+
+			//var amIRoot = (element.length === 0);
+
+			if (self._allMatch(element))
+				return true;
+
+			/*
+			if (!amIRoot && details && details.needRecord() {
+				details.setElemMatchKey(element);
+			}
+			*/
+			return false;
+		};
+	return this._elementPath._matches(doc, details, checker);
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement( e ){ //  const BSONElement& e
+	// File: expression_array.cpp lines: 201-205
+	if (!(e instanceof Array)) {
+		return false;
+	}
+	return this._allMatch(e);
+};
+
+
+/**
+ *
+ * return the length of the internal array
+ * @method numChildren
+ * @param
+ *
+ */
+proto.numChildren = function numChildren( /*  */ ){
+// File: expression_array.h lines: 166-165
+	return this._list.length;
+};
+
+
+/**
+ *
+ * return the internal path
+ * @method path
+ * @param
+ *
+ */
+proto.path = function path( /*  */ ){
+// File: expression_array.h lines: 169-168
+	return this._path;
+};
+
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /*  */ ){
+// File: expression_array.h lines: 145-152
+	var e = new AllElemMatchOp();
+	e.init( this._path );
+	e._list = this._list.slice(0);
+	return e;
+};
+

lib/pipeline/matcher/AndMatchExpression.js

@@ -0,0 +1,82 @@
+"use strict";
+
+var ListOfMatchExpression = require('./ListOfMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var AndMatchExpression = module.exports = function AndMatchExpression(){
+	base.call(this);
+	this._expressions = [];
+	this._matchType = 'AND';
+}, klass = AndMatchExpression, base =  ListOfMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString( level ) { //  StringBuilder& debug, int level
+// File: expression_tree.cpp lines: 85-88
+	return this._debugAddSpace(level) + "$and\n" + this._debugList(level);
+};
+
+/**
+ *
+ * matches checks the input doc against the internal path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc, details) { //  const MatchableDocument* doc, MatchDetails* details
+	// File: expression_tree.cpp lines: 64-72
+	var tChild;
+	for (var i = 0; i < this.numChildren(); i++) {
+		tChild = this.getChild(i);
+		if (!tChild.matches(doc, details)) {
+			if (details) {
+				details.resetOutput();
+			}
+			return false;
+		}
+	}
+	return true;
+};
+
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement( e ){ //  const BSONElement& e
+	// File: expression_tree.cpp lines: 75-81
+	for (var i = 0; i < this.numChildren(); i++) {
+		if (!this.getChild(i).matchesSingleElement(e)) {
+			return false;
+		}
+	}
+	return true;
+};
+
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /*  */ ){
+// File: expression_tree.h lines: 67-72
+	var e = new AndMatchExpression();
+	for (var i = 0; i < this.numChildren(); i++) {
+		e.add(this.getChild(i).shallowClone());
+	}
+};
+

lib/pipeline/matcher/ArrayFilterEntries.js

@@ -0,0 +1,203 @@
+"use strict";
+var Value = require('../Value');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ArrayFilterEntries = module.exports = function ArrayFilterEntries(){
+	this._hasNull = false;
+	this._hasEmptyArray = false;
+	this._equalities = [];
+	this._regexes = [];
+}, klass = ArrayFilterEntries, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+// File: expression_leaf.h lines: 266-266
+proto._equalities = undefined;
+
+
+// File: expression_leaf.h lines: 265-265
+proto._hasEmptyArray = undefined;
+
+
+// File: expression_leaf.h lines: 264-264
+proto._hasNull = undefined;
+
+
+// File: expression_leaf.h lines: 267-267
+proto._regexes = undefined;
+
+
+/**
+ *
+ * Push the input expression onto the _equalities array
+ * @method addEquality
+ * @param e
+ *
+ */
+proto.addEquality = function addEquality(e) {
+	//File expression_leaf.cpp lines 369-387
+
+	if(e instanceof Object && Object.keys(e)[0][0] === '$'){
+		return {'code':'BAD_VALUE', 'desc':'cannot next $ under $in'};
+	}
+
+	if( e instanceof RegExp ) {
+		return {'code': 'BAD_VALUE', 'desc':'ArrayFilterEntries equality cannot be a regex'};
+	}
+
+	if( e === null ) {
+		this._hasNull = true;
+	}
+
+	if (e instanceof Array && e.length === 0) {
+		this._hasEmptyArray = true;
+	}
+
+	this._equalities.push( e );
+	return {'code':'OK'};
+};
+
+/**
+ *
+ * Push the input regex onto the _regexes array
+ * @method addRegex
+ * @param expr
+ *
+ */
+proto.addRegex = function addRegex(expr) {
+	// File: expression_leaf.cpp lines: 389-391
+	this._regexes.push( expr );
+	return {'code':'OK'};
+};
+
+/**
+ *
+ * Check if the input element is contained inside of _equalities
+ * @method contains
+ * @param elem
+ *
+ */
+proto.contains = function contains(elem) {
+	// File: expression_leaf.h lines: 249-248
+	for (var i = 0; i < this._equalities.length; i++) {
+		if(typeof(elem) == typeof(this._equalities[i])){
+			if(Value.compare(elem, this._equalities[i]) === 0) {
+				return true;
+			}
+		}
+	}
+	return false;
+};
+
+/**
+ *
+ * Copy our internal fields to the input
+ * @method copyTo
+ * @param toFillIn
+ *
+ */
+proto.copyTo = function copyTo(toFillIn) {
+	// File: expression_leaf.cpp lines: 407-412
+	toFillIn._hasNull = this._hasNull;
+	toFillIn._hasEmptyArray = this._hasEmptyArray;
+	toFillIn._equalities = this._equalities.slice(0); // Copy array
+	toFillIn._regexes = this._regexes.slice(0); // Copy array
+};
+
+/**
+ *
+ * Return the _equalities property
+ * @method equalities
+ *
+ */
+proto.equalities = function equalities(){
+	// File: expression_leaf.h lines: 248-247
+	return this._equalities;
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression_leaf.cpp lines: 394-404
+	if (this._hasNull != other._hasNull) {return false;}
+	if (this._regexes.length != other._regexes.length) {return false;}
+
+	for (var i = 0; i < this._regexes.length; i++) {
+		if ( !this._regexes[i].equivalent( other._regexes[i] ) ) {
+			return false;
+		}
+	}
+	return Value.compare(this._equalities, other._equalities);
+};
+
+/**
+ *
+ * Return the _hasEmptyArray property
+ * @method hasEmptyArray
+ *
+ */
+proto.hasEmptyArray = function hasEmptyArray(){
+	// File: expression_leaf.h lines: 256-255
+	return this._hasEmptyArray;
+};
+
+/**
+ *
+ * Return the _hasNull property
+ * @method hasNull
+ *
+ */
+proto.hasNull = function hasNull(){
+	// File: expression_leaf.h lines: 254-253
+	return this._hasNull;
+};
+
+/**
+ *
+ * Return the length of the _regexes property
+ * @method numRegexes
+ *
+ */
+proto.numRegexes = function numRegexes(){
+	// File: expression_leaf.h lines: 251-250
+	return this._regexes.length;
+};
+
+/**
+ *
+ * Return the regex at the given index
+ * @method regex
+ * @param idx
+ *
+ */
+proto.regex = function regex(idx) {
+	// File: expression_leaf.h lines: 252-251
+	return this._regexes[idx];
+};
+
+/**
+ *
+ * Return whether we have a single item and it is null
+ * @method singleNull
+ *
+ */
+proto.singleNull = function singleNull(){
+	// File: expression_leaf.h lines: 255-254
+	return this.size() == 1 && this._hasNull;
+};
+
+/**
+ *
+ * Return the length of both _regexes and _equalities
+ * @method size
+ *
+ */
+proto.size = function size(){
+	// File: expression_leaf.h lines: 257-256
+	return this._equalities.length + this._regexes.length;
+};
+

lib/pipeline/matcher/ArrayMatchingMatchExpression.js

@@ -0,0 +1,130 @@
+"use strict";
+
+var MatchExpression = require('./MatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ArrayMatchingMatchExpression = module.exports = function ArrayMatchingMatchExpression(matchType){
+	base.call(this);
+	this._matchType = matchType;
+	// File: expression_array.h lines: 55-55
+	this._elementPath = new ElementPath();
+}, klass = ArrayMatchingMatchExpression, base = MatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes,
+	ElementPath = require('./ElementPath.js');
+
+// File: expression_array.h lines: 54-54
+proto._path = undefined;
+
+/**
+ *
+ * Check if the input element is equivalent to us
+ * @method equivalent
+ * @param
+ *
+ */
+proto.equivalent = function equivalent(other){
+	// File: expression_array.cpp lines: 63-79
+	if ( this._matchType != other._matchType)
+		return false;
+
+	var realOther = new ArrayMatchingMatchExpression(other);
+
+	if (this._path != realOther._path)
+		return false;
+
+	if (this.numChildren() != realOther.numChildren())
+		return false;
+
+	for (var i = 0; i < this.numChildren(); i++)
+		if (!this.getChild(i).equivalent(realOther.getChild(i)))
+			return false;
+	return true;
+};
+
+/**
+ *
+ * Initialize the input path as our element path
+ * @method initPath
+ * @param path
+ *
+ */
+proto.initPath = function initPath(path){
+	// File: expression_array.cpp lines: 27-31
+	this._path = path;
+	var status = this._elementPath.init(this._path);
+	this._elementPath.setTraverseLeafArray(false);
+	return status;
+};
+
+
+
+/**
+ *
+ * matches checks the input doc against the internal path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc, details){
+	var self = this,
+		checker = function(element) {
+			// we got the whole path, now check it
+			if (!(element instanceof Array))
+				return false;
+
+			//var amIRoot = (element.length === 0);
+
+			if (!self.matchesArray(element, details))
+				return false;
+
+			/*
+			if (!amIRoot && details && details.needRecord() {
+				details.setElemMatchKey(element);
+			}
+			*/
+			return true;
+		};
+	return this._elementPath._matches(doc, details, checker);
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(element){
+	// File: expression_array.cpp lines: 56-59
+	if (!(element instanceof Array))
+		return false;
+	return this.matchesArray(element, null);
+};
+
+/**
+ *
+ * return the internal path
+ * @method path
+ * @param
+ *
+ */
+proto.path = function path(){
+	// File: expression_array.h lines: 52-51
+	return this._path;
+};
+
+/**
+ *
+ * Check if the input array matches
+ * @method path
+ * @param anArray
+ * @param details
+ *
+ */
+proto.matchesArray = function matchesArray(anArray, details){
+	throw new Error("not implemented");
+};

lib/pipeline/matcher/AtomicMatchExpression.js

@@ -0,0 +1,67 @@
+"use strict";
+var MatchExpression = require('./MatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var AtomicMatchExpression = module.exports = function AtomicMatchExpression(){
+	base.call(this);
+	this._matchType = 'ATOMIC';
+}, klass = AtomicMatchExpression, base =  MatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression.cpp lines: 48-50
+	return this._debugAddSpace( level ) + "$atomic\n";
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression.h lines: 198-199
+	return other._matchType == 'ATOMIC';
+};
+
+/**
+ *
+ * matches checks the input doc against the internal element path to see if it is a match
+ * @method matches
+ * @param doc
+ *
+ */
+proto.matches = function matches(doc) {
+	// File: expression.h lines: 184-185
+	return true;
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e) {
+	// File: expression.h lines: 188-189
+	return true;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression.h lines: 192-193
+	return new AtomicMatchExpression();
+};

lib/pipeline/matcher/ComparisonMatchExpression.js

@@ -0,0 +1,161 @@
+"use strict";
+var LeafMatchExpression = require('./LeafMatchExpression.js');
+var Value = require('../Value');
+
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ComparisonMatchExpression = module.exports = function ComparisonMatchExpression( type ){
+	base.call(this);
+	this._matchType = type;
+}, klass = ComparisonMatchExpression, base =  LeafMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+// File: expression_leaf.h lines: 88-88
+proto._rhs = undefined;
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression_leaf.cpp lines: 135-154
+	var retStr = this._debugAddSpace( level ) + this.path() + " ";
+	switch (this._matchType){
+		case 'LT':
+			retStr += '$lt';
+			break;
+		case 'LTE':
+			retStr += '$lte';
+			break;
+		case 'EQ':
+			retStr += '==';
+			break;
+		case 'GT':
+			retStr += '$gt';
+			break;
+		case 'GTE':
+			retStr += '$gte';
+			break;
+		default:
+			retStr += "Unknown comparison!";
+			break;
+	}
+
+	retStr += (this._rhs ? this._rhs.toString() : '?');
+	if ( this.getTag() ) {
+		retStr += this.getTag().debugString();
+	}
+	return retStr + '\n';
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression_leaf.cpp lines: 53-61
+	if (other._matchType != this._matchType)  return false;
+	return this.path() === other.path() && Value.compare(this._rhs,other._rhs) === 0;
+};
+
+/**
+ *
+ * Return the _rhs property
+ * @method getData
+ *
+ */
+proto.getData = function getData(){
+	// File: expression_leaf.h lines: 85-84
+	return this._rhs;
+};
+
+/**
+ *
+ * Return the _rhs property
+ * @method getRHS
+ *
+ */
+proto.getRHS = function getRHS(){
+	// File: expression_leaf.h lines: 79-78
+	return this._rhs;
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ * @param type
+ *
+ */
+proto.init = function init(path,rhs) {
+	// File: expression_leaf.cpp lines: 65-87
+	this._rhs = rhs;
+	if ( (rhs instanceof Object && Object.keys(rhs).length === 0)) { return {'code':'BAD_VALUE', 'description':'Need a real operand'};}
+
+	if ( rhs === undefined ) { return {'code':'BAD_VALUE', 'desc':'Cannot compare to undefined'};}
+	if (!(this._matchType in {"LT":1, "LTE":1, "EQ":1, "GT":1, "GTE":1})) {
+		return {'code':'BAD_VALUE', 'description':'Bad match type for ComparisonMatchExpression'};
+	}
+	return this.initPath( path );
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e){
+	// File: expression_leaf.cpp lines: 91-132
+	if( typeof(e) != typeof(this._rhs) ){
+		if( this._rhs === null) {
+			if(this._matchType in {'EQ':1, 'LTE':1, 'GTE':1}){
+				if(e === undefined || e === null || (e instanceof Object && Object.keys(e).length === 0)) {
+					return true;
+				}
+			}
+			return false;
+		}
+		if((e === null || e === undefined) && (this._rhs ===null || this._rhs === undefined)) {
+			return ["EQ","LTE","GTE"].indexOf(this._matchType) != -1;
+		}
+
+		if (this._rhs.constructor.name in {'MaxKey':1,'MinKey':1} ) {
+			return this._matchType != "EQ";
+		}
+		return false;
+	}
+
+	if( this._rhs instanceof Array) {
+		if( this._matchType != 'EQ') {
+			return false;
+		}
+	}
+
+	var x = Value.compare( e, this._rhs );
+
+	switch( this._matchType ) {
+		case "LT":
+			return x == -1;
+		case "LTE":
+			return x <= 0;
+		case "EQ":
+			return x === 0;
+		case "GT":
+			return x === 1;
+		case "GTE":
+			return x >= 0;
+		default:
+			throw new Error("Invalid comparison type evaluated.");
+	}
+	return false;
+};
+

lib/pipeline/matcher/Context.js

@@ -0,0 +1,73 @@
+"use strict";
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var Context = module.exports = function (){
+
+}, klass = Context, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// File: path.h lines: 62-62
+proto._arrayOffset = undefined;
+
+// File: path.h lines: 61-61
+proto._element = undefined;
+
+// File: path.h lines: 63-63
+proto._outerArray = undefined;
+
+/**
+ *
+ * Return the _arrayOffset property
+ * @method arrayOffset
+ *
+ */
+proto.arrayOffset = function arrayOffset( ){
+	// File: path.h lines: 57-56
+	return this._arrayOffset;
+};
+
+/**
+ *
+ * Return the _element property
+ * @method element
+ *
+ */
+proto.element = function element( ){
+	// File: path.h lines: 56-55
+	return this._element;
+};
+
+/**
+ *
+ * Return the _outerArray property
+ * @method outerArray
+ *
+ */
+proto.outerArray = function outerArray( ){
+	// File: path.h lines: 58-57
+	return this._outerArray;
+};
+
+/**
+ *
+ * Set _element to a new empty object
+ * @method reset
+ *
+ */
+proto.reset = function reset(){
+	// File: path.cpp lines: 37-38
+	this._element = {};
+};
+
+/**
+ *
+ * Set _arrayOffset property to the input element
+ * @method setArrayOffset
+ * @param e
+ *
+ */
+proto.setArrayOffset = function setArrayOffset(e){
+	// File: path.h lines: 54-53
+	this._arrayOffset = e;
+};
+
+

lib/pipeline/matcher/ElemMatchObjectMatchExpression.js

@@ -0,0 +1,97 @@
+"use strict";
+var ArrayMatchingMatchExpression = require('./ArrayMatchingMatchExpression.js');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ElemMatchObjectMatchExpression = module.exports = function ElemMatchObjectMatchExpression(){
+	base.call(this);
+}, klass = ElemMatchObjectMatchExpression, base = ArrayMatchingMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// File: expression_array.h lines: 77-77
+proto._sub = undefined;
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level){
+	// File: expression_array.cpp lines: 106-109
+	var debug = this._debugAddSpace(level);
+	debug = debug + this.path() + " $elemMatch\n";
+	return debug + this._sub.debugString(level + 1);
+};
+
+/**
+ *
+ * Return the _sub property
+ * @method getChild
+ *
+ */
+proto.getChild = function getChild(){
+	// File: expression_array.h lines: 74-73
+	return this._sub;
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ * @param type
+ *
+ */
+proto.init = function init(path, sub){
+	// File: expression_array.cpp lines: 85-87
+	this._sub = sub;
+	return this.initPath(path);
+};
+
+/**
+ *
+ * Check if one of the items in the input array matches _sub
+ * @method matchesArray
+ * @param anArray
+ * @param details
+ *
+ */
+proto.matchesArray = function matchesArray(anArray, details){
+	// File: expression_array.cpp lines: 90-103
+	for (var i in anArray) {
+		var inner = anArray[i];
+		if (!(inner instanceof Object))
+			continue;
+		if (this._sub.matchesBSON(inner, null)) {
+			if (details && details.needRecord()) {
+				details.setElemMatchKey(i);
+			}
+			return true;
+		}
+	}
+	return false;
+};
+
+/**
+ *
+ * Return 1 since we have a single "sub"
+ * @method numChildren
+ *
+ */
+proto.numChildren = function numChildren(){
+	// File: expression_array.h lines: 73-72
+	return 1;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression_array.h lines: 65-68
+	var element = new ElemMatchObjectMatchExpression();
+	element.init(this.path(), this._sub.shallowClone());
+	return element;
+};

lib/pipeline/matcher/ElemMatchValueMatchExpression.js

@@ -0,0 +1,140 @@
+"use strict";
+var ArrayMatchingMatchExpression = require('./ArrayMatchingMatchExpression.js');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ElemMatchValueMatchExpression = module.exports = function ElemMatchValueMatchExpression(){
+	base.call(this);
+	this._matchType = 'ELEM_MATCH_VALUE';
+	this._subs = [];
+}, klass = ElemMatchValueMatchExpression, base =  ArrayMatchingMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes;
+
+// File: expression_array.h lines: 108-108
+proto._subs = undefined;
+
+/**
+ *
+ * Check if the input element matches all items in the array
+ * @method _arrayElementMatchesAll
+ * @param element
+ *
+ */
+proto._arrayElementMatchesAll = function _arrayElementMatchesAll(element){
+	// File: expression_array.cpp lines: 152-157
+	for (var i = 0; i < this._subs.length; i++ ) {
+		if (!this._subs[i].matchesSingleElement(element))
+			return false;
+	}
+	return true;
+};
+
+/**
+ *
+ * push an item onto the internal array
+ * @method add
+ * @param sub
+ *
+ */
+proto.add = function add(sub){
+	// File: expression_array.cpp lines: 132-134
+	if (!sub) throw new Error(sub + " ElemMatchValueMatchExpression:36");
+	this._subs.push(sub);
+};
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level){
+	// File: expression_array.cpp lines: 160-165
+	var debug = this._debugAddSpace(level);
+	debug = debug + this.path() + " $elemMatch\n";
+	for (var i = 0; i < this._subs.length; i++) {
+		debug = debug + this._subs[i].debugString(level + 1);
+	}
+	return debug;
+};
+
+/**
+ *
+ * Get the given child in the internal array
+ * @method getChild
+ * @param i
+ *
+ */
+proto.getChild = function getChild(i){
+	// File: expression_array.h lines: 103-102
+	return this._subs[i];
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ * @param sub
+ *
+ */
+proto.init = function init(path, sub){
+	// File: expression_array.cpp lines: 121-124
+	this.initPath(path);
+	if (sub)
+		this.add(sub);
+	return {code:ErrorCodes.OK};
+};
+
+/**
+ *
+ * Check if one of the items in the input array matches everything in the internal array
+ * @method matchesArray
+ * @param anArray
+ * @param details
+ *
+ */
+proto.matchesArray = function matchesArray(anArray, details){
+	// File: expression_array.cpp lines: 137-149
+	for (var i in anArray) {
+		var inner = anArray[i];
+
+		if (this._arrayElementMatchesAll(inner)) {
+			if (details && details.needRecord()) {
+				details.setElemMatchKey(i);
+			}
+			return true;
+		}
+	}
+	return false;
+};
+
+/**
+ *
+ * Return the number of items in the internal array
+ * @method numChildren
+ *
+ */
+proto.numChildren = function numChildren(){
+	// File: expression_array.h lines: 102-101
+	return this._subs.length;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression_array.h lines: 91-97
+	var element = new ElemMatchValueMatchExpression();
+	element.init(this.path());
+	for (var i = 0; i < this._subs.length; ++i) {
+		element.add(this._subs[i].shallowClone());
+	}
+	return element;
+};

lib/pipeline/matcher/ElementPath.js

@@ -0,0 +1,232 @@
+"use strict";
+
+var FieldRef = require('./FieldRef');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ElementPath = module.exports = function ElementPath(){
+	this._fieldRef = new FieldRef();
+	this._shouldTraverseLeafArray = false;
+}, klass = ElementPath, base =  Object  , proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+// File: path.h lines: 41-41
+//         FieldRef _fieldRef;
+
+proto._fieldRef = undefined;
+
+
+// File: path.h lines: 42-42
+//         bool _shouldTraverseLeafArray;
+
+proto._shouldTraverseLeafArray = undefined;
+
+/**
+ * getFieldDottedOrArray
+ *
+ * @method getFieldDottedArray
+ * @param doc
+ * @param path
+ * @param idxPathObj This is an object with a pathID element. This allows for pass by ref in calling function.
+ * */
+klass.getFieldDottedOrArray = function getFieldDottedOrArray(doc, path, idxPathObj){
+	// File path_internal.cpp lines 31-72
+	if (path.numParts() === 0 ) { return doc; }
+
+	var res,curr = doc,
+		stop = false,
+		partNum = 0;
+	while (partNum < path.numParts() && !stop) {
+		res = curr[path.getPart( partNum)];
+		if(res == {}){
+			stop = true;
+		} else if (res instanceof Object) {
+			curr = res;
+			partNum++;
+		} else if (res instanceof Array) {
+			stop = true;
+		} else {
+			if (partNum + 1 < path.numParts() ) {
+				res = {};
+			}
+			stop = true;
+		}
+
+	}
+
+	//idxPathObj.pathID = partNum;
+	return res;
+};
+
+/**
+ * isAllDigits does what it says on the tin.
+ *
+ * @method isAllDigits
+ * @param str
+ */
+
+klass.isAllDigits = function isAllDigits ( str ){
+	// File path_internal.cpp lines 23-29
+	var digitCheck = /\D/g;
+	if (digitCheck.exec(str) === null){ return true; }
+	return false;
+};
+
+
+
+
+
+/**
+ *
+ * return the internal fieldRef object
+ * @method fieldRef
+ * @param
+ *
+ */
+proto.fieldRef = function fieldRef( /*  */ ){
+// File: path.h lines: 37-36
+	return this._fieldRef;
+};
+
+
+/**
+ *
+ * Initialize necessary items on this instance
+ * @method init
+ * @param path
+ *
+ */
+proto.init = function init( path ){  //  const StringData& path
+// File: path.cpp lines: 26-29
+	this._shouldTraverseLeafArray = true;
+	this._fieldRef.parse( path );
+	return {'code':'OK'};
+};
+
+
+/**
+ *
+ * Set whether paths should traverse leaves inside arrays
+ * @method setTraverseLeafArray
+ * @param
+ *
+ */
+proto.setTraverseLeafArray = function setTraverseLeafArray( b ){ //  bool b
+// File: path.h lines: 35-34
+	this._shouldTraverseLeafArray = b;
+};
+
+
+/**
+ *
+ * Return whether arrays should traverse leaf arrays
+ * @method shouldTraverseLeafArray
+ * @param
+ *
+ */
+proto.shouldTraverseLeafArray = function shouldTraverseLeafArray( /*  */ ){
+// File: path.h lines: 38-37
+	return this._shouldTraverseLeafArray;
+};
+
+proto.objAtPath = function objAtPath(doc) {
+	return klass.objAtPath(doc, this._fieldRef._path);
+};
+
+klass.objAtPath = function objAtPath(doc, path) {
+	if(path.length === 0) {
+		return doc;
+	}
+	if (path.length > 0 && Object.keys(doc).length === 0){
+		return {};
+	}
+	if (doc === null || doc === undefined) {
+		return doc;
+	}
+	var tpath = path.split('.');
+	return klass.objAtPath(doc[tpath[0]],tpath.slice(1).join('.'));
+};
+
+
+/**
+ *
+ * Helper to wrap our path into the static method
+ * @method _matches
+ * @param doc
+ * @param details
+ * @param function checker this function is used to check for a valid item at the end of the path
+ *
+ */
+proto._matches = function _matches(doc, details, checker) {
+	return klass._matches(doc, this._fieldRef._array, this._shouldTraverseLeafArray, details, checker);
+};
+
+/**
+ *
+ * _matches exists because we don't have pathIterators, so we need a recursive function call
+ * through the path pieces
+ * @method _matches
+ * @param doc
+ * @param path
+ * @param details
+ * @param function checker this function is used to check for a valid item at the end of the path
+ *
+ */
+klass._matches = function _matches(doc, path, shouldTraverseLeafArray, details, checker){
+	// File: expression_array.cpp lines: 34-53
+	var k, result, ii, il,
+		curr = doc,
+		item = doc;
+	for (k = 0; k < path.length; k++) {
+		if((curr instanceof Object) && (path[k] in curr)) {
+			item = curr[path[k]];
+		}
+		if(path[k].length === 0)
+			continue;
+		item = curr[path[k]];
+		if (item instanceof Object && item.constructor === Object) {
+			if (!(isNaN(parseInt(path[k], 10)))) {
+				result = checker(item[path[k]]);
+				if (result) {
+					if (details && details.needRecord())
+						details.setElemMatchKey(ii.toString());
+					return result;
+				}
+			}
+			curr = item;
+			continue;
+		} else if (item instanceof Object && item.constructor === Array) {
+			if (k == path.length-1) {
+				if ((shouldTraverseLeafArray) && (isNaN(parseInt(path[k], 10)))) {
+					for (ii = 0, il = item.length; ii < il; ii++) {
+						result = checker(item[ii]);
+						if (result) {
+							if (details && details.needRecord())
+								details.setElemMatchKey(ii.toString());
+							return result;
+						}
+					}
+					if(item.length === 0)
+						return checker({});
+					
+				}
+				curr = item;
+				break; // this is the end of the path, so check this array
+			} else if (!(isNaN(parseInt(path[k+1], 10)))) {
+				curr = item;
+				continue; // the *next* path section is an item in the array so we don't check this whole array
+			}
+			// otherwise, check each item in the array against the rest of the path
+			for(ii = 0, il = item.length; ii < il; ii++){
+				var subitem = item[ii];
+				if (subitem.constructor !== Object) continue;	// can't look for a subfield in a non-object value.
+				if (this._matches(subitem, path.slice(k), shouldTraverseLeafArray, null, checker)) { // check the item against the rest of the path
+					if (details && details.needRecord())
+						details.setElemMatchKey(ii.toString());
+					return true;
+				}
+			}
+			return false; // checked all items in the array and found no matches
+		}
+	}
+	return checker(item);
+};

lib/pipeline/matcher/EqualityMatchExpression.js

@@ -0,0 +1,24 @@
+"use strict";
+
+var ComparisonMatchExpression = require('./ComparisonMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var EqualityMatchExpression = module.exports = function EqualityMatchExpression(){
+	base.call(this);
+	this._matchType = 'EQ';
+}, klass = EqualityMatchExpression, base =  ComparisonMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Return a new instance of this class, with fields set the same as ourself
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /*  */ ){
+// File: expression_leaf.h lines: 98-101
+	var e = new EqualityMatchExpression();
+	e.init ( this.path(), this._rhs );
+	return e;
+};
+

lib/pipeline/matcher/ExistsMatchExpression.js

@@ -0,0 +1,81 @@
+"use strict";
+var LeafMatchExpression = require('./LeafMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var ExistsMatchExpression = module.exports = function ExistsMatchExpression(){
+	base.call(this);
+	this._matchType = 'EXISTS';
+}, klass = ExistsMatchExpression, base =  LeafMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression_leaf.cpp lines: 286-294
+	return this._debugAddSpace( level ) + this.path() + " exists" + (this.getTag() ? " " + this.getTag().debugString() : "") + "\n";
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression_leaf.cpp lines: 297-302
+	if(this._matchType != other._matchType)	{
+		return false;
+	}
+	return this.path() == other.path();
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ * @param type
+ *
+ */
+proto.init = function init(path) {
+	// File: expression_leaf.cpp lines: 278-279
+	return this.initPath( path );
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e) {
+	// File: expression_leaf.cpp lines: 282-283
+	if(typeof(e) == 'undefined')
+		return false;
+	if(e === null)
+		return true;
+	if(typeof(e) == 'object')
+		return (Object.keys(e).length > 0);
+	else
+		return true;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression_leaf.h lines: 220-223
+	var e = new ExistsMatchExpression();
+	e.init(this.path());
+	return e;
+};
+

lib/pipeline/matcher/FalseMatchExpression.js

@@ -0,0 +1,70 @@
+"use strict";
+var MatchExpression = require('./MatchExpression');
+
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var FalseMatchExpression = module.exports = function FalseMatchExpression(){
+	base.call(this);
+	this._matchType = 'ALWAYS_FALSE';
+}, klass = FalseMatchExpression, base =  MatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression.cpp lines: 53-55
+	return this._debugAddSpace( level ) + "$false\n";
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression.h lines: 222-223
+	return other._matchType === 'ALWAYS_FALSE';
+};
+
+/**
+ *
+ * matches checks the input doc against the internal element path to see if it is a match
+ * @method matches
+ * @param doc
+ * @param details
+ *
+ */
+proto.matches = function matches(doc,details) {
+	// File: expression.h lines: 208-209
+	return false;
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e) {
+	// File: expression.h lines: 212-213
+	return false;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression.h lines: 216-217
+	return new FalseMatchExpression();
+};
+

lib/pipeline/matcher/FieldRef.js

@@ -0,0 +1,111 @@
+"use strict";
+
+var FieldRef = module.exports = function FieldRef (){
+	this._array = [];
+	this._path = '';
+}, klass = FieldRef, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+proto.parse = function parse( str ) {
+	this._array = str.split('.');
+	this._path = this._array.join('.');
+};
+
+proto.setPart = function setPart( i, part ) {
+	this._array[i] = part;
+	this._path = this._array.join('.');
+};
+
+proto.getPart = function getPArt( i ) {
+	return this._array[i];
+};
+
+
+proto.isPrefixOf = function isPrefixOf( other ) {
+	return ( other._path.indexOf(this.path) === 0 );
+};
+
+proto.commonPrefixSize = function commonPrefixSize ( other ) {
+	var i = 0;
+	while(other._array[i] == this._array[i]) { i++; }
+	return i;
+};
+
+proto.dottedField = function dottedField( ) {
+	var offset = 0;
+	if(arguments.length == 1){
+		offset = arguments[0];
+	}
+	return this._array.slice( offset ).join('.');
+};
+
+proto.equalsDottedField = function equalsDottedField ( other ) {
+	return this._path == other._path;
+};
+
+proto.compare = function compare( other ) {
+	return (this._path < other._path ? -1 : this._path > other._path ? 1 : 0);
+};
+
+proto.clear = function clear() {
+	this._path = '';
+	this._array = [];
+};
+
+proto.numParts = function numParts() {
+	return this._array.length;
+};
+
+proto.numReplaced = function numReplaced() {
+	throw new Error('Why?');
+};
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

lib/pipeline/matcher/GTEMatchExpression.js

@@ -0,0 +1,24 @@
+"use strict";
+
+var ComparisonMatchExpression = require('./ComparisonMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var GTEMatchExpression = module.exports = function GTEMatchExpression(){
+	base.call(this);
+	this._matchType = 'GTE';
+}, klass = GTEMatchExpression, base =  ComparisonMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+/**
+ *
+ * Return a new instance of this class, with fields set the same as ourself
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /*  */ ){
+// File: expression_leaf.h lines: 141-144
+	var e = new GTEMatchExpression();
+	e.init( this.path(), this._rhs );
+	return e;
+};
+

lib/pipeline/matcher/GTMatchExpression.js

@@ -0,0 +1,25 @@
+"use strict";
+
+var ComparisonMatchExpression = require('./ComparisonMatchExpression.js');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var GTMatchExpression = module.exports = function GTMatchExpression(){
+	base.call(this);
+	this._matchType = 'GT';
+}, klass = GTMatchExpression, base = ComparisonMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+
+/**
+ *
+ * Return a new instance of this class, with fields set the same as ourself
+ * @method shallowClone
+ * @param
+ *
+ */
+proto.shallowClone = function shallowClone( /* */ ){
+	// File: expression_leaf.h lines: 130-133
+	var e = new GTMatchExpression();
+	e.init( this.path(), this._rhs );
+	return e;
+};
+

lib/pipeline/matcher/InMatchExpression.js

@@ -0,0 +1,161 @@
+"use strict";
+var LeafMatchExpression = require('./LeafMatchExpression');
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var InMatchExpression = module.exports = function InMatchExpression(){
+	base.call(this);
+	this._matchType = 'MATCH_IN';
+	this._arrayEntries = new ArrayFilterEntries();
+}, klass = InMatchExpression, base =  LeafMatchExpression, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// DEPENDENCIES
+var errors = require("../../Errors.js"),
+	ErrorCodes = errors.ErrorCodes,
+	ArrayFilterEntries = require("./ArrayFilterEntries.js");
+
+// File: expression_leaf.h lines: 294-294
+proto._arrayEntries = null;
+
+/**
+ *
+ * Check if the input element matches a real element
+ * @method _matchesRealElement
+ * @param e
+ *
+ */
+proto._matchesRealElement = function _matchesRealElement(e) {
+	// File: expression_leaf.cpp lines: 422-431
+	if(this._arrayEntries.contains(e)) { // array wrapper.... so no e "in" array
+		return true;
+	}
+
+	for (var i = 0; i < this._arrayEntries.numRegexes(); i++) {
+		if(e.match && e.match(this._arrayEntries.regex(i)._regex)) {
+			return true;
+		} else if (e instanceof RegExp) {
+			if(e.toString() == this._arrayEntries.regex(i)._regex.toString()) {
+				return true;
+			}
+		}
+	}
+
+	if(typeof(e) == 'undefined') {
+		return true; // Every Set contains the Null Set, man.
+	}
+
+	return false;
+};
+
+/**
+ *
+ * Copy our array to the input array
+ * @method copyTo
+ * @param toFillIn
+ *
+ */
+proto.copyTo = function copyTo(toFillIn) {
+	// File: expression_leaf.cpp lines: 481-483
+	toFillIn.init(this.path());
+	this._arrayEntries.copyTo( toFillIn._arrayEntries );
+};
+
+/**
+ *
+ * Writes a debug string for this object
+ * @method debugString
+ * @param level
+ *
+ */
+proto.debugString = function debugString(level) {
+	// File: expression_leaf.cpp lines: 455-463
+	return this._debugAddSpace( level ) + this.path() + ";$in: TODO " + (this.getTag() ? this.getTag().debugString() : '') + "\n";
+};
+
+/**
+ *
+ * checks if this expression is == to the other
+ * @method equivalent
+ * @param other
+ *
+ */
+proto.equivalent = function equivalent(other) {
+	// File: expression_leaf.cpp lines: 466-472
+	if ( other._matchType != 'MATCH_IN' ) {
+		return false;
+	}
+	return this.path() == other.path() && this._arrayEntries.equivalent( other._arrayEntries );
+};
+
+/**
+ *
+ * Return the _arrayEntries property
+ * @method getArrayFilterEntries
+ *
+ */
+proto.getArrayFilterEntries = function getArrayFilterEntries(){
+	// File: expression_leaf.h lines: 280-279
+	return this._arrayEntries;
+};
+
+/**
+ *
+ * Return the _arrayEntries property
+ * @method getData
+ *
+ */
+proto.getData = function getData(){
+	// File: expression_leaf.h lines: 290-289
+	return this._arrayEntries;
+};
+
+/**
+ *
+ * Initialize the necessary items
+ * @method init
+ * @param path
+ *
+ */
+proto.init = function init(path) {
+	// File: expression_leaf.cpp lines: 418-419
+	return this.initPath( path );
+};
+
+/**
+ *
+ * Check if the input element matches
+ * @method matchesSingleElement
+ * @param e
+ *
+ */
+proto.matchesSingleElement = function matchesSingleElement(e) {
+	// File: expression_leaf.cpp lines: 434-452
+	if( this._arrayEntries === null && typeof(e) == 'object' && Object.keys(e).length === 0) {
+		return true;
+	}
+	if (this._matchesRealElement( e )) {
+		return true;
+	}
+	/*if (e instanceof Array){
+		for (var i = 0; i < e.length; i++) {
+			if(this._matchesRealElement( e[i] )) {
+				return true;
+			}
+		}
+
+	}*/
+	return false;
+};
+
+/**
+ *
+ * clone this instance to a new one
+ * @method shallowClone
+ *
+ */
+proto.shallowClone = function shallowClone(){
+	// File: expression_leaf.cpp lines: 475-478
+	var e = new InMatchExpression();
+	this.copyTo( e );
+	return e;
+};
+

lib/pipeline/matcher/IndexKeyMatchableDocument.js

@@ -0,0 +1,58 @@
+"use strict";
+
+// Autogenerated by cport.py on 2013-09-17 14:37
+var IndexKeyMatchableDocument = module.exports = function IndexKeyMatchableDocument(pattern, doc){
+	this._pattern = pattern;
+	this._doc = doc;
+}, klass = IndexKeyMatchableDocument, base =  Object  , proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+// File: matcher.cpp lines: 52-52
+proto._doc = undefined;
+
+// File: matcher.cpp lines: 51-51
+proto._pattern = undefined;
+
+// File: matcher.cpp lines: 52-52
+proto._doc = undefined;
+
+// File: matcher.cpp lines: 51-51
+proto._pattern = undefined;
+
+/**
+ *
+ * get the element at the input path
+ * @method _getElement
+ * @param path
+ *
+ */
+proto._getElement = function _getElement(path){
+	// File: matcher.cpp lines: 63-77
+	var patternElement, docElement;
+
+	for (var i in this._pattern) {
+		patternElement = this._pattern[i];
+		//verify( docIterator.more() );
+		if(i >= this._doc.length) throw new Error("Ran out of docs in IndexKeyMatchableDocument:35");
+		docElement = this._doc[i];
+
+		if (path.equalsDottedField(patternElement.fieldName())) {
+			return docElement;
+		}
+	}
+
+	return {};
+};
+
+/**
+ *
+ * This method returns a JSON representation of the Document
+ * @method toBSON
+ * @param
+ *
+ */
+proto.toJSON = function toJSON(){
+	// File: matcher.cpp lines: 39-42
+	// TODO: this isn't quite correct because of dots
+	// don't think it'll ever be called though
+	return this._doc.replaceFieldNames(this._pattern);
+};