Merge pull request #23 from RiveraGroup/feature/mongo_2.6.5_dependencies

Feature/mongo 2.6.5 dependencies

lib/pipeline/DepsTracker.js

@@ -0,0 +1,105 @@
+"use strict";
+
+/**
+ * Allows components in an aggregation pipeline to report what they need from their input.
+ *
+ * @class DepsTracker
+ * @namespace mungedb-aggregate.pipeline
+ * @module mungedb-aggregate
+ * @constructor
+ */
+var DepsTracker = module.exports = function DepsTracker() {
+	// fields is a set of strings
+	this.fields = {};
+	this.needWholeDocument = false;
+	this.needTextScore = false;
+}, klass = DepsTracker, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+var ParsedDeps = require("./ParsedDeps");
+
+/**
+ * Returns a projection object covering the dependencies tracked by this class.
+ * @method toProjection
+ * @return {Object} projection of caller's dependencies
+ */
+proto.toProjection = function toProjection() {
+	var proj = {};
+
+	// if(this.needTextScore) {
+		// bb.append(Document::metaFieldTextScore, BSON("$meta" << "textScore"));
+	// }
+
+	if (this.needWholeDocument) {
+		return proj;
+	}
+
+	if (Object.keys(this.fields).length === 0) {
+		// Projection language lacks good a way to say no fields needed. This fakes it.
+		proj._id = 0;
+		proj.$noFieldsNeeded = 1;
+		return proj;
+	}
+
+	var last = "",
+		needId = false;
+
+	Object.keys(this.fields).sort().forEach(function (it) {
+		if (it.slice(0,3) == "_id" && (it.length == 3 || it.charAt(3) == ".")) {
+			// _id and subfields are handled specially due in part to SERVER-7502
+			needId = true;
+			return;
+		}
+
+		if (last !== "" && it.slice(0, last.length) === last) {
+			// we are including a parent of *it so we don't need to include this
+			// field explicitly. In fact, due to SERVER-6527 if we included this
+			// field, the parent wouldn't be fully included. This logic relies
+			// on on set iterators going in lexicographic order so that a string
+			// is always directly before of all fields it prefixes.
+			return;
+		}
+
+		last = it + ".";
+		proj[it] = 1;
+	});
+
+	if (needId)
+		proj._id = 1;
+	else
+		proj._id = 0;
+
+	return proj;
+};
+
+/**
+ * Takes a depsTracker and builds a simple recursive lookup table out of it.
+ * @method toParsedDeps
+ * @return {ParsedDeps}
+ */
+proto.toParsedDeps = function toParsedDeps() {
+	var doc = {};
+
+	if (this.needWholeDocument || this.needTextScore) {
+		// can't use ParsedDeps in this case
+		// TODO: not sure what appropriate equivalent to boost::none is
+		return;
+	}
+
+	var last = "";
+	Object.keys(this.fields).sort().forEach(function (it) {
+		if (last !== "" && it.slice(0, last.length) === last) {
+			// we are including a parent of *it so we don't need to include this
+			// field explicitly. In fact, due to SERVER-6527 if we included this
+			// field, the parent wouldn't be fully included. This logic relies
+			// on on set iterators going in lexicographic order so that a string
+			// is always directly before of all fields it prefixes.
+			return;
+		}
+
+		last = it + ".";
+		// TODO: set nested field to true; i.e. a.b.c = true, not a = true
+		doc[it] = true;
+	});
+
+	return new ParsedDeps(doc);
+};

lib/pipeline/Document.js

@@ -35,10 +35,23 @@ klass.toJson = function toJson(doc) {
 //SKIPPED: toBsonWithMetaData
 //SKIPPED: fromBsonWithMetaData
 
-//SKIPPED: MutableDocument
-
-//SKIPPED: getNestedFieldHelper
-//SKIPPED: getNestedField -- same as getNestedFieldHelper in our code
+//SKIPPED: most of MutableDocument except for getNestedField and setNestedField, squashed into Document here (because that's how they use it)
+function getNestedFieldHelper(obj, path) {
+	// NOTE: DEVIATION FROM MONGO: from MutableDocument, implemented a bit differently here but should be same basic functionality
+	var paths = Array.isArray(path) ? path : path.split(".");
+	for (var i = 0, l = paths.length, o = obj; i < l && o instanceof Object; i++) {
+		o = o[paths[i]];
+	}
+	return o;
+};
+klass.getNestedField = klass.getNestedFieldHelper;  // NOTE: due to ours being static these are the same
+klass.setNestedField = function setNestedField(obj, path, val) {
+	// NOTE: DEVIATION FROM MONGO: from MutableDocument, implemented a bit differently here but should be same basic functionality
+	var paths = Array.isArray(path) ? path : path.split("."),
+		key = paths.pop(),
+		parent = klass.getNestedField(obj, paths);
+	if (parent) parent[key] = val;
+};
 //SKIPPED: getApproximateSize -- not implementing mem usage right now
 //SKIPPED: hash_combine
 

lib/pipeline/ParsedDeps.js

@@ -0,0 +1,85 @@
+"use strict";
+
+/**
+ * This class is designed to quickly extract the needed fields into a Document.
+ * It should only be created by a call to DepsTracker.toParsedDeps.
+ *
+ * @class ParsedDeps
+ * @namespace mungedb-aggregate.pipeline
+ * @module mungedb-aggregate
+ * @constructor
+ * @param {Object} fields	The fields needed in a Document
+ */
+var ParsedDeps = module.exports = function ParsedDeps(fields) {
+	this._fields = fields;
+}, klass = ParsedDeps, base = Object, proto = klass.prototype = Object.create(base.prototype, {constructor:{value:klass}});
+
+var Value = require("./Value");
+
+/**
+ * Extracts fields from the input into a new Document, based on the caller.
+ *
+ * @method extractFields
+ * @param {Object} input	The JSON object to extract from
+ * @return {Document}
+ */
+proto.extractFields = function extractFields(input) {
+	return proto._documentHelper(input, this._fields);
+};
+
+/**
+ * Private: Handles array-type values for extractFields()
+ *
+ * @method _arrayHelper
+ * @param {Object} array	Array to iterate over
+ * @param {Object} neededFields
+ * @return {Array}
+ */
+proto._arrayHelper = function _arrayHelper(array, neededFields) {
+	var values = [];
+
+	for (var it in array) {
+		if (it instanceof Array)
+			values.push(_arrayHelper(it, neededFields));
+		else if (it instanceof Object)
+			values.push(proto._documentHelper(it, neededFields));
+	}
+
+	return values;
+};
+
+/**
+ * Private: Handles object-type values for extractFields()
+ *
+ * @method _documentHelper
+ * @param {Object} json	Object to iterate over and filter
+ * @param {Object} neededFields	Fields to not exclude
+ * @return {Document}
+ */
+proto._documentHelper = function _documentHelper(json, neededFields) {
+	var doc = {};
+
+	for (var fieldName in json) {
+		var jsonElement = json[fieldName],
+			isNeeded = neededFields[fieldName];
+
+		if (isNeeded === undefined)
+			continue;
+
+		if (Value.getType(isNeeded) === 'boolean') {
+			doc[fieldName] = jsonElement;
+			continue;
+		}
+
+		if (!isNeeded instanceof Object) throw new Error("dassert failure");
+
+		if (Value.getType(isNeeded) === 'object') {
+			if (jsonElement instanceof Array)
+				doc[fieldName] = proto._arrayHelper(jsonElement, isNeeded);
+			if (jsonElement instanceof Object)
+				doc[fieldName] = proto._documentHelper(jsonElement, isNeeded);
+		}
+	}
+
+	return doc;
+};

lib/pipeline/documentSources/DocumentSource.js

@@ -185,47 +185,6 @@ proto.getDependencies = function getDependencies(deps) {
 	return klass.GetDepsReturn.NOT_SUPPORTED;
 };
 
-/**
- * This takes dependencies from getDependencies and
- * returns a projection that includes all of them
- *
- * @method	depsToProjection
- * @param	{Object} deps	set (unique array) of strings
- * @returns	{Object}	JSONObj
- **/
-klass.depsToProjection = function depsToProjection(deps) {
-	var needId = false,
-		bb = {};
-	if (deps._id === undefined)
-		bb._id = 0;
-
-	var last = "";
-	Object.keys(deps).sort().forEach(function(it){
-		if (it.indexOf('_id') === 0 && (it.length === 3 || it[3] === '.')) {
-			needId = true;
-			return;
-		} else {
-			if (last !== "" && it.slice(0, last.length) === last){
-				// we are including a parent of *it so we don't need to
-				// include this field explicitly. In fact, due to
-				// SERVER-6527 if we included this field, the parent
-				// wouldn't be fully included.
-				return;
-			}
-		}
-		last = it + ".";
-		bb[it] = 1;
-	});
-
-	if (needId) // we are explicit either way
-		bb._id = 1;
-	else
-		bb._id = 0;
-
-
-	return bb;
-};
-
 proto._serialize = function _serialize(explain) {
 	throw new Error("not implemented");
 };
@@ -237,23 +196,6 @@ proto.serializeToArray = function serializeToArray(array, explain) {
 	}
 };
 
-klass.parseDeps = function parseDeps(deps) {
-	var md = {};
-
-	var last,
-		depKeys = Object.keys(deps);
-	for (var i = 0; i < depKeys.length; i++) {
-		var it = depKeys[i],
-			value = deps[it];
-
-		if (!last && it.indexOf(last) >= 0)
-			continue;
-		last = it + '.';
-		md[it] = true;
-	}
-	return md;
-};
-
 /**
  * A function compatible as a getNext for document sources.
  * Does nothing except pass the documents through. To use,
@@ -274,59 +216,3 @@ klass.GET_NEXT_PASS_THROUGH = function GET_NEXT_PASS_THROUGH(callback) {
 	});
 	return out; // For the sync people in da house
 };
-
-klass.documentFromJsonWithDeps = function documentFromJsonWithDeps(bson, neededFields) {
-	var arrayHelper = function(bson, neededFields) {
-		var values = [];
-
-		var bsonKeys = Object.keys(bson);
-		for (var i = 0; i < bsonKeys.length; i++) {
-			var key = bsonKeys[i],
-				bsonElement = bson[key];
-
-			if (bsonElement instanceof Object) {
-				var sub = klass.documentFromJsonWithDeps(bsonElement, isNeeded);
-				values.push(sub);
-			}
-
-			if (bsonElement instanceof Array) {
-				values.push(arrayHelper(bsonElement, neededFields));
-			}
-		}
-
-		return values;
-	};
-
-	var md = {};
-
-	var bsonKeys = Object.keys(bson);
-	for (var i = 0; i < bsonKeys.length; i++) {
-		var fieldName = bsonKeys[i],
-			bsonElement = bson[fieldName],
-			isNeeded = neededFields ? neededFields[fieldName] : null;
-
-		if (!isNeeded)
-			continue;
-
-		if (typeof(isNeeded) === 'boolean') {
-			md[fieldName] = bsonElement;
-			continue;
-		}
-
-		if (!isNeeded instanceof Object)
-			throw new Error("instanceof should be an instance of Object");
-
-		if (bsonElement instanceof Object) {
-			var sub = klass.documentFromJsonWithDeps(bsonElement, isNeeded);
-
-			md[fieldName] = sub;
-		}
-
-		if (bsonElement instanceof Array) {
-			md[fieldName] = arrayHelper(bsonElement, isNeeded);
-		}
-	}
-
-	return md;
-
-};

test/lib/pipeline/DepsTracker_test.js

@@ -0,0 +1,87 @@
+"use strict";
+var assert = require("assert"),
+	DepsTracker = require("../../../lib/pipeline/DepsTracker");
+
+// Mocha one-liner to make these tests self-hosted
+if(!module.parent)return(require.cache[__filename]=null,(new(require("mocha"))({ui:"exports",reporter:"spec",grep:process.env.TEST_GREP})).addFile(__filename).run(process.exit));
+
+exports.DepsTracker = {
+
+	"#toProjection()": {
+
+		"should be able to convert dependencies to a projection": function(){
+			var deps = new DepsTracker(),
+				expected = {_id:0,a:1,b:1};
+			deps.fields = {a:1,b:1};
+			assert.deepEqual(expected, deps.toProjection());
+		},
+
+		"should be able to convert dependencies with subfields to a projection": function(){
+			var deps = new DepsTracker(),
+				expected = {_id:0,a:1};
+			deps.fields = {a:1,"a.b":1};
+			assert.deepEqual(expected, deps.toProjection());
+		},
+
+		"should be able to convert dependencies with _id to a projection": function(){
+			var deps = new DepsTracker(),
+				expected = {a:1,b:1,_id:1};
+			deps.fields = {_id:1,a:1,b:1};
+			assert.deepEqual(expected, deps.toProjection());
+		},
+
+		"should be able to convert dependencies with id and subfields to a projection": function(){
+			var deps = new DepsTracker(),
+				expected = {_id:1,b:1};
+			deps.fields = {"_id.a":1,b:1};
+			assert.deepEqual(expected, deps.toProjection());
+		},
+
+		"should return empty object if needWholeDocument is true": function() {
+			var deps = new DepsTracker(),
+				expected = {};
+			deps.needWholeDocument = true;
+			assert.deepEqual(expected, deps.toProjection());
+		},
+
+		"should return $noFieldsNeeded if there are no dependencies": function() {
+			var deps = new DepsTracker(),
+				expected = {_id:0,$noFieldsNeeded:1};
+			assert.deepEqual(expected, deps.toProjection());
+		},
+
+	},
+
+	"#toParsedDeps()": {
+
+		"should not parse if needWholeDocument is true": function() {
+			var deps = new DepsTracker(),
+				expected; // undefined;
+			deps.needWholeDocument = true;
+			assert.strictEqual(expected, deps.toParsedDeps());
+		},
+
+		"should not parse if needTextScore is true": function() {
+			var deps = new DepsTracker(),
+				expected; // undefined;
+			deps.needTextScore = true;
+			assert.strictEqual(expected, deps.toParsedDeps());
+		},
+
+		"should be able to parse dependencies": function() {
+			var deps = new DepsTracker(),
+				expected = {_fields:{a:true,b:true}};
+			deps.fields = {a:1,b:1};
+			assert.deepEqual(expected, deps.toParsedDeps());
+		},
+
+		"should be able to parse dependencies with subfields": function() {
+			var deps = new DepsTracker(),
+				expected = {_fields:{a:true}};
+			deps.fields = {a:1,"a.b":1};
+			assert.deepEqual(expected, deps.toParsedDeps());
+		},
+
+	},
+
+};

test/lib/pipeline/ParsedDeps.js

@@ -0,0 +1,74 @@
+"use strict";
+var assert = require("assert"),
+	ParsedDeps = require("../../../lib/pipeline/ParsedDeps");
+
+module.exports = {
+	"ParsedDeps": {
+		"#extractFields": {
+			"should be able to convert a document to its projected form": function() {
+				var deps = {'a': true, 'b': true},
+					doc = {a:23, b:64, c:92},
+					parse = new ParsedDeps(deps);
+
+				var proj = parse.extractFields(doc);
+				assert.deepEqual({a:23,b:64}, proj);
+			}
+		},
+		"#_documentHelper": {
+			"should skip fields that are not needed": function() {
+				var json = {'foo':'bar'},
+					neededFields = {},
+					parse = new ParsedDeps(),
+					expected = {};
+				assert.deepEqual(expected, parse._documentHelper(json, neededFields));
+			},
+			"should return values that are booleans": function() {
+				var json = {'foo':'bar'},
+					neededFields = {'foo':true},
+					parse = new ParsedDeps(),
+					expected = {'foo':'bar'};
+				assert.deepEqual(expected, parse._documentHelper(json, neededFields));
+			},
+			"should call _arrayHelper on values that are arrays": function() {
+				var json = {'foo':[{'bar':'baz'}]},
+					neededFields = {'foo':true},
+					parse = new ParsedDeps(),
+					expected = {'foo':true};
+				// TODO: mock out _arrayHelper to return true
+				parse._arrayHelper = function() {
+					return true;
+				};
+				assert.deepEqual(expected, parse._documentHelper(json, neededFields));
+			},
+			"should recurse on values that are objects": function() {
+				var json = {'foo':{'bar':'baz'}},
+					neededFields = {'foo':true},
+					parse = new ParsedDeps(),
+					expected = {'foo':{'bar':'baz'}};
+				assert.deepEqual(expected, parse._documentHelper(json, neededFields));
+			}
+		},
+		"#_arrayHelper": {
+			"should call _documentHelper on values that are objects": function() {
+				var array = [{'foo':'bar'}],
+					neededFields = {'foo':true},
+					parse = new ParsedDeps(),
+					expected = [true];
+				// TODO: mock out _documentHelper to return true
+				parse._documentHelper = function() {
+					return true;
+				};
+				assert.deepEqual(expected, parse._arrayHelper(array, neededFields));
+			},
+			"should recurse on values that are arrays": function() {
+				var array = [[{'foo':'bar'}]],
+					neededFields = {'foo':true},
+					parse = new ParsedDeps(),
+					expected = [[{'foo':'bar'}]];
+				assert.deepEqual(expected, parse._arrayHelper(array, neededFields));
+			}
+		}
+	}
+};
+
+if (!module.parent)(new(require("mocha"))()).ui("exports").reporter("spec").addFile(__filename).run();

test/lib/pipeline/documentSources/DocumentSource.js

@@ -7,50 +7,8 @@ module.exports = {
 
 	"DocumentSource": {
 
-		"#depsToProjection()": {
-			"should be able to convert dependencies to a projection": function(){
-				var array = {'a':1,'b':1},
-					expected = '{"_id":0,"a":1,"b":1}',
-					proj = DocumentSource.depsToProjection(array);
-
-				assert.equal(expected, JSON.stringify(proj));
-			},
-			"should be able to convert dependencies with subfields to a projection": function(){
-				var array = {'a':1,'a.b':1},
-					expected = '{"_id":0,"a":1}',
-					proj = DocumentSource.depsToProjection(array);
-
-				assert.equal(expected, JSON.stringify(proj));
-			},
-			"should be able to convert dependencies with _id to a projection": function(){
-				var array = {"_id":1,'a':1,'b':1},
-					expected = '{"a":1,"b":1,"_id":1}',
-					proj = DocumentSource.depsToProjection(array);
-
-				assert.equal(expected, JSON.stringify(proj));
-			},
-			"should be able to convert dependencies with id and subfields to a projection": function(){
-				var array = {'_id.a':1,'b':1},
-					expected = '{"_id":1,"b":1}',
-					proj = DocumentSource.depsToProjection(array);
-
-				assert.equal(expected, JSON.stringify(proj));
-			},
-		},
-
-		"#documentFromJsonWithDeps()": {
-			"should be able to convert a document to its projected form": function() {
-				var deps = {'a': true, 'b': true},
-					doc = {a:23, b:64, c:92};
-
-				var proj = DocumentSource.documentFromJsonWithDeps(doc, deps);
-				assert.deepEqual({a:23,b:64}, proj);
-			}
-		}
-
 	}
 
 };
 
 if (!module.parent)(new(require("mocha"))()).ui("exports").reporter("spec").addFile(__filename).run();
-