CLI: Added an experimental --sparse option to limit pbjs output to ac…

…tually referenced types within main files; Other: Added a few more common google types from google/api, see #433

.travis.yml

@@ -5,17 +5,19 @@ node_js:
   - 4.3.2
   - 6
   - 7
+branches:
+  only: master
+script: npm test && npm run bench
+
 env: CXX=g++-4.8
 addons:
   apt:
     sources: ubuntu-toolchain-r-test
     packages: g++-4.8
-script: npm test && npm run bench
-branches:
-  only: master
+
 matrix:
   include:
   - node_js: 6
-    script: set -e; if [ -n "$SAUCE_USERNAME" ]; then npm install [email protected] [email protected]; travis_wait npm run zuul; sleep 3; fi
+    script: set -e; if [ -n "$SAUCE_USERNAME" ]; then npm install zuul@^3.11.1 zuul-ngrok@^4.0.0; travis_wait npm run zuul; sleep 3; fi
   - node_js: 6
-    script: npm run coverage-ci
+    script: npm install coveralls@^2.11.15; npm run coverage-ci

cli/pbjs.js

@@ -31,7 +31,7 @@ exports.main = function(args, callback) {
             lint   : "l"
         },
         string: [ "target", "out", "path", "wrap", "root", "lint" ],
-        boolean: [ "keep-case", "create", "encode", "decode", "verify", "convert", "delimited", "beautify", "comments", "es6" ],
+        boolean: [ "keep-case", "create", "encode", "decode", "verify", "convert", "delimited", "beautify", "comments", "es6", "sparse" ],
         default: {
             target    : "json",
             create    : true,
@@ -51,6 +51,9 @@ exports.main = function(args, callback) {
         files  = argv._,
         paths  = typeof argv.path === "string" ? [ argv.path ] : argv.path || [];
 
+    // protobuf.js package directory contains additional, otherwise non-bundled google types
+    paths.push(path.relative(process.cwd(), path.join(__dirname, "..")) || ".");
+
     if (!files.length) {
         var descs = Object.keys(targets).filter(function(key) { return !targets[key].private; }).map(function(key) {
             return "                  " + util.pad(key, 14, true) + targets[key].description;
@@ -71,6 +74,8 @@ exports.main = function(args, callback) {
                 "",
                 "  -o, --out       Saves to a file instead of writing to stdout.",
                 "",
+                "  --sparse        Exports only those types referenced from a main file (experimental).",
+                "",
                 chalk.bold.gray("  Module targets only:"),
                 "",
                 "  -w, --wrap      Specifies the wrapper to use. Also accepts a path to require a custom wrapper.",
@@ -124,17 +129,33 @@ exports.main = function(args, callback) {
 
     var root = new protobuf.Root();
 
+    var mainFiles = [];
+
     // Search include paths when resolving imports
     root.resolvePath = function pbjsResolvePath(origin, target) {
-        var filepath = protobuf.util.path.resolve(origin, target);
-        if (fs.existsSync(filepath))
-            return filepath;
+        var normOrigin = protobuf.util.path.normalize(origin),
+            normTarget = protobuf.util.path.normalize(target);
+        if (!normOrigin)
+            mainFiles.push(normTarget);
+
+        var resolved = protobuf.util.path.resolve(normOrigin, normTarget, true);
+        var idx = resolved.lastIndexOf("google/protobuf/");
+        if (idx > -1) {
+            var altname = resolved.substring(idx);
+            if (altname in protobuf.common)
+                resolved = altname;
+        }
+
+        if (fs.existsSync(resolved))
+            return resolved;
+
         for (var i = 0; i < paths.length; ++i) {
-            var ifilepath = protobuf.util.path.resolve(paths[i] + "/", target);
-            if (fs.existsSync(ifilepath))
-                return ifilepath;
+            var iresolved = protobuf.util.path.resolve(paths[i] + "/", target);
+            if (fs.existsSync(iresolved))
+                return iresolved;
         }
-        return filepath;
+
+        return resolved;
     };
 
     // Use es6 syntax if not explicitly specified on the command line and the es6 wrapper is used
@@ -153,19 +174,28 @@ exports.main = function(args, callback) {
         });
         process.stdin.on("end", function() {
             var source = Buffer.concat(data).toString("utf8");
-            if (source.charAt(0) !== "{") {
-                protobuf.parse(source, root, parseOptions);
-            } else {
-                var json = JSON.parse(source);
-                root.setOptions(json.options).addJSON(json);
+            try {
+                if (source.charAt(0) !== "{") {
+                    protobuf.parse.filename = "-";
+                    protobuf.parse(source, root, parseOptions);
+                } else {
+                    var json = JSON.parse(source);
+                    root.setOptions(json.options).addJSON(json);
+                }
+                callTarget();
+            } catch (err) {
+                if (callback)
+                    return callback(err);
+                throw err;
             }
-            callTarget();
         });
 
     // Load from disk
     } else {
         try {
-            root.loadSync(files, parseOptions); // sync is deterministic while async is not
+            root.loadSync(files, parseOptions).resolveAll(); // sync is deterministic while async is not
+            if (argv.sparse)
+                sparsify(root);
             callTarget();
         } catch (err) {
             if (callback) {
@@ -176,6 +206,62 @@ exports.main = function(args, callback) {
         }
     }
 
+    function markReferenced(tobj) {
+        tobj.referenced = true;
+        // also mark a type's fields and oneofs
+        if (tobj.fieldsArray)
+            tobj.fieldsArray.forEach(function(fobj) {
+                fobj.referenced = true;
+            });
+        if (tobj.oneofsArray)
+            tobj.oneofsArray.forEach(function(oobj) {
+                oobj.referenced = true;
+            });
+        // also mark an extension field's extended type, but not its (other) fields
+        if (tobj.extensionField)
+            tobj.extensionField.parent.referenced = true;
+    }
+
+    function sparsify(root) {
+
+        // 1. mark directly or indirectly referenced objects
+        util.traverse(root, function(obj) {
+            if (!obj.filename)
+                return;
+            if (mainFiles.indexOf(obj.filename) > -1)
+                util.traverseResolved(obj, markReferenced);
+        });
+
+        // 2. empty unreferenced objects
+        util.traverse(root, function(obj) {
+            var parent = obj.parent;
+            if (!parent || obj.referenced) // root or referenced
+                return;
+            // remove unreferenced namespaces
+            if (obj instanceof protobuf.Namespace) {
+                var hasReferenced = false;
+                util.traverse(obj, function(iobj) {
+                    if (iobj.referenced)
+                        hasReferenced = true;
+                });
+                if (hasReferenced) { // replace with plain namespace if a namespace subclass
+                    if (obj instanceof protobuf.Type || obj instanceof protobuf.Service) {
+                        var robj = new protobuf.Namespace(obj.name, obj.options);
+                        robj.nested = obj.nested;
+                        parent.add(robj);
+                    }
+                } else // remove completely if nothing inside is referenced
+                    parent.remove(obj);
+
+            // remove everything else unreferenced
+            } else if (!(obj instanceof protobuf.Namespace))
+                parent.remove(obj);
+        });
+
+        // 3. validate that everything is fine
+        root.resolveAll();
+    }
+
     function callTarget() {
         target(root, argv, function targetCallback(err, output) {
             if (err) {

cli/util.js

@@ -33,6 +33,38 @@ exports.requireAll = function requireAll(dirname) {
     return all;
 };
 
+exports.traverse = function traverse(current, fn) {
+    fn(current);
+    if (current.fieldsArray)
+        current.fieldsArray.forEach(function(field) {
+            traverse(field, fn);
+        });
+    if (current.oneofsArray)
+        current.oneofsArray.forEach(function(oneof) {
+            traverse(oneof, fn);
+        });
+    if (current.methodsArray)
+        current.methodsArray.forEach(function(method) {
+            traverse(method, fn);
+        });
+    if (current.nestedArray)
+        current.nestedArray.forEach(function(nested) {
+            traverse(nested, fn);
+        });
+};
+
+exports.traverseResolved = function traverseResolved(current, fn) {
+    fn(current);
+    if (current.resolvedType)
+        traverseResolved(current.resolvedType, fn);
+    if (current.resolvedKeyType)
+        traverseResolved(current.resolvedKeyType, fn);
+    if (current.resolvedRequestType)
+        traverseResolved(current.resolvedRequestType, fn);
+    if (current.resolvedResponseType)
+        traverseResolved(current.resolvedResponseType, fn);
+};
+
 exports.inspect = function inspect(object, indent) {
     if (!object)
         return "";

google/api/annotations.json

@@ -0,0 +1,83 @@
+{
+  "nested": {
+    "google": {
+      "nested": {
+        "api": {
+          "nested": {
+            "http": {
+              "type": "HttpRule",
+              "id": 72295728,
+              "extend": "google.protobuf.MethodOptions"
+            },
+            "HttpRule": {
+              "oneofs": {
+                "pattern": {
+                  "oneof": [
+                    "get",
+                    "put",
+                    "post",
+                    "delete",
+                    "patch",
+                    "custom"
+                  ]
+                }
+              },
+              "fields": {
+                "get": {
+                  "type": "string",
+                  "id": 2
+                },
+                "put": {
+                  "type": "string",
+                  "id": 3
+                },
+                "post": {
+                  "type": "string",
+                  "id": 4
+                },
+                "delete": {
+                  "type": "string",
+                  "id": 5
+                },
+                "patch": {
+                  "type": "string",
+                  "id": 6
+                },
+                "custom": {
+                  "type": "CustomHttpPattern",
+                  "id": 8
+                },
+                "selector": {
+                  "type": "string",
+                  "id": 1
+                },
+                "body": {
+                  "type": "string",
+                  "id": 7
+                },
+                "additionalBindings": {
+                  "rule": "repeated",
+                  "type": "HttpRule",
+                  "id": 11
+                }
+              }
+            }
+          }
+        },
+        "protobuf": {
+          "nested": {
+            "MethodOptions": {
+              "fields": {},
+              "extensions": [
+                [
+                  1000,
+                  536870911
+                ]
+              ]
+            }
+          }
+        }
+      }
+    }
+  }
+}

google/api/annotations.proto

@@ -0,0 +1,11 @@
+syntax = "proto3";
+
+package google.api;
+
+import "google/api/http.proto";
+import "google/protobuf/descriptor.proto";
+
+extend google.protobuf.MethodOptions {
+
+    HttpRule http = 72295728;
+}