Limit concurrent blocks processed

2 files changed, 65 insertions, 10 deletions

diff --git a/lib/template.js b/lib/template.js
index 91b4850..08e1877 100644
--- a/lib/template.js
+++ b/lib/template.js
@@ -6,6 +6,7 @@ var nunjucks = require("nunjucks");
 var git = require("./utils/git");
 var stringUtils = require("./utils/string");
 var fs = require("./utils/fs");
+var batch = require("./utils/batch");
 var pkg = require("../package.json");
 
 
@@ -368,16 +369,19 @@ TemplateEngine.prototype.postProcess = function(content) {
     return Q(content)
     .then(that.replaceBlocks)
     .then(function(content) {
-        return Q.all(_.map(that.blocks, function(blk, blkId) {
-            return Q()
-            .then(function() {
-                if (!blk.post) return Q();
-                return blk.post();
-            })
-            .then(function() {
-                delete that.blocks[blkId];
-            });
-        }))
+        return batch.execEach(that.blocks, {
+            max: 20,
+            fn: function(blk, blkId) {
+                return Q()
+                .then(function() {
+                    if (!blk.post) return Q();
+                    return blk.post();
+                })
+                .then(function() {
+                    delete that.blocks[blkId];
+                });
+            }
+        })
         .thenResolve(content);
     });
 };
diff --git a/lib/utils/batch.js b/lib/utils/batch.js
new file mode 100644
index 0000000..5b92be5
--- /dev/null
+++ b/lib/utils/batch.js
@@ -0,0 +1,51 @@
+var Q = require("q");
+var _ = require("lodash");
+
+// Execute a method for all element
+function execEach(items, options) {
+    var concurrents = 0, d = Q.defer(), pending = [];
+
+    options = _.defaults(options || {}, {
+        max: 100,
+        fn: function(item) {}
+    });
+
+
+    function startItem(item, i) {
+        if (concurrents >= options.max) {
+            pending.push([item, i]);
+            return;
+        }
+
+        concurrents++;
+        Q()
+        .then(function() {
+            return options.fn(item, i);
+        })
+        .then(function() {
+            concurrents--;
+
+            // Next pending
+            var next = pending.shift();
+
+            if (concurrents == 0 && !next) {
+                d.resolve();
+            } else if (next) {
+                startItem.apply(null, next);
+            }
+        })
+        .fail(function(err) {
+            pending = [];
+            d.reject(err);
+        })
+    }
+
+    _.each(items, startItem);
+
+    return d.promise;
+}
+
+module.exports = {
+    execEach: execEach
+};
+