Merge pull request #39 from GitbookIO/feature/search

Feature/search

6 files changed, 190 insertions, 63 deletions

diff --git a/lib/generate/site/index.js b/lib/generate/site/index.js
index 9c3c10c..474de2a 100644
--- a/lib/generate/site/index.js
+++ b/lib/generate/site/index.js
@@ -8,6 +8,8 @@ var fs = require("../fs");
 var parse = require("../../parse");
 var BaseGenerator = require("../generator");
 
+var indexer = require('./search_indexer');
+
 // Swig filter for returning the count of lines in a code section
 swig.setFilter('lines', function(content) {
     return content.split('\n').length;
@@ -21,6 +23,11 @@ swig.setFilter('mdLink', function(link) {
 var Generator = function() {
     BaseGenerator.apply(this, arguments);
 
+    // Attach methods to instance
+    _.bindAll(this);
+
+    this.indexer = indexer();
+
     // Load base template
     this.template = swig.compileFile(path.resolve(this.options.theme, 'templates/site.html'));
     this.langsTemplate = swig.compileFile(path.resolve(this.options.theme, 'templates/langs.html'));
@@ -53,6 +60,11 @@ Generator.prototype._writeTemplate = function(tpl, options, output) {
     });
 };
 
+Generator.prototype.indexPage = function(lexed, pagePath) {
+    this.indexer.add(lexed, pagePath);
+    return Q();
+};
+
 // Convert a markdown file to html
 Generator.prototype.convertFile = function(content, _input) {
     var that = this;
@@ -67,7 +79,17 @@ Generator.prototype.convertFile = function(content, _input) {
 
     return Q()
     .then(function() {
-        return parse.page(content, {
+        // Lex page
+        return parse.lex(content);
+    })
+    .then(function(lexed) {
+        // Index page in search
+        return that.indexPage(lexed, _output)
+        .then(_.constant(lexed));
+    })
+    .then(function(lexed) {
+        // Get HTML generated sections
+        return parse.page(lexed, {
             repo: that.options.githubId,
             dir: path.dirname(input) || '/'
         });
@@ -111,6 +133,18 @@ Generator.prototype.copyAssets = function() {
         path.join(that.options.output, "gitbook")
     );
 };
-Generator.prototype.finish = Generator.prototype.copyAssets;
+
+// Dump search index to disk
+Generator.prototype.writeSearchIndex = function() {
+    return fs.writeFile(
+        path.join(this.options.output, 'search_index.json'),
+        this.indexer.dump()
+    );
+};
+
+Generator.prototype.finish = function() {
+    return this.copyAssets()
+    .then(this.writeSearchIndex);
+};
 
 module.exports = Generator;
\ No newline at end of file
diff --git a/lib/generate/site/search_indexer.js b/lib/generate/site/search_indexer.js
new file mode 100644
index 0000000..b239b8b
--- /dev/null
+++ b/lib/generate/site/search_indexer.js
@@ -0,0 +1,71 @@
+var Q = require("q");
+var _ = require("lodash");
+
+var lunr = require('lunr');
+var marked = require('marked');
+var textRenderer = require('marked-text-renderer');
+
+
+function Indexer() {
+    if(!(this instanceof Indexer)) {
+        return new Indexer();
+    }
+
+    _.bindAll(this);
+
+    // Setup lunr index
+    this.idx = lunr(function () {
+        this.ref('url');
+
+        this.field('title', { boost: 10 });
+        this.field('body');
+    });
+
+    this.renderer = textRenderer();
+}
+
+Indexer.prototype.text = function(nodes) {
+    // Copy section
+    var section = _.toArray(nodes);
+
+    // marked's Render expects this, we don't use it yet
+    section.links = {};
+
+    var options = _.extend({}, marked.defaults, {
+        renderer: this.renderer
+    });
+
+    return marked.parser(section, options);
+};
+
+Indexer.prototype.addSection = function(path, section) {
+    var url = [path, section.id].join('#');
+
+    var title = this.text(
+        _.filter(section, {'type': 'heading'})
+    );
+
+    var body = this.text(
+        _.omit(section, {'type': 'heading'})
+    );
+
+    // Add to lunr index
+    this.idx.add({
+        url: url,
+        title: title,
+        body: body,
+    });
+};
+
+Indexer.prototype.add = function(lexedPage, url) {
+    var sections = lexedPage;
+
+    _.map(sections, _.partial(this.addSection, url));
+};
+
+Indexer.prototype.dump = function() {
+    return JSON.stringify(this.idx);
+};
+
+// Exports
+module.exports = Indexer;
diff --git a/lib/parse/index.js b/lib/parse/index.js
index ec98347..0e333e4 100644
--- a/lib/parse/index.js
+++ b/lib/parse/index.js
@@ -2,6 +2,7 @@ module.exports = {
     summary: require('./summary'),
     langs: require('./langs'),
     page: require('./page'),
+    lex: require('./lex'),
     progress: require('./progress'),
     navigation: require('./navigation'),
 };
diff --git a/lib/parse/lex.js b/lib/parse/lex.js
new file mode 100644
index 0000000..6b3236e
--- /dev/null
+++ b/lib/parse/lex.js
@@ -0,0 +1,74 @@
+var _ = require('lodash');
+var marked = require('marked');
+
+// Split a page up into sections (lesson, exercises, ...)
+function splitSections(nodes) {
+    var section = [];
+
+    return _.reduce(nodes, function(sections, el) {
+        if(el.type === 'hr') {
+            sections.push(section);
+            section = [];
+        } else {
+            section.push(el);
+        }
+
+        return sections;
+    }, []).concat([section]); // Add remaining nodes
+}
+
+// What is the type of this section
+function sectionType(nodes, idx) {
+    var codeNodes = _.filter(nodes, {
+        type: 'code'
+    }).length;
+
+    if(codeNodes === 3 && (idx % 2) == 1) {
+        return 'exercise';
+    }
+
+    return 'normal';
+}
+
+// Generate a uniqueId to identify this section in our code
+function sectionId(section, idx) {
+    return _.uniqueId('gitbook_');
+}
+
+function lexPage(src) {
+    // Lex file
+    var nodes = marked.lexer(src);
+
+    return _.chain(splitSections(nodes))
+    .map(function(section, idx) {
+        // Detect section type
+        section.type = sectionType(section, idx);
+        return section;
+    })
+    .map(function(section, idx) {
+        // Give each section an ID
+        section.id = sectionId(section, idx);
+        return section;
+
+    })
+    .filter(function(section) {
+        return !_.isEmpty(section);
+    })
+    .reduce(function(sections, section) {
+        var last = _.last(sections);
+
+        // Merge normal sections together
+        if(last && last.type === section.type && last.type === 'normal') {
+            last.push.apply(last, [{'type': 'hr'}].concat(section));
+        } else {
+            // Add to list of sections
+            sections.push(section);
+        }
+
+        return sections;
+    }, [])
+    .value();
+}
+
+// Exports
+module.exports = lexPage;
diff --git a/lib/parse/page.js b/lib/parse/page.js
index b17f593..56a9e60 100644
--- a/lib/parse/page.js
+++ b/lib/parse/page.js
@@ -2,6 +2,7 @@ var _ = require('lodash');
 var marked = require('marked');
 var hljs = require('highlight.js');
 
+var lex = require('./lex');
 var renderer = require('./renderer');
 
 var lnormalize = require('../utils/lang').normalize;
@@ -19,35 +20,6 @@ marked.setOptions({
 });
 
 
-// Split a page up into sections (lesson, exercises, ...)
-function splitSections(nodes) {
-    var section = [];
-
-    return _.reduce(nodes, function(sections, el) {
-        if(el.type === 'hr') {
-            sections.push(section);
-            section = [];
-        } else {
-            section.push(el);
-        }
-
-        return sections;
-    }, []).concat([section]); // Add remaining nodes
-}
-
-// What is the type of this section
-function sectionType(nodes, idx) {
-    var codeNodes = _.filter(nodes, {
-        type: 'code'
-    }).length;
-
-    if(codeNodes === 3 && (idx % 2) == 1) {
-        return 'exercise';
-    }
-
-    return 'normal';
-}
-
 // Render a section using our custom renderer
 function render(section, _options) {
     // Copy section
@@ -67,35 +39,9 @@ function render(section, _options) {
 function parsePage(src, options) {
     options = options || {};
 
-    // Lex file
-    var nodes = marked.lexer(src);
-
-    return _.chain(splitSections(nodes))
-    .map(function(section, idx) {
-        // Detect section type
-        section.type = sectionType(section, idx);
-        return section;
-    })
-    .filter(function(section) {
-        return !_.isEmpty(section);
-    })
-    .reduce(function(sections, section) {
-        var last = _.last(sections);
-
-        // Merge normal sections together
-        if(last && last.type === section.type && last.type === 'normal') {
-            last.push.apply(last, [{'type': 'hr'}].concat(section));
-        } else {
-            // Add to list of sections
-            sections.push(section);
-        }
-
-        return sections;
-    }, [])
+    // Lex if not already lexed
+    return (_.isArray(src) ? src : lex(src))
     .map(function(section) {
-        // Generate a uniqueId to identify this section in our code
-        var id = _.uniqueId('gitbook_');
-
         // Transform given type
         if(section.type === 'exercise') {
             var nonCodeNodes = _.reject(section, {
@@ -118,7 +64,7 @@ function parsePage(src, options) {
             var lang = validLangs ? langs[0] : null;
 
             return {
-                id: id,
+                id: section.id,
                 type: section.type,
                 content: render(nonCodeNodes),
                 lang: lang,
@@ -132,12 +78,11 @@ function parsePage(src, options) {
 
         // Render normal pages
         return {
-            id: id,
+            id: section.id,
             type: section.type,
             content: render(section, options)
         };
-    })
-    .value();
+    });
 }
 
 // Exports
diff --git a/package.json b/package.json
index ff792f4..58e24e5 100644
--- a/package.json
+++ b/package.json
@@ -7,6 +7,8 @@
         "q": "1.0.1",
         "lodash": "2.4.1",
         "marked": "0.3.2",
+        "marked-text-renderer": "0.0.1",
+        "lunr": "0.5.2",
         "swig": "1.3.2",
         "send": "0.2.0",
         "fstream-ignore": "0.0.7",