Add assets inliner modifier for HTML

17 files changed, 364 insertions, 41 deletions

diff --git a/lib/output/ebook/index.js b/lib/output/ebook/index.js
new file mode 100644
index 0000000..46a94e3
--- /dev/null
+++ b/lib/output/ebook/index.js
@@ -0,0 +1,5 @@
+
+module.exports = {
+
+
+};
diff --git a/lib/output/ebook/onPage.js b/lib/output/ebook/onPage.js
new file mode 100644
index 0000000..ab15133
--- /dev/null
+++ b/lib/output/ebook/onPage.js
@@ -0,0 +1,24 @@
+var website = require('../website');
+var Modifier = require('../modifier');
+
+/**
+    Write a page for ebook output
+
+    @param {Output} output
+    @param {Page} page
+*/
+function onPage(output, page) {
+    var options = output.getOptions();
+
+    // Inline assets
+    return Modifier.modifyHTML(page, [
+        Modifier.inlineAssets(options.get('root'))
+    ])
+
+    // Write page using website generator
+    .then(function(resultPage) {
+        return website.onPage(output, resultPage);
+    });
+}
+
+module.exports = onPage;
diff --git a/lib/output/index.js b/lib/output/index.js
index 45bdf65..d353490 100644
--- a/lib/output/index.js
+++ b/lib/output/index.js
@@ -1,4 +1,5 @@
 module.exports = {
     generate:           require('./generateBook'),
-    JSONGenerator:      require('./json')
+    JSONGenerator:      require('./json'),
+    EbookGenerator:     require('./ebook')
 };
diff --git a/lib/output/json/onPage.js b/lib/output/json/onPage.js
index a84e297..1143aaa 100644
--- a/lib/output/json/onPage.js
+++ b/lib/output/json/onPage.js
@@ -10,7 +10,8 @@ var Writer = require('../writer');
 */
 function onPage(output, page) {
     return Modifier.modifyHTML(page, [
-        Modifier.HTML.resolveLinks()
+        Modifier.addHeadingId,
+        Modifier.resolveLinks
     ])
     .then(function(resultPage) {
         // Generate the JSON
diff --git a/lib/output/modifiers/__tests__/addHeadingId.js b/lib/output/modifiers/__tests__/addHeadingId.js
new file mode 100644
index 0000000..7277440
--- /dev/null
+++ b/lib/output/modifiers/__tests__/addHeadingId.js
@@ -0,0 +1,29 @@
+jest.autoMockOff();
+
+var cheerio = require('cheerio');
+
+describe('addHeadingId', function() {
+    var addHeadingId = require('../addHeadingId');
+
+    pit('should add an ID if none', function() {
+        var $ = cheerio.load('<h1>Hello World</h1><h2>Cool !!</h2>');
+
+        return addHeadingId($)
+        .then(function() {
+            var html = $.html();
+            expect(html).toBe('<h1 id="hello-world">Hello World</h1><h2 id="cool-">Cool !!</h2>');
+        });
+    });
+
+    pit('should not change existing IDs', function() {
+        var $ = cheerio.load('<h1 id="awesome">Hello World</h1>');
+
+        return addHeadingId($)
+        .then(function() {
+            var html = $.html();
+            expect(html).toBe('<h1 id="awesome">Hello World</h1>');
+        });
+    });
+});
+
+
diff --git a/lib/output/modifiers/addHeadingId.js b/lib/output/modifiers/addHeadingId.js
index 751f4b8..e2e2720 100644
--- a/lib/output/modifiers/addHeadingId.js
+++ b/lib/output/modifiers/addHeadingId.js
@@ -1,9 +1,23 @@
 var slug = require('github-slugid');
-var HTMLModifier = require('./html');
+var editHTMLElement = require('./editHTMLElement');
 
-var addHeadingID = HTMLModifier('h1,h2,h3,h4,h5,h6', function(heading) {
+/**
+    Add ID to an heading
+
+    @param {HTMLElement} heading
+*/
+function addId(heading) {
     if (heading.attr('id')) return;
     heading.attr('id', slug(heading.text()));
-});
+}
+
+/**
+    Add ID to all headings
+
+    @param {HTMLDom} $
+*/
+function addHeadingId($) {
+    return editHTMLElement($, 'h1,h2,h3,h4,h5,h6', addId);
+}
 
-module.exports = addHeadingID;
+module.exports = addHeadingId;
diff --git a/lib/output/modifiers/annotateText.js b/lib/output/modifiers/annotateText.js
new file mode 100644
index 0000000..2832f91
--- /dev/null
+++ b/lib/output/modifiers/annotateText.js
@@ -0,0 +1,94 @@
+var escape = require('escape-html');
+
+// Selector to ignore
+var ANNOTATION_IGNORE = '.no-glossary,code,pre,a,script,h1,h2,h3,h4,h5,h6';
+
+function pregQuote( str ) {
+    return (str+'').replace(/([\\\.\+\*\?\[\^\]\$\(\)\{\}\=\!\<\>\|\:])/g, '\\$1');
+}
+
+function replaceText($, el, search, replace, text_only ) {
+    return $(el).each(function(){
+        var node = this.firstChild,
+            val,
+            new_val,
+
+            // Elements to be removed at the end.
+            remove = [];
+
+        // Only continue if firstChild exists.
+        if ( node ) {
+
+            // Loop over all childNodes.
+            while (node) {
+
+                // Only process text nodes.
+                if ( node.nodeType === 3 ) {
+
+                    // The original node value.
+                    val = node.nodeValue;
+
+                    // The new value.
+                    new_val = val.replace( search, replace );
+
+                    // Only replace text if the new value is actually different!
+                    if ( new_val !== val ) {
+
+                        if ( !text_only && /</.test( new_val ) ) {
+                            // The new value contains HTML, set it in a slower but far more
+                            // robust way.
+                            $(node).before( new_val );
+
+                            // Don't remove the node yet, or the loop will lose its place.
+                            remove.push( node );
+                        } else {
+                            // The new value contains no HTML, so it can be set in this
+                            // very fast, simple way.
+                            node.nodeValue = new_val;
+                        }
+                    }
+                }
+
+                node = node.nextSibling;
+            }
+        }
+
+        // Time to remove those elements!
+        if (remove.length) $(remove).remove();
+    });
+}
+
+/**
+    Annotate text using a list of GlossaryEntry
+
+    @param {List<GlossaryEntry>}
+    @param {HTMLDom} $
+*/
+function annotateText(entries, $) {
+    entries.forEach(function(entry) {
+        var entryId = entry.getID();
+        var name = entry.getName();
+        var description = entry.getDescription();
+
+        var searchRegex =  new RegExp( '\\b(' + pregQuote(name.toLowerCase()) + ')\\b' , 'gi' );
+
+        $('*').each(function() {
+            var $this = $(this);
+
+            if (
+                $this.is(ANNOTATION_IGNORE) ||
+                $this.parents(ANNOTATION_IGNORE).length > 0
+            ) return;
+
+            replaceText($, this, searchRegex, function(match) {
+                return '<a href="/GLOSSARY.html#' + entryId + '" '
+                    + 'class="glossary-term" title="' + escape(description) + '">'
+                    + match
+                    + '</a>';
+            });
+        });
+
+    });
+}
+
+module.exports = annotateText;
diff --git a/lib/output/modifiers/editHTMLElement.js b/lib/output/modifiers/editHTMLElement.js
new file mode 100644
index 0000000..9897dcc
--- /dev/null
+++ b/lib/output/modifiers/editHTMLElement.js
@@ -0,0 +1,15 @@
+var Promise = require('../../utils/promise');
+
+/**
+    Edit all elements matching a selector
+*/
+function editHTMLElement($, selector, fn) {
+    var $elements = $(selector);
+
+    return Promise.forEach($elements, function(el) {
+        var $el = $(el);
+        fn($el);
+    });
+}
+
+module.exports = editHTMLElement;
diff --git a/lib/output/modifiers/fetchRemoteImages.js b/lib/output/modifiers/fetchRemoteImages.js
new file mode 100644
index 0000000..792bbb6
--- /dev/null
+++ b/lib/output/modifiers/fetchRemoteImages.js
@@ -0,0 +1,38 @@
+var path = require('path');
+var crc = require('crc');
+
+var editHTMLElement = require('./editHTMLElement');
+var fs = require('../../utils/fs');
+var location = require('../../utils/location');
+
+/**
+    Fetch all remote images
+
+    @param {String} rootFolder
+    @param {HTMLDom} $
+    @return {Promise}
+*/
+function fetchRemoteImages(rootFolder, $) {
+    return editHTMLElement($, 'img', function($img) {
+        var src = $img.attr('src');
+        var extension = path.extname(src);
+
+        if (!location.isExternal(src)) {
+            return;
+        }
+
+        // We avoid generating twice the same PNG
+        var hash = crc.crc32(src).toString(16);
+        var fileName = hash + extension;
+        var filePath = path.join(rootFolder, fileName);
+
+        return fs.assertFile(filePath, function() {
+            return fs.download(src, filePath);
+        })
+        .then(function() {
+            $img.replaceWith('<img src="/' + fileName + '" />');
+        });
+    });
+}
+
+module.exports = fetchRemoteImages;
diff --git a/lib/output/modifiers/htmlTransform.js b/lib/output/modifiers/htmlTransform.js
deleted file mode 100644
index 528b08d..0000000
--- a/lib/output/modifiers/htmlTransform.js
+++ /dev/null
@@ -1,16 +0,0 @@
-var Promise = require('../../utils/promise');
-
-/**
-
-
-*/
-function transformTags() {
-    var $elements = $(query);
-
-    return Promise.serie($elements, function(el) {
-        var $el = that.$(el);
-        return fn.call(that, $el);
-    });
-}
-
-module.exports = transformTags;
diff --git a/lib/output/modifiers/index.js b/lib/output/modifiers/index.js
index ced8716..75b5927 100644
--- a/lib/output/modifiers/index.js
+++ b/lib/output/modifiers/index.js
@@ -1,4 +1,13 @@
 
 module.exports = {
-    modifyHTML:     require('./modifyHTML')
+    modifyHTML:         require('./modifyHTML'),
+    inlineAssets:       require('./inlineAssets'),
+
+    // HTML transformations
+    addHeadingId:       require('./addHeadingId'),
+    svgToImg:           require('./svgToImg'),
+    fetchRemoteImages:  require('./fetchRemoteImages'),
+    svgToPng:           require('./svgToPng'),
+    resolveLinks:       require('./resolveLinks'),
+    annotateText:       require('./annotateText')
 };
diff --git a/lib/output/modifiers/inlineAssets.js b/lib/output/modifiers/inlineAssets.js
index 190a945..044953b 100644
--- a/lib/output/modifiers/inlineAssets.js
+++ b/lib/output/modifiers/inlineAssets.js
@@ -1,11 +1,21 @@
+var svgToImg = require('./svgToImg');
+var svgToPng = require('./svgToPng');
+var fetchRemoteImages = require('./fetchRemoteImages');
 
+var Promise = require('../../utils/promise');
 
 /**
+    Inline all assets in a page
 
+    @param {String} rootFolder
 */
-function inlineAssets() {
-
-
+function inlineAssets(rootFolder) {
+    return function($) {
+        return Promise()
+        .then(fetchRemoteImages.bind(null, rootFolder))
+        .then(svgToImg.bind(null, rootFolder))
+        .then(svgToPng.bind(null, rootFolder));
+    };
 }
 
 module.exports = inlineAssets;
diff --git a/lib/output/modifiers/modifyHTML.js b/lib/output/modifiers/modifyHTML.js
index c1dad74..d46a44f 100644
--- a/lib/output/modifiers/modifyHTML.js
+++ b/lib/output/modifiers/modifyHTML.js
@@ -1,15 +1,24 @@
-
+var cheerio = require('cheerio');
+var Promise = require('../../utils/promise');
 
 /**
     Apply a list of operations to a page and
     output the new page.
 
     @param {Page}
-    @param {List<Transformation>}
+    @param {List|Array<Transformation>}
+    @return {Promise<Page>}
 */
 function modifyHTML(page, operations) {
+    var html = page.getContent();
+    var $ = cheerio.load(html);
 
+    return Promise.forEach(operations, function(op) {
+        op($);
+    })
+    .then(function(resultHTML) {
+        return page.set('content', resultHTML);
+    });
 }
 
-
 module.exports = modifyHTML;
diff --git a/lib/output/modifiers/resolveLinks.js b/lib/output/modifiers/resolveLinks.js
new file mode 100644
index 0000000..5c2514f
--- /dev/null
+++ b/lib/output/modifiers/resolveLinks.js
@@ -0,0 +1,13 @@
+
+
+/**
+    Resolve all HTML links
+
+    @param {String}
+    @param {HTMLDom} $
+*/
+function resolveLinks() {
+
+}
+
+module.exports = resolveLinks;
diff --git a/lib/output/modifiers/svgToImg.js b/lib/output/modifiers/svgToImg.js
index b36770a..d088e3e 100644
--- a/lib/output/modifiers/svgToImg.js
+++ b/lib/output/modifiers/svgToImg.js
@@ -1,7 +1,17 @@
-var cheerio = require('cheerio');
+var path = require('path');
 var domSerializer = require('dom-serializer');
 
-// Render a cheerio DOM as html
+var editHTMLElement = require('./editHTMLElement');
+var fs = require('../../utils/fs');
+
+/**
+    Render a cheerio DOM as html
+
+    @param {HTMLDom} $
+    @param {HTMLElement} dom
+    @param {Object}
+    @return {String}
+*/
 function renderDOM($, dom, options) {
     if (!dom && $._root && $._root.children) {
         dom = $._root.children;
@@ -11,18 +21,30 @@ function renderDOM($, dom, options) {
 }
 
 /**
+    Replace SVG tag by IMG
 
+    @param {String} baseFolder
+    @param {HTMLDom} $
 */
-var svgToImg = HTMLModifier('svg', function($svg, $) {
-    var content = '<?xml version="1.0" encoding="UTF-8"?>' + renderDOM($, $svg);
-
-
-
-});
-
-function svgToImg(page) {
-    var $ = cheerio.load(page.content);
-
+function svgToImg(baseFolder, $) {
+    return editHTMLElement($, 'svg', function($svg) {
+        var content = '<?xml version="1.0" encoding="UTF-8"?>' +
+            renderDOM($, $svg);
+
+        // Generate a filename
+        return fs.uniqueFilename(baseFolder, 'image.svg')
+        .then(function(fileName) {
+            var filePath = path.join(baseFolder, fileName);
+
+            // Write the svg to the file
+            return fs.writeFile(filePath, content, 'utf8')
+
+            // Return as image
+            .then(function() {
+                $svg.replaceWith('<img src="/' + fileName + '" />');
+            });
+        });
+    });
 }
 
 module.exports = svgToImg;
diff --git a/lib/output/modifiers/svgToPng.js b/lib/output/modifiers/svgToPng.js
new file mode 100644
index 0000000..4f8644a
--- /dev/null
+++ b/lib/output/modifiers/svgToPng.js
@@ -0,0 +1,38 @@
+var crc = require('crc');
+var path = require('path');
+
+var imagesUtil = require('../../utils/images');
+var fs = require('../../utils/fs');
+
+var editHTMLElement = require('./editHTMLElement');
+
+/**
+    Convert all SVG images to PNG
+
+    @param {String} rootFolder
+    @param {HTMLDom} $
+    @return {Promise}
+*/
+function svgToPng(rootFolder, $) {
+    return editHTMLElement($, 'img', function($img) {
+        var src = $img.attr('src');
+        if (path.extname(src) !== '.svg') {
+            return;
+        }
+
+        // We avoid generating twice the same PNG
+        var hash = crc.crc32(src).toString(16);
+        var fileName = hash + '.png';
+        var filePath = path.join(rootFolder, fileName);
+
+        return fs.assertFile(filePath, function() {
+            return imagesUtil.convertSVGToPNG(src, filePath);
+        })
+        .then(function() {
+            $img.replaceWith('<img src="/' + fileName + '" />');
+        });
+    });
+}
+
+
+module.exports = svgToPng;
diff --git a/lib/utils/fs.js b/lib/utils/fs.js
index 42fd3c6..96a0e49 100644
--- a/lib/utils/fs.js
+++ b/lib/utils/fs.js
@@ -97,12 +97,29 @@ function rmDir(base) {
     });
 }
 
+/**
+    Assert a file, if it doesn't exist, call "generator"
+
+    @param {String} filePath
+    @param {Function} generator
+    @return {Promise}
+*/
+function assertFile(filePath, generator) {
+    return fileExists(filePath)
+    .then(function(exists) {
+        if (exists) return;
+
+        generator();
+    });
+}
+
 module.exports = {
     exists: fileExists,
     existsSync: fs.existsSync,
     mkdirp: Promise.nfbind(mkdirp),
     readFile: Promise.nfbind(fs.readFile),
     writeFile: Promise.nfbind(fs.writeFile),
+    assertFile: assertFile,
     stat: Promise.nfbind(fs.stat),
     statSync: fs.statSync,
     readdir: Promise.nfbind(fs.readdir),