var Q = require("q"); var _ = require("lodash"); var path = require("path"); var lunr = require("lunr"); var parsers = require("gitbook-parsers"); var fs = require("./utils/fs"); var parseNavigation = require("./utils/navigation"); var parseProgress = require("./utils/progress"); var pageUtil = require("./utils/page"); var pathUtil = require("./utils/path"); var links = require("./utils/links"); var i18n = require("./utils/i18n"); var logger = require("./utils/logger"); var Configuration = require("./configuration"); var TemplateEngine = require("./template"); var PluginsList = require("./pluginslist"); var generators = require("./generators"); var Book = function(root, context, parent) { this.context = _.defaults(context || {}, { // Extend book configuration config: {}, // Log function log: function(msg) { process.stdout.write(msg); }, // Log level logLevel: "info" }); // Log this.log = logger(this.context.log, this.context.logLevel); // Root folder of the book this.root = path.resolve(root); // Parent book this.parent = parent; // Configuration this.config = new Configuration(this, this.context.config); Object.defineProperty(this, "options", { get: function () { return this.config.options; } }); // Template this.template = new TemplateEngine(this); // Summary this.summary = {}; this.navigation = []; // Glossary this.glossary = []; // Langs this.langs = []; // Sub-books this.books = []; // Files in the book this.files = []; // List of plugins this.plugins = new PluginsList(this); // Structure files this.summaryFile = null; this.glossaryFile = null; this.readmeFile = null; this.langsFile = null; // Search Index this.searchIndexEnabled = true; this.searchIndexSize = 0; this.searchIndex = lunr(function () { this.ref("url"); this.field("title", { boost: 10 }); this.field("body"); }); // Bind methods _.bindAll(this); }; // Return string representation Book.prototype.toString = function() { return "[Book "+this.root+"]"; }; // Initialize and parse the book: config, summary, glossary Book.prototype.parse = function() { var that = this; var multilingual = false; return this.parseConfig() .then(function() { return that.parsePlugins(); }) .then(function() { return that.parseLangs() .then(function() { multilingual = that.langs.length > 0; if (multilingual) that.log.info.ln("Parsing multilingual book, with", that.langs.length, "languages"); // Sub-books that inherit from the current book configuration that.books = _.map(that.langs, function(lang) { that.log.info.ln("Preparing language book", lang.lang); return new Book( path.join(that.root, lang.path), _.merge({}, that.context, { config: _.extend({}, that.options, { "output": path.join(that.options.output, lang.lang), "language": lang.lang }) }), that ); }); }); }) .then(function() { if (multilingual) return; return that.listAllFiles(); }) .then(function() { if (multilingual) return; return that.parseReadme(); }) .then(function() { if (multilingual) return; return that.parseSummary(); }) .then(function() { if (multilingual) return; return that.parseGlossary(); }) .then(function() { // Init sub-books return _.reduce(that.books, function(prev, book) { return prev.then(function() { return book.parse(); }); }, Q()); }) .thenResolve(this); }; // Generate the output Book.prototype.generate = function(generator) { var that = this; that.options.generator = generator || that.options.generator; that.log.info.ln("start generation with", that.options.generator, "generator"); return Q() // Clean output folder .then(function() { that.log.info("clean", that.options.generator, "generator"); return fs.clean(that.options.output) .progress(function(p) { that.log.debug.ln("remove", p.file, "("+p.i+"/"+p.count+")"); }) .then(function() { that.log.info.ok(); }); }) // Create generator .then(function() { var Generator = generators[generator]; if (!Generator) throw "Generator \""+that.options.generator+"\" doesn't exist"; generator = new Generator(that); return generator.prepare(); }) // Generate content .then(function() { if (that.isMultilingual()) { return that.generateMultiLingual(generator); } else { // Separate list of files into the different operations needed var ops = _.groupBy(that.files, function(file) { if (file[file.length -1] == "/") { return "directories"; } else if (_.contains(parsers.extensions, path.extname(file)) && that.navigation[file]) { return "content"; } else { return "files"; } }); return Q() // First, let"s create folder .then(function() { return _.reduce(ops.directories || [], function(prev, folder) { return prev.then(function() { that.log.debug.ln("transferring folder", folder); return Q(generator.transferFolder(folder)); }); }, Q()); }) // Then, let"s copy other files .then(function() { return Q.all(_.map(ops.files || [], function(file) { that.log.debug.ln("transferring file", file); return Q(generator.transferFile(file)); })); }) // Finally let"s generate content .then(function() { var nFiles = (ops.content || []).length; return _.reduce(ops.content || [], function(prev, file, i) { return prev.then(function() { var p = ((i*100)/nFiles).toFixed(0)+"%"; that.log.debug.ln("processing", file, p); return Q(generator.convertFile(file)) .fail(function(err) { // Transform error message to signal file throw that.normError(err, { fileName: file }); }); }); }, Q()); }); } }) // Finish generation .then(function() { return generator.callHook("finish:before"); }) .then(function() { return generator.finish(); }) .then(function() { return generator.callHook("finish"); }) .then(function() { that.log.info.ln("generation is finished"); }); }; // Generate the output for a multilingual book Book.prototype.generateMultiLingual = function() { var that = this; return Q() .then(function() { // Generate sub-books return _.reduce(that.books, function(prev, book) { return prev.then(function() { return book.generate(that.options.generator); }); }, Q()); }); }; // Extract files from ebook generated Book.prototype.generateFile = function(output, options) { var book = this; options = _.defaults(options || {}, { ebookFormat: path.extname(output).slice(1) }); output = output || path.resolve(book.root, "book."+options.ebookFormat); return fs.tmp.dir() .then(function(tmpDir) { book.setOutput(tmpDir); return book.generate(options.ebookFormat) .then(function() { var copyFile = function(lang) { var _outputFile = output; var _tmpDir = tmpDir; if (lang) { _outputFile = _outputFile.slice(0, -path.extname(_outputFile).length)+"_"+lang+path.extname(_outputFile); _tmpDir = path.join(_tmpDir, lang); } book.log.debug.ln("copy ebook to", _outputFile); return fs.copy( path.join(_tmpDir, "index."+options.ebookFormat), _outputFile ); }; // Multi-langs book return Q() .then(function() { if (book.isMultilingual()) { return Q.all( _.map(book.langs, function(lang) { return copyFile(lang.lang); }) ) .thenResolve(book.langs.length); } else { return copyFile().thenResolve(1); } }) .then(function(n) { book.log.info.ok(n+" file(s) generated"); return fs.remove(tmpDir); }); }); }); }; // Parse configuration Book.prototype.parseConfig = function() { var that = this; that.log.info("loading book configuration...."); return that.config.load() .then(function() { that.log.info.ok(); }); }; // Parse list of plugins Book.prototype.parsePlugins = function() { var that = this; // Load plugins return that.plugins.load(that.options.plugins) .then(function() { if (_.size(that.plugins.failed) > 0) return Q.reject(new Error("Error loading plugins: "+that.plugins.failed.join(",")+". Run \"gitbook install\" to install plugins from NPM.")); that.log.info.ok(that.plugins.count()+" plugins loaded"); that.log.debug.ln("normalize plugins list"); }); }; // Parse readme to extract defaults title and description Book.prototype.parseReadme = function() { var that = this; var structure = that.config.getStructure("readme"); that.log.debug.ln("start parsing readme:", structure); return that.findFile(structure) .then(function(readme) { if (!readme) throw "No README file"; if (!_.contains(that.files, readme.path)) throw "README file is ignored"; that.readmeFile = readme.path; that._defaultsStructure(that.readmeFile); that.log.debug.ln("readme located at", that.readmeFile); return that.template.renderFile(that.readmeFile) .then(function(content) { return readme.parser.readme(content) .fail(function(err) { throw that.normError(err, { name: err.name || "Readme Parse Error", fileName: that.readmeFile }); }); }); }) .then(function(readme) { that.options.title = that.options.title || readme.title; that.options.description = that.options.description || readme.description; }); }; // Parse langs to extract list of sub-books Book.prototype.parseLangs = function() { var that = this; var structure = that.config.getStructure("langs"); that.log.debug.ln("start parsing languages index:", structure); return that.findFile(structure) .then(function(langs) { if (!langs) return []; that.langsFile = langs.path; that._defaultsStructure(that.langsFile); that.log.debug.ln("languages index located at", that.langsFile); return that.template.renderFile(that.langsFile) .then(function(content) { return langs.parser.langs(content) .fail(function(err) { throw that.normError(err, { name: err.name || "Langs Parse Error", fileName: that.langsFile }); }); }); }) .then(function(langs) { that.langs = langs; }); }; // Parse summary to extract list of chapters Book.prototype.parseSummary = function() { var that = this; var structure = that.config.getStructure("summary"); that.log.debug.ln("start parsing summary:", structure); return that.findFile(structure) .then(function(summary) { if (!summary) throw "No SUMMARY file"; // Remove the summary from the list of files to parse that.summaryFile = summary.path; that._defaultsStructure(that.summaryFile); that.files = _.without(that.files, that.summaryFile); that.log.debug.ln("summary located at", that.summaryFile); return that.template.renderFile(that.summaryFile) .then(function(content) { return summary.parser.summary(content, { entryPoint: that.readmeFile, entryPointTitle: that.i18n("SUMMARY_INTRODUCTION"), files: that.files }) .fail(function(err) { throw that.normError(err, { name: err.name || "Summary Parse Error", fileName: that.summaryFile }); }); }); }) .then(function(summary) { that.summary = summary; that.navigation = parseNavigation(that.summary, that.files); }); }; // Parse glossary to extract terms Book.prototype.parseGlossary = function() { var that = this; var structure = that.config.getStructure("glossary"); that.log.debug.ln("start parsing glossary: ", structure); return that.findFile(structure) .then(function(glossary) { if (!glossary) return []; // Remove the glossary from the list of files to parse that.glossaryFile = glossary.path; that._defaultsStructure(that.glossaryFile); that.files = _.without(that.files, that.glossaryFile); that.log.debug.ln("glossary located at", that.glossaryFile); return that.template.renderFile(that.glossaryFile) .then(function(content) { return glossary.parser.glossary(content) .fail(function(err) { throw that.normError(err, { name: err.name || "Glossary Parse Error", fileName: that.glossaryFile }); }); }); }) .then(function(glossary) { that.glossary = glossary; }); }; // Parse a page Book.prototype.parsePage = function(filename, options) { var that = this, page = {}; options = _.defaults(options || {}, { // Transform svg images convertImages: false, // Interpolate before templating interpolateTemplate: _.identity, // Interpolate after templating interpolateContent: _.identity }); var interpolate = function(fn) { return Q(fn(page)) .then(function(_page) { page = _page || page; }); }; that.log.debug.ln("start parsing file", filename); var extension = path.extname(filename); var filetype = parsers.get(extension); if (!filetype) return Q.reject(new Error("Can't parse file: "+filename)); // Type of parser used page.type = filetype.name; // Path relative to book page.path = filename; // Path absolute in the system page.rawPath = path.resolve(that.root, filename); // Progress in the book page.progress = parseProgress(that.navigation, filename); that.log.debug.ln("render template", filename); // Read file content return that.readFile(page.path) .then(function(content) { page.content = content; return interpolate(options.interpolateTemplate); }) // Prepare page markup .then(function() { return filetype.page.prepare(page.content) .then(function(content) { page.content = content; }); }) // Generate template .then(function() { return that.template.renderPage(page); }) // Prepare and Parse markup .then(function(content) { page.content = content; that.log.debug.ln("use file parser", filetype.name, "for", filename); return filetype.page(page.content); }) // Post process sections .then(function(_page) { return _.reduce(_page.sections, function(prev, section) { return prev.then(function(_sections) { return that.template.postProcess(section.content || "") .then(function(content) { section.content = content; return _sections.concat([section]); }); }); }, Q([])); }) // Prepare html .then(function(_sections) { return pageUtil.normalize(_sections, { book: that, convertImages: options.convertImages, input: filename, navigation: that.navigation, base: path.dirname(filename) || "./", output: path.dirname(filename) || "./", glossary: that.glossary }); }) // Interpolate output .then(function(_sections) { page.sections = _sections; return interpolate(options.interpolateContent); }) .then(function() { that.indexPage(page); return page; }); }; // Find file that can be parsed with a specific filename Book.prototype.findFile = function(filename) { var that = this; return _.reduce(parsers.extensions, function(prev, ext) { return prev.then(function(output) { // Stop if already find a parser if (output) return output; var filepath = filename+ext; return that.fileExists(filepath) .then(function(exists) { if (!exists) return null; return { parser: parsers.get(ext), path: filepath }; }); }); }, Q(null)); }; // Check if a file exists in the book Book.prototype.fileExists = function(filename) { return fs.exists( this.resolve(filename) ); }; // Check if a file path is inside the book Book.prototype.fileIsInBook = function(filename) { return pathUtil.isInRoot(this.root, filename); }; // Read a file Book.prototype.readFile = function(filename) { return fs.readFile( this.resolve(filename), { encoding: "utf8" } ); }; // Return stat for a file Book.prototype.statFile = function(filename) { return fs.stat(this.resolve(filename)); }; // List all files in the book Book.prototype.listAllFiles = function() { var that = this; return fs.list(this.root, { ignoreFiles: [".ignore", ".gitignore", ".bookignore"], ignoreRules: [ // Skip Git stuff ".git/", ".gitignore", // Skip OS X meta data ".DS_Store", // Skip stuff installed by plugins "node_modules", // Skip book outputs "_book", "*.pdf", "*.epub", "*.mobi", // Skip config files ".ignore", ".bookignore", "book.json", ] }) .then(function(_files) { that.files = _files; }); }; // Return true if the book is a multilingual book Book.prototype.isMultilingual = function() { return this.books.length > 0; }; // Return root of the parent Book.prototype.parentRoot = function() { if (this.parent) return this.parent.parentRoot(); return this.root; }; // Return true if it"s a sub-book Book.prototype.isSubBook = function() { return !!this.parent; }; // Test if the file is the entry point Book.prototype.isEntryPoint = function(fp) { return fp == this.readmeFile; }; // Alias to book.config.get Book.prototype.getConfig = function(key, def) { return this.config.get(key, def); }; // Resolve a path in the book source // Enforce that the output path in the root folder Book.prototype.resolve = function() { return pathUtil.resolveInRoot.apply(null, [this.root].concat(_.toArray(arguments))); }; // Convert an abslute path into a relative path to this Book.prototype.relative = function(p) { return path.relative(this.root, p); }; // Normalize a path to .html and convert README -> index Book.prototype.contentPath = function(link) { if ( path.basename(link, path.extname(link)) == "README" || link == this.readmeFile ) { link = path.join(path.dirname(link), "index"+path.extname(link)); } link = links.changeExtension(link, ".html"); return link; }; // Normalize a link to .html and convert README -> index Book.prototype.contentLink = function(link) { return links.normalize(this.contentPath(link)); }; // Index a page into the search index Book.prototype.indexPage = function(page) { var nav = this.navigation[page.path]; if (!nav || !this.searchIndexEnabled) return; this.log.debug.ln("index page", page.path); // Extract text from the page var text = pageUtil.extractText(page.sections); // Limit size of index (to avoid #941) this.searchIndexSize = this.searchIndexSize + text.length; if (this.searchIndexSize > this.config.get('search.maxIndexSize')) { this.log.warn.ln("search index is too big, indexing is now disabled"); this.searchIndexEnabled = false; return; } this.searchIndex.add({ url: this.contentLink(page.path), title: nav.title, body: text }); }; // Default structure paths to an extension Book.prototype._defaultsStructure = function(filename) { var that = this; var extension = path.extname(filename); that.readmeFile = that.readmeFile || that.config.getStructure("readme")+extension; that.summaryFile = that.summaryFile || that.config.getStructure("summary")+extension; that.glossaryFile = that.glossaryFile || that.config.getStructure("glossary")+extension; that.langsFile = that.langsFile || that.config.getStructure("langs")+extension; }; // Change output path Book.prototype.setOutput = function(p) { var that = this; this.options.output = path.resolve(p); _.each(this.books, function(book) { book.setOutput(path.join(that.options.output, book.options.language)); }); }; // Translate a strign according to the book language Book.prototype.i18n = function() { var args = Array.prototype.slice.call(arguments); return i18n.__.apply({}, [this.config.normalizeLanguage()].concat(args)); }; // Normalize error Book.prototype.normError = function(err, opts, defs) { if (_.isString(err)) err = new Error(err); // Extend err _.extend(err, opts || {}); _.defaults(err, defs || {}); err.lineNumber = err.lineNumber || err.lineno; err.columnNumber = err.columnNumber || err.colno; err.toString = function() { var attributes = []; if (this.fileName) attributes.push("In file \""+this.fileName+"\""); if (this.lineNumber) attributes.push("Line "+this.lineNumber); if (this.columnNumber) attributes.push("Column "+this.columnNumber); return (this.name || "Error")+": "+this.message+((attributes.length > 0)? " ("+attributes.join(", ")+")" : ""); }; return err; }; // Init and return a book Book.init = function(root, opts) { var book = new Book(root, opts); var extensionToUse = ".md"; var chaptersPaths = function(chapters) { return _.reduce(chapters || [], function(accu, chapter) { var o = { title: chapter.title }; if (chapter.path) o.path = chapter.path; return accu.concat( [o].concat(chaptersPaths(chapter.articles)) ); }, []); }; book.log.info.ln("init book at", root); return fs.mkdirp(root) .then(function() { book.log.info.ln("detect structure from SUMMARY (if it exists)"); return book.parseSummary(); }) .fail(function() { return Q(); }) .then(function() { var summary = book.summaryFile || "SUMMARY.md"; var chapters = book.summary.chapters || []; extensionToUse = path.extname(summary); if (chapters.length === 0) { chapters = [ { title: "Summary", path: "SUMMARY"+extensionToUse }, { title: "Introduction", path: "README"+extensionToUse } ]; } return Q(chaptersPaths(chapters)); }) .then(function(chapters) { // Create files that don"t exist return Q.all(_.map(chapters, function(chapter) { if (!chapter.path) return Q(); var absolutePath = path.resolve(book.root, chapter.path); return fs.exists(absolutePath) .then(function(exists) { if(exists) { book.log.info.ln("found", chapter.path); return; } else { book.log.info.ln("create", chapter.path); } return fs.mkdirp(path.dirname(absolutePath)) .then(function() { return fs.writeFile(absolutePath, "# "+chapter.title+"\n"); }); }); })); }) .then(function() { book.log.info.ln("initialization is finished"); }); }; module.exports= Book;