// Copyright (c) 2018 Gordon Williams, Pur3 Ltd. See the file LICENSE for copying permission.

// Generate keywords.js, and convert Markdown into HTML

// needs marked + jsdoc + tern + acorn

console.log("Espruino documentation builder");

console.log("--------------------------------");

var OFFLINE = true;

var PROCESS_THUMBNAILS = true; // create thumbnails

var PROCESS_INFERENCE = true; // type inference

var PROCESS_IMAGE_RESIZE = true; // resize images

for (var i=2;i<process.argv.length;i++) {

var a = process.argv[i];

if (a=="--website") OFFLINE = false;

else if (a=="--no-thumbs") PROCESS_THUMBNAILS = false;

else if (a=="--no-infer") PROCESS_INFERENCE = false;

else if (a=="--no-resize") PROCESS_IMAGE_RESIZE = false;

else {

console.log("Usage:");

console.log(" build.js # Build for offline viewing");

console.log(" build.js --website # Build to be used on Espruino website");

console.log("");

console.log(" --no-thumbs Skip creating thumbnails");

console.log(" --no-infer Skip type inference on code (for reference links)");

console.log(" --no-resize Skip image resizing");

process.exit(1);

}

}

var fs = require('fs');

var path = require('path');

if (fs.existsSync==undefined) fs.existsSync = path.existsSync;

var common = require("./common.js");

var child_process = require('child_process');

var BASEDIR = path.resolve(__dirname, "..");

var HTML_DIR = BASEDIR+"/html/";

var IMAGE_DIR = "refimages/";

var THUMB_WIDTH = 180;

var THUMB_HEIGHT = 140;

var ESPRUINO_DIR = path.resolve(BASEDIR, "../Espruino");

var FUNCTION_KEYWORD_FILE = path.resolve(BASEDIR, "../Espruino/function_keywords.js");

var KEYWORD_JS_FILE = path.resolve(HTML_DIR, "keywords.js");

var warnings = [];

function WARNING(s) {

console.log("WARNING: "+s);

warnings.push(s);

}

var marked = require('marked');

// Set default options except highlight which has no default

var markedOptions = {

gfm: true, // github markdown

tables: true,

breaks: false,

pedantic: false,

sanitize: false,

smartLists: true,

smartypants: false,

langPrefix: 'lang-'

};

if (OFFLINE) {

var hljs;

try {

hljs = require('highlight.js')

markedOptions.highlight = function (code) {

return hljs.highlightAuto(code).value;

};

} catch(e) {

console.log("======================================================");

console.log(" highlight.js not installed - not syntax highlighting");

console.log("======================================================");

var d = Date.now()+2000;

while (Date.now()<d);

}

}

marked.setOptions(markedOptions);

// Work out file ordering

var fileOrdering = {};

if (fs.existsSync(BASEDIR+"/ordering.txt")) {

fs.readFileSync(BASEDIR+"/ordering.txt").toString().split("\n").map(function(line, linenumber) {

line = line.substr(line.indexOf(" ")+1);

fileOrdering[line] = linenumber;

});

} else {

WARNING("No 'ordering.txt' - not ordering tutorials");

console.log(' - Generate with: git ls-tree -r --name-only HEAD | xargs -I{} git log -1 --format="%at {}" -- {} | sort > ordering.txt');

}

// Get list of files that have been renamed

var renamedFiles = {};

if (fs.existsSync(BASEDIR+"/RENAMED.csv")) {

fs.readFileSync(BASEDIR+"/RENAMED.csv").toString().split("\n").map(function(line) {

if (line[0]=="#") return;

var csv = line.trim().split(",");

if (csv=="") return;

if (csv.length==2) {

renamedFiles[csv[0]] = csv[1];

} else WARNING("RENAMED.csv: Badly formatted line "+JSON.stringify(line));

});

} else {

console.log("ERROR: No 'RENAMED.csv'");

process.exit(1);

}

if (OFFLINE) {

// if offline, copy the offline.css file in

fs.createReadStream(path.resolve(BASEDIR, "bin/offline.css")).pipe(

fs.createWriteStream(path.resolve(HTML_DIR, "offline.css")));

}

function createSafeFilename(filename) {

}

function convertHTML(str) {

}

var fileDates = {};

// MD files

var markdownFiles = common.getMarkdown(BASEDIR).map(function (file) {

var p = path.relative(BASEDIR, file);

fileDates[p] = new Date(fs.statSync(p).ctime).getTime();

return p;

});

// JS files

var exampleDir = path.resolve(BASEDIR,"examples");

var exampleFiles = fs.readdirSync(exampleDir).filter(function(file) {

return file.substr(-3) == ".js";

}).map(function(file) {

var p = path.relative(BASEDIR, path.resolve(exampleDir,file));

fileDates[p] = new Date(fs.statSync(p).ctime).getTime();

return p;

});

var preloadedFiles = {};

var fileTitles = {};

// Create thumbnail images

var markdownThumbs = {};

if (PROCESS_THUMBNAILS) markdownFiles.concat(exampleFiles).forEach(function(filename) {

//console.log(filename);

var baseName = filename.slice(0,-3);

var sourceImage;

if (fs.existsSync(baseName+".thumb.png"))

sourceImage = baseName+".thumb.png";

else if (fs.existsSync(baseName+".thumb.jpg"))

sourceImage = baseName+".thumb.jpg";

else if (fs.existsSync(baseName+".thumb.svg"))

sourceImage = baseName+".thumb.svg";

else if (fs.existsSync(baseName+".png"))

sourceImage = baseName+".png";

else if (fs.existsSync(baseName+".jpg"))

sourceImage = baseName+".jpg";

if (!sourceImage) {

var contents = fs.readFileSync(filename).toString();

var match = contents.match(/!\[[^\]]*\]\(([^\)]*)\)/);

if (match)

sourceImage = filename.substr(0,filename.lastIndexOf("/")+1)+match[1];

}

if (sourceImage) {

var dstImage = IMAGE_DIR+createSafeFilename(baseName)+"_thumb.png";

var radius = 6;

var roundcorners = `\\( +clone -crop ${radius}x${radius}+0+0 -fill white -colorize 100% -draw 'fill black circle ${radius-1},${radius-1} ${radius-1},0' -background White -alpha shape \\( +clone -flip \\) \\( +clone -flop \\) \\( +clone -flip \\) \\) -flatten`;

var cmd = `convert "${sourceImage}" -resize ${THUMB_WIDTH}x${THUMB_HEIGHT} +repage ${roundcorners} -gravity South -extent ${THUMB_WIDTH}x -strip -define png:include-chunk=none "${path.resolve(HTML_DIR, dstImage)}"`;

//console.log(cmd);

child_process.exec(cmd);

console.log("THUMBNAIL "+filename+" --> "+dstImage);

markdownThumbs[filename] = dstImage;

} else {

console.log("NO THUMBNAIL FOR "+filename);

}

});

// Add 'unknown' thumbnail

{

let dstImage = IMAGE_DIR+"no_thumb.png";

child_process.exec(`convert files/blank_thumb.png -resize ${THUMB_WIDTH}x${THUMB_HEIGHT} +repage -gravity Center -extent ${THUMB_WIDTH}x${THUMB_HEIGHT} -strip -define png:include-chunk=none "${path.resolve(HTML_DIR, dstImage)}"`);

markdownThumbs[""] = dstImage;

}

function addToList(keywords, k, fileInfo) {

}

function grabInfo(markdownFiles, preloadedFiles) {

}

// Create a keywords structure that can be used for searching the website

function createKeywordsJS(keywords) {

}

function handleImages(file, contents) {

}

// Now handle our simple 'example' files

exampleFiles.forEach(function(exampleFile) {

console.log("Example File "+exampleFile);

var contents = fs.readFileSync(exampleFile).toString();

var slashStar = contents.indexOf("/*");

var starSlash = contents.indexOf("*/",slashStar);

if (slashStar>=0 && starSlash>=0) {

var markdown = contents.substr(slashStar+2, starSlash-(3+slashStar)).trim();

var newFile = "";

if (markdown.indexOf("Copyright")<0)

newFile += "<!--- Copyright (c) 2018 Gordon Williams, Pur3 Ltd. See the file LICENSE for copying permission. -->\n";

// newFile += exampleFile+"\n";

// newFile += "====================================\n";

// newFile += "\n";

newFile += markdown + "\n";

newFile += "\n\n";

newFile += "Source Code\n";

newFile += "-----------\n\n";

newFile += "```\n";

newFile += contents.substr(starSlash+2).trim()+"\n";

newFile += "```\n";

// add file

markdownFiles.push(exampleFile);

preloadedFiles[exampleFile] = newFile;

} else WARNING(exampleFile+": no comment block at the start");

});

//console.log(markdownFiles);

var fileInfo = grabInfo(markdownFiles, preloadedFiles);

//console.log(fileInfo.keywords);

grabWebsiteKeywords(fileInfo.keywords);

htmlFiles = {}; // filename -> file on disk with full path (for copy/etc)

htmlLinks = {}; // filename -> actual URL link

knownPageLinks = []; // pages that we know exist on this site

// Go through all referenced files on the main Espruino site

// TODO: in offline mode we should detect these links and go direct

"Order Search Reference Download binaries".split(" ").forEach(function(file) {

var htmlLink = file + (OFFLINE?".html":"");

knownPageLinks.push((OFFLINE?"":"/")+htmlLink);

});

// Add JS files to our known files list

["devices","modules","boards"].forEach(function(moduledir) {

fs.readdirSync(path.resolve(BASEDIR,moduledir)).filter(function(file) {

return file.substr(-3) == ".js";

}).forEach(function(file) {

knownPageLinks.push("/modules/"+file);

});

});

// Go through all Markdown files

markdownFiles.forEach(function (file) {

var htmlFile = file.substring(file.lastIndexOf("/")+1);

htmlFile = htmlFile.replace(/ /g,"+");

htmlFile = htmlFile.substring(0,htmlFile.lastIndexOf("."));

htmlFiles[file] = HTML_DIR+htmlFile+".html";

var htmlLink = htmlFile + (OFFLINE?".html":"");

htmlLinks[file] = htmlLink;

knownPageLinks.push((OFFLINE?"":"/")+htmlLink);

});

//console.log(JSON.stringify(knownPageLinks,null,2));process.exit(1);

// For files that have been renamed, create placeholder files that redirect

Object.keys(renamedFiles).forEach(function(originalFile) {

var newFile = renamedFiles[originalFile];

var htmlFile = HTML_DIR+originalFile.replace(/ /g,"+")+".html";

var html = `<!---REDIRECT ${newFile}--->

<p>This page has moved to <a href="${newFile}">${newFile}</a></p>

<script>window.location = "${newFile}";</script>`;

fs.writeFileSync(htmlFile, html);

});

// ---------------------------------------------- Inference code

/* Works out which of Espruino's built-in commands are used in which

files */

//Load tern inference thingybob

var infer = require("tern/lib/infer");

var defs = [JSON.parse(fs.readFileSync(path.resolve(BASEDIR, "bin/espruino.json")))];

// Our list of Reference URLs that are used

var urls = {};

function inferFile(filename, fileContents, baseLineNumber) {

}

function inferMarkdownFile(filename, fileContents) {

}

function getThumbLinkHTML(a) {

var thumb = (a.path in markdownThumbs) ? markdownThumbs[a.path] : markdownThumbs[""];

return `<a class="thumblink" href="${htmlLinks[a.path]}" title="${convertHTML(a.title)}"><img src="${thumb}" alt="${convertHTML(a.title)}" title="${convertHTML(a.title)}"></img><span>${convertHTML(a.title)}</span></a>`

}

// -------------------------------------------------------------

markdownFiles.forEach(function (file) {

var contents = preloadedFiles[file] ? preloadedFiles[file] : fs.readFileSync(file).toString();

if (file.substr(-3)==".md") {

var contentLines = contents.split("\n");

if (contentLines[3].trim()!="" || contentLines[4].trim().substr(0,6)!="<span " || contentLines[5].trim()!="") {

console.log("=============");

console.log(JSON.stringify(contentLines[3]));

console.log(JSON.stringify(contentLines[4]));

console.log(JSON.stringify(contentLines[5]));

console.log("=============");

throw new Error("Expecting to find warning comment in "+file+", but didn't.");

}

contentLines.splice(4,2); // remove comment line

contents = contentLines.join("\n");

}

//console.log(file,contents.length);

// Check over images... 

contents = handleImages(file, contents);

// replace simple links

contents = contents.replace(/\[\[http[s]?:\/\/youtu.be\/([a-zA-Z0-9\-_ ]+)\]\]/g,

'<iframe allowfullscreen="" frameborder="0" height="450" src="https://www.youtube.com/embed/$1" width="800"></iframe>'); // youtube

contents = contents.replace(/\[\[([a-zA-Z0-9_\- ]+).js\]\]/g,"[$1](/modules/$1.js) ([About Modules](/Modules))");

contents = contents.replace(/\[\[([a-zA-Z0-9_\- ]+)\]\]/g,"[$1](/$1)");

contents = contents.replace(/!\[\]\((Font[a-zA-Z0-9_]+\.js)\)/g, function(match, filename) {

console.log("Font Preview: "+JSON.stringify(filename));

var js = "";

if (filename.startsWith("Font") && filename.endsWith(".js")) {

var fontName = filename.slice(4,-3);

js = '<br /> `require("Font'+fontName+'").add(Graphics);` <br /> `g.setFont("'+fontName+'");`';

}

return "!["+filename+"]("+require("./fontpreview.js").getFontPreview("../modules/"+filename)+")"+js;

});

for (var i=0;i<3;i++) // cope with multiple spaces in links (nasty!)

contents = contents.replace(/(\[.+\]\([^) ]+) ([^)]+\))/g,"$1+$2"); // spaces in links

// Hacks for 'broken' markdown parsing

contents = contents.replace(/\n\n```([^\n]+)```\n\n/g,"\n\n```\n$1\n```\n\n"); // turn in-line code on its own into separate paragraph

contents = contents.replace(/```([^ \n][^\n]+)```/g,"``` $1 ```"); // need spaces after ```

// Hide keywords

contents = contents.replace(/\n(\* KEYWORDS: .*)/g, "<!---\n$1\n--->");

contents = contents.replace(/\n(\* USES: .*)/g, "<!---\n$1\n--->");

// TODO - 'Tutorial 2' -> 'Tutorial+2', recognize pages that are references in docs themselves

var contentLines = contents.split("\n");

var appendMatching = function(kwName, infoList, ifNone) {

};

for (i in contentLines) {

var match;

// try and handle module documentation

match = contentLines[i].match(/^\* APPEND_JSDOC: (.*)/);

if (match!=null) {

var jsfilename = file.substr(0, file.lastIndexOf("/")+1) + match[1];

var js = fs.readFileSync(jsfilename).toString();

console.log("APPEND_JSDOC "+jsfilename);

var doc = common.getJSDocumentation(js);

/* setting the language to Java lets it be highlighted correctly

while not adding the 'send to Espruino' icon */

contentLines[i] = "```Java\n"+doc+"```\n";

}

// handle buy links

match = contentLines[i].match(/^\* BUYFROM:(.*)/);

if (match!=null) {

var info = match[1].trim().split(",");

var html = `<div style="float:right;z-index:-1" class="panel panel-info" >

<div class="panel-heading"><h4 style="text-align:center;margin:0px;">Buy Now</h4></div>

<div class="panel-body center-block">

<h1 style="margin-top:0px"><a name="buy"></a><small>From</small> ${info[0]}</h1>

`;

if (info[1]) html += `<p><small>Or ${info[1]} in volume</small></p>`;

for (var j=2;j<info.length;j++) {

var link = info[j].split("|");

var name = link[0].includes("shop.espruino.com") ? "Espruino Shop" : "&#x1F30E; Distributors";

var btnType = link[0].includes("shop.espruino.com") ? "btn-primary" : "btn-default";

if (link[1]) name += " ("+link[1]+")";

html += ` <a role="button" class="btn ${btnType}" style="width:100%" href="${link[0]}">${name}</a>${(j<info.length-1)?"<br/>":""}\n`;

}

html += ` </div>

</div>`;

contentLines[i] = html;

}

}

// Check for LIST_LINKS_AS_KEYWORDS and add any links in lists `* [...](...)

var listLinksAsKeywordsIdx = contentLines.indexOf("* LIST_LINKS_AS_KEYWORDS");

if (listLinksAsKeywordsIdx>=0) {

contentLines.splice(listLinksAsKeywordsIdx,1); // remove first line (copyright)

for (i in contentLines) {

var match;

match = contentLines[i].match(/^\* \[(.*)\]\((.*)\)$/);

if (match!=null) {

var description = match[1];

var link = match[2];

var pageInfo = {

path : link,

title : description,

};

fileTitles[pageInfo.path] = pageInfo.title;

htmlLinks[link] = link;

addToList(fileInfo.keywords, description, pageInfo);

}

}

}

contentLines.splice(0,1); // remove first line (copyright)

var title = contentLines[0];

contents = contentLines.join("\n");

// ========================================================================== Get Markdown

if (PROCESS_INFERENCE) inferMarkdownFile(file, contents);

html = marked(contents).replace(/lang-JavaScript/g, 'sh_javascript');

// ==========================================================================

// Append tutorial links

appendMatching("APPEND_KEYWORD", fileInfo.keywords, "");

appendMatching("APPEND_USES", fileInfo.parts, "<p>(No tutorials are available yet)</p>");

// Check for Pinouts

var regex = /<ul>\n<li>APPEND_PINOUT: (.*)<\/li>\n<\/ul>/;

var match = html.match(regex);

if (match!=null) {

var htmlfilename = HTML_DIR+"boards/" + match[1] + ".html";

console.log("APPEND_PINOUT "+htmlfilename);

var pinout = fs.readFileSync(htmlfilename).toString();

html = html.replace(regex, pinout);

}

// Check for TOC and create it if needed

var tocTag = "<li>APPEND_TOC</li>";

var tocIdx = html.indexOf(tocTag);

if (tocIdx>=0) {

// parse out all headings, and create lists based on them

regex = /<([hH])([123]) id="([a-z\-]*)"[^>]*>/g;

var toc = "";

var currentLevel = 1;

var m = regex.exec(html);

while (m) {

if (m.index > tocIdx) { // only add headings from AFTER this TOC tag

var heading = {level:0|m[2], hash:m[3]};

var endIdx = html.indexOf("</"+m[1]+m[2], m.index);

heading.title = html.substring(m.index+m[0].length, endIdx);

heading.title = heading.title.replace(/<\/?a[^>]*>/g,""); // remove anchors

while (currentLevel<heading.level) { toc+="<ul>";currentLevel++; }

while (currentLevel>heading.level) { toc+="</ul>";currentLevel--; }

toc += '<li><a href="#'+heading.hash+'">'+heading.title+"</a></li>\n";

// add keywords

var pageInfo = {

path : file+"#"+heading.hash,

title : title+": "+heading.title,

};

fileTitles[pageInfo.path] = pageInfo.title;

addToList(fileInfo.keywords, heading.title, pageInfo);

}

m = regex.exec(html);

}

while (currentLevel>1) { toc+="</ul>";currentLevel--; }

html = html.substr(0,tocIdx) + toc + html.substr(tocIdx+tocTag.length);

}

// If compiling for offline, quick hack to modify most links to go direct

if (OFFLINE) // FIXME - move this into the code below, do it properly with knownPageLinks

html = html.replace(/(<[aA] .*href=")\/([^"#]*)/g, '$1$2.html');

// Scan all links in HTML to find possible replaced files

{

regex = /href="([^"]+)"/g;

var m = regex.exec(html);

while (m) {

var url = m[1];

var baseURL = url.replace(/#.*$/,"");

if (baseURL!="") {

if (baseURL in renamedFiles) {

WARNING(`${file}: ${JSON.stringify(baseURL)} was renamed to ${JSON.stringify(renamedFiles[baseURL])}`);

} else {

if (baseURL.startsWith("http:") || baseURL.startsWith("https:")) {

// we're ok...

} else { // TODO: check if we know the page link

/*if (knownPageLinks.indexOf(baseURL)<0)

WARNING(`${file}: ${JSON.stringify(baseURL)} is not found`);*/

}

}

}

m = regex.exec(html);

}

}

// work out of we have any images that might be at the top of the page

var hasImageContent = html.indexOf("<iframe ")>=0; // do we have a video?

// do we have an image in the first few lines of markdown?

for (var i=0;i<15 && i<contentLines.length;i++)

if (contentLines[i].match(/!\[[^\]]*\]\(([^\)]*)\)/)) // find an image tag

hasImageContent = true;

// if there's no image/video then we add the thumbnail in the top-right

var thumbnail = "";

if (!hasImageContent && file in markdownThumbs)

thumbnail = `<img class="topthumbnail" src="${markdownThumbs[file]}">`;

// GitHub link

var github_url = "https://github.com/espruino/EspruinoDocs/blob/master/"+path.relative(BASEDIR, file);

// assemble final page

html = '<div style="min-height:700px;">' +

thumbnail +

html + '</div>'+

'<p style="text-align:right;font-size:75%;">This page is auto-generated from <a href="'+github_url+'">GitHub</a>. If you see any mistakes or have suggestions, please <a href="https://github.com/espruino/EspruinoDocs/issues/new?title='+file+'">let us know</a>.</p>';

if (OFFLINE) {

html = `<html><head>

<title>${title}</title>

<link rel="stylesheet" href="offline.css" type="text/css" />

</head>

<body style="background-image:none;">

<div id="wrap">${html}</div>

</body>

</html>

`;

}

fs.writeFileSync(htmlFiles[file], html);

});

// -----------------------------------------------------------

// write out the references

var refPath = path.resolve(BASEDIR, "references.json");

console.log("---------------------");

console.log("Writing references to "+refPath);

fs.writeFileSync(refPath, JSON.stringify(urls,null,1));

// Write out keywords

fs.writeFileSync(KEYWORD_JS_FILE, "var keywords = "+JSON.stringify(createKeywordsJS(fileInfo.keywords),null,1)+";");

console.log("====================================================== WARNINGS");

console.log("====================================================== WARNINGS");

console.log("====================================================== WARNINGS");

warnings.forEach(w=>console.log(w));

console.log("====================================================== WARNINGS");

console.log("====================================================== WARNINGS");

console.log("====================================================== WARNINGS");

// -----------------------------------------------------------

// Newest tutorials

/*child_process.exec(

'git log --name-status | grep "^A" | cut -c3- | grep -e ".*\.md$" -e "examples/.*\.js$" | grep -v "^boards" | head -20',

function(error, stdout, stderr){

var newTutorials = stdout.split("\n").map(x=>x.trim());

newTutorials.forEach(tutorial=>{

var p = path.relative(BASEDIR, tutorial);

if (!p) return;

var thumb = getThumbLinkHTML({

path: p,

title: fileTitles[p]

});

console.log(thumb);

});

});*/