require("dotenv").config(); const http = require('http'); const fs = require("fs"); const axios = require('axios'); const uuid4 = require('uuid4'); const puppeteer = require('puppeteer'); const JSZip = require('jszip'); const server = http.createServer(async (req, res) => { const incomingURL = new URL(`http://localhost:8000${req.url}`); if (incomingURL.searchParams.get("website")) { const website = new URL(incomingURL.searchParams.get("website")); console.log(`Fetching '${website.toString()}'`); let fn = sendPage; if (website.toString().endsWith(".pdf")) { fn = sendPDF; } if (website.toString().endsWith(".epub")) { fn = sendEpub; } if (await fn(website)) { fs.readFile(__dirname + "/success.html", function (err,data) { if (err) { res.writeHead(404); res.end(JSON.stringify(err)); return; } res.writeHead(200, {'Content-Type': 'text/html'}); res.end(data); }); } else { fs.readFile(__dirname + "/failure.html", function (err,data) { if (err) { res.writeHead(404); res.end(JSON.stringify(err)); return; } res.writeHead(500, {'Content-Type': 'text/html'}); res.end(data); }); } } else { let url = req.url === "/" ? "/index.html": req.url; fs.readFile(__dirname + url || "/index.html", function (err,data) { if (err) { res.writeHead(404); res.end(JSON.stringify(err)); return; } if (url.endsWith(".js")) { res.writeHead(200, {'Content-Type': 'application/javascript'}); } else if (url.endsWith(".json")) { res.writeHead(200, {'Content-Type': 'application/json'}); } else if (url.endsWith(".png")) { res.writeHead(200, {'Content-Type': 'image/png'}); } else { res.writeHead(200, {'Content-Type': 'text/html'}); } res.end(data); }); } }); server.listen(8000); async function sendPDF(website, tries = 0) { try { const response = await axios.get(website.toString(), { responseType: 'arraybuffer' }) const title = decodeURIComponent(website.toString().substring(website.toString().lastIndexOf("/")+1, website.toString().lastIndexOf("."))); await sendToRemarkable(title, Buffer.from(response.data, 'binary')); return true; } catch (ex) { console.log(ex); if (tries < 5) { return await sendPDF(website, ++tries); } else { return false; } } } async function sendEpub(website, tries = 0) { try { const response = await axios.get(website.toString(), { responseType: 'arraybuffer' }) const title = decodeURIComponent(website.toString().substring(website.toString().lastIndexOf("/")+1, website.toString().lastIndexOf("."))); await sendToRemarkable(title, Buffer.from(response.data, 'binary'), 'epub'); return true; } catch (ex) { console.log(ex); if (tries < 5) { return await sendEpub(website, ++tries); } else { return false; } } } async function sendPage(website, tries = 0) { const browser = await puppeteer.launch({ ignoreHTTPSErrors: true, executablePath: process.env.CHROMIUM_PATH, args: ['--disable-dev-shm-usage', '--no-sandbox'] }); try { const page = await browser.newPage(); await page.goto(website.toString(), { referer: "https://www.google.com/" }); const title = await page.title() console.log("Page loaded. Title - " + title) await page.addStyleTag({ content: ` body { font-family: Helvetica, Georgia, serif; font-size: 20pt; line-height: 1.2em; background: none; color: black; text-align: left; } h1, h2, h3, h4, h5 { page-break-after: avoid; font-weight: bold; margin-top: 4px; } h2, h3, h4, h5 { padding-top: 16px; } b, strong { font-weight: bold; } u { text-decoration: underline; } i, em { font-style: italic; } table, figure, ul, img { page-break-inside: avoid; } a { color: black; } a:after { content: " [" attr(href) "] "; font-size: 0.7em; } a[href^="#"]:after, a[href^="/"]:after, a[href^="javascript:"]:after { content: ""; } blockquote { margin: 10px 2px; line-height: 1.5em; border: 0; border-left: 8px solid grey; padding-left: 8px; } table { width: 100%; margin: 4px; border: 1px solid black; } table td, table th { border: 1px solid black; padding: 2px } table thead, table thead th { font-weight: bold; border-bottom-width: 2px; } code { background: none !important; font-family: monospace; } pre { overflow: visible; white-space: pre-wrap; } ul li { list-style: disc !important; margin-left: 50px; } h1 { font-size: 1.7em; } p { margin-bottom: 12px; } header { margin-bottom: 14px; border-bottom: 8px solid black; text-align: center; } header blockquote { border: 0 !important; } /* SCP-Wiki */ .creditRate, .collapsible-block-folded, .collapsible-block-unfolded-link, .footer-wikiwalk-nav, .licensebox, .translation_block, #u-author_block, .u-faq, .info-container, .diamond-part, [class*='licensebox'] { display: none !important; } .collapsible-block-unfolded { display: block !important; } .anom-bar-container { max-width: 80% !important; font-size: 10pt; } .anom-bar-container a:after { content: "" !important; } .disrupt-class:before, .disrupt-class:after, .risk-class:before, .risk-class:after, .anom-bar-container .main-class:before, .anom-bar-container .main-class:after { display: none !important; content: "" !important; border: none !important; } `}); await page.evaluate(async () => { return await new Promise(resolve => { var REGEXPS={unlikelyCandidates:/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,okMaybeItsACandidate:/and|article|body|column|content|main|shadow/i};function isNodeVisible(a){return(!a.style||"none"!=a.style.display)&&!a.hasAttribute("hidden")&&(!a.hasAttribute("aria-hidden")||"true"!=a.getAttribute("aria-hidden")||a.className&&a.className.indexOf&&-1!==a.className.indexOf("fallback-image"))}function isProbablyReaderable(a,b){b||(b=isNodeVisible);var c=a.querySelectorAll("p, pre"),d=a.querySelectorAll("div > br");if(d.length){var e=new Set(c);[].forEach.call(d,function(a){e.add(a.parentNode)}),c=Array.from(e)}var f=0;return[].some.call(c,function(a){if(!b(a))return!1;var c=a.className+" "+a.id;if(REGEXPS.unlikelyCandidates.test(c)&&!REGEXPS.okMaybeItsACandidate.test(c))return!1;if(a.matches("li p"))return!1;var d=a.textContent.trim().length;return!(140>d)&&(f+=Math.sqrt(d-140),!!(20]*>/gi,normalize:/\s{2,}/g,videos:/\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,shareElements:/(\b|_)(share|sharedaddy)(\b|_)/i,nextLink:/(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,prevLink:/(prev|earl|old|new|<|«)/i,whitespace:/^\s*$/,hasContent:/\S$/,srcsetUrl:/(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,b64DataUrl:/^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i},DIV_TO_P_ELEMS:["A","BLOCKQUOTE","DL","DIV","IMG","OL","P","PRE","TABLE","UL","SELECT"],ALTER_TO_DIV_EXCEPTIONS:["DIV","ARTICLE","SECTION","P"],PRESENTATIONAL_ATTRIBUTES:["align","background","bgcolor","border","cellpadding","cellspacing","frame","hspace","rules","style","valign","vspace"],DEPRECATED_SIZE_ATTRIBUTE_ELEMS:["TABLE","TH","TD","HR","PRE"],PHRASING_ELEMS:["ABBR","AUDIO","B","BDO","BR","BUTTON","CITE","CODE","DATA","DATALIST","DFN","EM","EMBED","I","IMG","INPUT","KBD","LABEL","MARK","MATH","METER","NOSCRIPT","OBJECT","OUTPUT","PROGRESS","Q","RUBY","SAMP","SCRIPT","SELECT","SMALL","SPAN","STRONG","SUB","SUP","TEXTAREA","TIME","VAR","WBR"],CLASSES_TO_PRESERVE:["page"],HTML_ESCAPE_MAP:{lt:"<",gt:">",amp:"&",quot:"\"",apos:"'"},_postProcessContent:function(e){this._fixRelativeUris(e),this._keepClasses||this._cleanClasses(e)},_removeNodes:function(e,t){if(this._docJSDOMParser&&e._isLiveNodeList)throw new Error("Do not pass live node lists to _removeNodes");for(var a=e.length-1;0<=a;a--){var n=e[a],l=n.parentNode;l&&(!t||t.call(this,n,a,e))&&l.removeChild(n)}},_replaceNodeTags:function(e,t){if(this._docJSDOMParser&&e._isLiveNodeList)throw new Error("Do not pass live node lists to _replaceNodeTags");for(var a,n=e.length-1;0<=n;n--)a=e[n],this._setNodeTag(a,t)},_forEachNode:function(e,t){Array.prototype.forEach.call(e,t,this)},_someNode:function(e,t){return Array.prototype.some.call(e,t,this)},_everyNode:function(e,t){return Array.prototype.every.call(e,t,this)},_concatNodeLists:function(){var e=Array.prototype.slice,t=e.call(arguments),a=t.map(function(t){return e.call(t)});return Array.prototype.concat.apply([],a)},_getAllNodesWithTag:function(e,t){return e.querySelectorAll?e.querySelectorAll(t.join(",")):[].concat.apply([],t.map(function(t){var a=e.getElementsByTagName(t);return Array.isArray(a)?a:Array.from(a)}))},_cleanClasses:function(e){var t=this._classesToPreserve,a=(e.getAttribute("class")||"").split(/\s+/).filter(function(e){return-1!=t.indexOf(e)}).join(" ");for(a?e.setAttribute("class",a):e.removeAttribute("class"),e=e.firstElementChild;e;e=e.nextElementSibling)this._cleanClasses(e)},_fixRelativeUris:function(e){function t(e){if(a==n&&"#"==e.charAt(0))return e;try{return new URL(e,a).href}catch(e){}return e}var a=this._doc.baseURI,n=this._doc.documentURI,i=this._getAllNodesWithTag(e,["a"]);this._forEachNode(i,function(e){var a=e.getAttribute("href");if(a)if(0!==a.indexOf("javascript:"))e.setAttribute("href",t(a));else if(1===e.childNodes.length&&e.childNodes[0].nodeType===this.TEXT_NODE){var n=this._doc.createTextNode(e.textContent);e.parentNode.replaceChild(n,e)}else{for(var i=this._doc.createElement("span");0»] /.test(a))i=/ [\\\/>»] /.test(a),a=n.replace(/(.*)[\|\-\\\/>»] .*/gi,"$1"),3>e(a)&&(a=n.replace(/[^\|\-\\\/>»]*[\|\-\\\/>»](.*)/gi,"$1"));else if(-1!==a.indexOf(": ")){var l=this._concatNodeLists(t.getElementsByTagName("h1"),t.getElementsByTagName("h2")),r=a.trim(),d=this._someNode(l,function(e){return e.textContent.trim()===r});d||(a=n.substring(n.lastIndexOf(":")+1),3>e(a)?a=n.substring(n.indexOf(":")+1):5a.length){var o=t.getElementsByTagName("h1");1===o.length&&(a=this._getInnerText(o[0]))}a=a.trim().replace(this.REGEXPS.normalize," ");var s=e(a);return 4>=s&&(!i||s!=e(n.replace(/[\|\-\\\/>»]+/g,""))-1)&&(a=n),a},_prepDocument:function(){var e=this._doc;this._removeNodes(this._getAllNodesWithTag(e,["style"])),e.body&&this._replaceBrs(e.body),this._replaceNodeTags(this._getAllNodesWithTag(e,["font"]),"SPAN")},_nextElement:function(e){for(var t=e;t&&t.nodeType!=this.ELEMENT_NODE&&this.REGEXPS.whitespace.test(t.textContent);)t=t.nextSibling;return t},_replaceBrs:function(e){this._forEachNode(this._getAllNodesWithTag(e,["br"]),function(e){for(var t=e.nextSibling,a=!1;(t=this._nextElement(t))&&"BR"==t.tagName;){a=!0;var n=t.nextSibling;t.parentNode.removeChild(t),t=n}if(a){var i=this._doc.createElement("p");for(e.parentNode.replaceChild(i,e),t=i.nextSibling;t;){if("BR"==t.tagName){var l=this._nextElement(t.nextSibling);if(l&&"BR"==l.tagName)break}if(!this._isPhrasingContent(t))break;var r=t.nextSibling;i.appendChild(t),t=r}for(;i.lastChild&&this._isWhitespace(i.lastChild);)i.removeChild(i.lastChild);"P"===i.parentNode.tagName&&this._setNodeTag(i.parentNode,"DIV")}})},_setNodeTag:function(e,t){if(this.log("_setNodeTag",e,t),this._docJSDOMParser)return e.localName=t.toLowerCase(),e.tagName=t.toUpperCase(),e;for(var a=e.ownerDocument.createElement(t);e.firstChild;)a.appendChild(e.firstChild);e.parentNode.replaceChild(a,e),e.readability&&(a.readability=e.readability);for(var n=0;nMath.abs(n)){var i=!1;i=0