Bunch of updates. YOLO

getToken.js

@@ -0,0 +1,6 @@
+const { Remarkable, ItemResponse } = require('remarkable-typescript');
+(async () => {
+  const client = new Remarkable();
+  const token = await client.register({ code: 'whahovrq' });
+  console.log(token);
+})()

index.js

@@ -1,10 +1,10 @@
 require("dotenv").config();
 const http = require('http');
 const fs = require("fs");
+const axios = require('axios');
+const uuid4 = require('uuid4');
 const puppeteer = require('puppeteer');
-const { Remarkable } = require('remarkable-typescript');
-
-let token = process.env.REMARKABLE_TOKEN;
+const JSZip = require('jszip');
 
 const server = http.createServer(async (req, res) => {
   const incomingURL = new URL(`http://localhost:8000${req.url}`);
@@ -12,7 +12,13 @@ const server = http.createServer(async (req, res) => {
   if (incomingURL.searchParams.get("website")) {
     const website = new URL(incomingURL.searchParams.get("website"));
     console.log(`Fetching '${website.toString()}'`);
-    if (await sendPage(website)) {
+
+    let fn = sendPage;
+    if (website.toString().endsWith(".pdf")) {
+      fn = sendPDF;
+    }
+
+    if (await fn(website)) {
       fs.readFile(__dirname + "/success.html", function (err,data) {
         if (err) {
           res.writeHead(404);
@@ -33,7 +39,7 @@ const server = http.createServer(async (req, res) => {
         res.end(data);
       });
     }
-  }else {
+  } else {
     let url = req.url === "/" ? "/index.html": req.url;
     fs.readFile(__dirname + url || "/index.html", function (err,data) {
       if (err) {
@@ -59,33 +65,135 @@ const server = http.createServer(async (req, res) => {
 
 server.listen(8000);
 
+async function sendPDF(website, tries = 0) {
+  try {
+    const response = await axios.get(website.toString(), {
+      responseType: 'arraybuffer'
+    })
+    const title = website.toString().substring(website.toString().lastIndexOf("/")+1, website.toString().lastIndexOf("."))
+    await sendToRemarkable(title, Buffer.from(response.data, 'binary'));
+
+    return true;
+  } catch (ex) {
+    console.log(ex);
+    if (tries < 5) {
+      return await sendPDF(website, ++tries);
+    } else {
+      return false;
+    }
+  }
+}
+
 async function sendPage(website, tries = 0) {
   const browser = await puppeteer.launch({
+    ignoreHTTPSErrors: true,
     executablePath: process.env.CHROMIUM_PATH,
     args: ['--disable-dev-shm-usage', '--no-sandbox']
   });
   try {
     const page = await browser.newPage();
-    await page.emulate(Object.assign({}, puppeteer.devices["iPad Pro"], { userAgent: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" }));
     await page.goto(website.toString(), { referer: "https://www.google.com/" });
     const title = await page.title()
     console.log("Page loaded. Title - " + title)
 
-    await page.evaluate( function(){
-      [...document.querySelectorAll('*')].forEach(node => {
-        const pos = window.getComputedStyle(node).getPropertyValue("position");
-        if (pos == "fixed" || pos == "sticky") {
-          node.style.position = "unset";
-        }
-      })
-    } );
+    await page.addStyleTag({ content: `
+    body {
+      font-family: Georgia, serif;
+      font-size: 18pt;
+      background: none;
+      color: black;
+      text-align: left;
+    }
+    h1, h2, h3, h4, h5 {
+      page-break-after: avoid;
+    }
+    table, figure, ul, img {
+      page-break-inside: avoid;
+    }
+    a {
+      color: black;
+    }
+    a:after {
+      content: " [" attr(href) "] ";
+      font-size: 0.7em;
+    }
+    a[href^="#"]:after, a[href^="/"]:after {
+      content: "";
+    }
+    blockquote {
+      margin: 10px 2px;
+      line-height: 2em;
+      border: 0;
+    }
 
-    const myPDF = await page.pdf({ format: 'A3', margin: {top: 5, bottom: 5} });
+    code {
+      background: none !important;
+      font-family: monospace;
+    }
+
+    ul li {
+      list-style: disc !important;
+    }
+
+    h1 {
+      font-size: 1.7em;
+    }
+
+    p {
+      margin-bottom: 12px;
+    }
+
+    header {
+      margin-bottom: 14px;
+      border-bottom: 8px solid black;
+      text-align: center;
+    }
+    `});
+
+    await page.evaluate(async () => {
+      return await new Promise(resolve => {
+        var REGEXPS={unlikelyCandidates:/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,okMaybeItsACandidate:/and|article|body|column|content|main|shadow/i};function isNodeVisible(node){return(!node.style||node.style.display!="none")&&!node.hasAttribute("hidden")&&(!node.hasAttribute("aria-hidden")||node.getAttribute("aria-hidden")!="true"||(node.className&&node.className.indexOf&&node.className.indexOf("fallback-image")!==-1))}function isProbablyReaderable(doc,isVisible){if(!isVisible){isVisible=isNodeVisible}var nodes=doc.querySelectorAll("p, pre");var brNodes=doc.querySelectorAll("div > br");if(brNodes.length){var set=new Set(nodes);[].forEach.call(brNodes,function(node){set.add(node.parentNode)});nodes=Array.from(set)}var score=0;return[].some.call(nodes,function(node){if(!isVisible(node)){return false}var matchString=node.className+" "+node.id;if(REGEXPS.unlikelyCandidates.test(matchString)&&!REGEXPS.okMaybeItsACandidate.test(matchString)){return false}if(node.matches("li p")){return false}var textContentLength=node.textContent.trim().length;if(textContentLength<140){return false}score+=Math.sqrt(textContentLength-140);if(score>20){return true}return false})}if(typeof exports==="object"){exports.isProbablyReaderable=isProbablyReaderable}
+        function Readability(doc,options){if(options&&options.documentElement){doc=options;options=arguments[2]}else if(!doc||!doc.documentElement){throw new Error("First argument to Readability constructor should be a document object.")}options=options||{};this._doc=doc;this._docJSDOMParser=this._doc.firstChild.__JSDOMParser__;this._articleTitle=null;this._articleByline=null;this._articleDir=null;this._articleSiteName=null;this._attempts=[];this._debug=!!options.debug;this._maxElemsToParse=options.maxElemsToParse||this.DEFAULT_MAX_ELEMS_TO_PARSE;this._nbTopCandidates=options.nbTopCandidates||this.DEFAULT_N_TOP_CANDIDATES;this._charThreshold=options.charThreshold||this.DEFAULT_CHAR_THRESHOLD;this._classesToPreserve=this.CLASSES_TO_PRESERVE.concat(options.classesToPreserve||[]);this._keepClasses=!!options.keepClasses;this._flags=this.FLAG_STRIP_UNLIKELYS|this.FLAG_WEIGHT_CLASSES|this.FLAG_CLEAN_CONDITIONALLY;var logEl;if(this._debug){logEl=function(e){var rv=e.nodeName+" ";if(e.nodeType==e.TEXT_NODE){return rv+'("'+e.textContent+'")'}var classDesc=e.className&&("."+e.className.replace(/ /g,"."));var elDesc="";if(e.id){elDesc="(#"+e.id+classDesc+")"}else if(classDesc){elDesc="("+classDesc+")"}return rv+elDesc};this.log=function(){if(typeof dump!=="undefined"){var msg=Array.prototype.map.call(arguments,function(x){return(x&&x.nodeName)?logEl(x):x}).join(" ");dump("Reader: (Readability) "+msg+"\n")}else if(typeof console!=="undefined"){var args=["Reader: (Readability) "].concat(arguments);console.log.apply(console,args)}}}else{this.log=function(){}}}Readability.prototype={FLAG_STRIP_UNLIKELYS:0x1,FLAG_WEIGHT_CLASSES:0x2,FLAG_CLEAN_CONDITIONALLY:0x4,ELEMENT_NODE:1,TEXT_NODE:3,DEFAULT_MAX_ELEMS_TO_PARSE:0,DEFAULT_N_TOP_CANDIDATES:5,DEFAULT_TAGS_TO_SCORE:"section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),DEFAULT_CHAR_THRESHOLD:500,REGEXPS:{unlikelyCandidates:/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,okMaybeItsACandidate:/and|article|body|column|content|main|shadow/i,positive:/article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,negative:/hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i,extraneous:/print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,byline:/byline|author|dateline|writtenby|p-author/i,replaceFonts:/<(\/?)font[^>]*>/gi,normalize:/\s{2,}/g,videos:/\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,shareElements:/(\b|_)(share|sharedaddy)(\b|_)/i,nextLink:/(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,prevLink:/(prev|earl|old|new|<|«)/i,whitespace:/^\s*$/,hasContent:/\S$/},DIV_TO_P_ELEMS:["A","BLOCKQUOTE","DL","DIV","IMG","OL","P","PRE","TABLE","UL","SELECT"],ALTER_TO_DIV_EXCEPTIONS:["DIV","ARTICLE","SECTION","P"],PRESENTATIONAL_ATTRIBUTES:["align","background","bgcolor","border","cellpadding","cellspacing","frame","hspace","rules","style","valign","vspace"],DEPRECATED_SIZE_ATTRIBUTE_ELEMS:["TABLE","TH","TD","HR","PRE"],PHRASING_ELEMS:["ABBR","AUDIO","B","BDO","BR","BUTTON","CITE","CODE","DATA","DATALIST","DFN","EM","EMBED","I","IMG","INPUT","KBD","LABEL","MARK","MATH","METER","NOSCRIPT","OBJECT","OUTPUT","PROGRESS","Q","RUBY","SAMP","SCRIPT","SELECT","SMALL","SPAN","STRONG","SUB","SUP","TEXTAREA","TIME","VAR","WBR"],CLASSES_TO_PRESERVE:["page"],_postProcessContent:function(articleContent){this._fixRelativeUris(articleContent);if(!this._keepClasses){this._cleanClasses(articleContent)}},_removeNodes:function(nodeList,filterFn){if(this._docJSDOMParser&&nodeList._isLiveNodeList){throw new Error("Do not pass live node lists to _removeNodes")}for(var i=nodeList.length-1;i>=0;i-=1){var node=nodeList[i];var parentNode=node.parentNode;if(parentNode){if(!filterFn||filterFn.call(this,node,i,nodeList)){parentNode.removeChild(node)}}}},_replaceNodeTags:function(nodeList,newTagName){if(this._docJSDOMParser&&nodeList._isLiveNodeList){throw new Error("Do not pass live node lists to _replaceNodeTags")}for(var i=nodeList.length-1;i>=0;i-=1){var node=nodeList[i];this._setNodeTag(node,newTagName)}},_forEachNode:function(nodeList,fn){Array.prototype.forEach.call(nodeList,fn,this)},_someNode:function(nodeList,fn){return Array.prototype.some.call(nodeList,fn,this)},_everyNode:function(nodeList,fn){return Array.prototype.every.call(nodeList,fn,this)},_concatNodeLists:function(){var slice=Array.prototype.slice;var args=slice.call(arguments);var nodeLists=args.map(function(list){return slice.call(list)});return Array.prototype.concat.apply([],nodeLists)},_getAllNodesWithTag:function(node,tagNames){if(node.Line truncated
+
+        [...document.querySelectorAll('details')].forEach(details => details.setAttribute('open', ''));
+
+        [...document.querySelectorAll('*')].forEach(node => {
+          const pos = window.getComputedStyle(node).getPropertyValue("position");
+          if (pos == "fixed" || pos == "sticky") {
+            node.style.position = "unset";
+          }
+        });
+
+        if (isProbablyReaderable(document.cloneNode(true))) {
+          var documentClone = document.cloneNode(true);
+          var article = new Readability(documentClone).parse();
+          var postedDate = document.querySelector('time[datetime]');
+          var content = `
+            <header>
+              <h1>${article.title}</h1>
+              ${article.byline ? `<blockquote>${article.byline}</blockquote>` : ""}
+              ${postedDate && postedDate.getAttribute('datetime') ? `<blockquote>${postedDate.getAttribute('datetime')}</blockquote>` : ""}
+            </header>
+            ` + article.content;
+          document.body.innerHTML = content;
+        }
+
+        var im = document.createElement("img");
+        im.src = `https://qr.cluster.fun/?website=${window.location.toString()}`;
+        im.style = "position:absolute;top:0;right:0;z-index:99999999";
+        im.onload = resolve;
+        im.onerror = () => {
+          document.body.removeChild(im);
+          resolve();
+        }
+        document.body.appendChild(im);
+      })
+    });
+
+    const myPDF = await page.pdf({ format: 'A4', margin: {top: 40, bottom: 40, left: 40, right: 40} });
     console.log("Saved to PDF")
 
-    const client = new Remarkable({ token });
-    await client.uploadPDF(title, myPDF);
-    console.log("Uploaded to reMarkable");
+    await sendToRemarkable(title, myPDF);
 
     return true;
   } catch (ex) {
@@ -99,3 +207,107 @@ async function sendPage(website, tries = 0) {
     await browser.close();
   }
 }
+
+async function sendToRemarkable(title, myPDF) {
+  try {
+    // Refresh token
+    let response = await axios.post(
+      "https://my.remarkable.com/token/json/2/user/new",
+      {},
+      {
+        headers: {
+          'Authorization': `Bearer ${process.env.REMARKABLE_TOKEN}`,
+        },
+      }
+    );
+    let token = response.data;
+    console.log(`Refreshed token: ${token}`);
+
+    // Get storage endpoint
+    response = await axios.get(
+      "https://service-manager-production-dot-remarkable-production.appspot.com/service/json/1/document-storage?environment=production&group=auth0%7C5a68dc51cb30df3877a1d7c4&apiVer=2",
+      {
+        headers: {
+          'Authorization': `Bearer ${token}`,
+        }
+      }
+    );
+    let storageHost = response.data.Host;
+    console.log(`Got storage host: ${storageHost}`);
+
+    // Generate upload request
+    const ID = uuid4();
+    response = await axios.put(
+      `https://${storageHost}/document-storage/json/2/upload/request`,
+      [{
+        "ID": ID,
+        "Type": "DocumentType",
+        "Version": 1
+      }],
+      {
+        headers: {
+          'Authorization': `Bearer ${token}`,
+        }
+      }
+    );
+    let uploadURL = response.data[0].BlobURLPut;
+    console.log(`Got upload URL: ${uploadURL}`);
+
+    // Build zip to upload
+    let zip = new JSZip();
+    zip.file(`${ID}.content`, JSON.stringify({
+      extraMetadata: {},
+      fileType: 'pdf',
+      lastOpenedPage: 0,
+      lineHeight: -1,
+      margins: 180,
+      pageCount: 0,
+      textScale: 1,
+      transform: {},
+    }));
+    zip.file(`${ID}.pagedata`, []);
+    zip.file(`${ID}.pdf`, myPDF);
+    const zipContent = await zip.generateAsync({ type: 'nodebuffer' });
+
+    // Upload zip
+    response = await axios.put(
+      uploadURL,
+      zipContent,
+      {
+        headers: {
+          'Content-Type': '',
+          'Authorization': `Bearer ${token}`,
+        }
+      }
+    );
+    console.log("Uploaded");
+
+    // Populate metadata
+    response = await axios.put(
+      `https://${storageHost}/document-storage/json/2/upload/update-status`,
+      [{
+        ID: ID,
+        deleted: false,
+        lastModified: new Date().toISOString(),
+        ModifiedClient: new Date().toISOString(),
+        metadatamodified: false,
+        modified: false,
+        parent: '',
+        pinned: false,
+        synced: true,
+        type: "DocumentType",
+        version: 1,
+        VissibleName: title,
+      }],
+      {
+        headers: {
+          'Authorization': `Bearer ${token}`,
+        }
+      }
+    );
+    console.log("Upload complete")
+  } catch (error) {
+    console.error(error.response);
+    throw error;
+  }
+}

package.json

@@ -9,10 +9,11 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "axios": "^0.19.2",
     "crypto-random-string": "^3.2.0",
     "dotenv": "^8.2.0",
-    "express": "^4.17.1",
+    "jszip": "^3.2.2",
     "puppeteer": "^2.1.1",
-    "remarkable-typescript": "^1.0.5"
+    "uuid4": "^1.1.4"
   }
 }