feat: xkcd cloner

License: MIT
Signed-off-by: Henrique Dias <hacdias@gmail.com>

bin/index.js

@@ -0,0 +1,125 @@
+#!/usr/bin/env node
+
+const fs = require('fs-extra')
+const yargs = require('yargs')
+const { basename, join } = require('path')
+const { getLatestId, getComic } = require('../lib/xkcd')
+const { homePage, comicPage } = require('../lib/html')
+const { pad, progress } = require('../lib/helpers')
+
+const argv = yargs
+  .usage('$0', 'Clones XKCD comics. By default it only downloads the missing comics.')
+  .scriptName('xkcd-clone')
+  .option('dir', {
+    alias: 'd',
+    describe: 'Output directory',
+    type: 'string',
+    demandOption: true
+  }).option('empty', {
+    alias: 'e',
+    describe: 'Redownload all comics',
+    type: 'boolean'
+  })
+  .help()
+  .argv
+
+async function write ({ data, img }, dir) {
+  try {
+    await fs.outputJSON(join(dir, 'info.json'), data, { spaces: '\t' })
+    const dest = fs.createWriteStream(join(dir, basename(data.img)))
+    img.body.pipe(dest)
+    await fs.outputFile(join(dir, 'index.html'), comicPage(data))
+  } catch (err) {
+    await fs.remove(dir)
+    throw err
+  }
+}
+
+async function run () {
+  console.log(`😊 Going to clone XKCD to ${argv.dir}`)
+
+  let added = []
+  let errored = []
+
+  try {
+    console.log(`🔍 Finding the latest comic`)
+    const latest = await getLatestId()
+    console.log(`😁 Found! We're on comic number ${latest}!`)
+
+    await fs.ensureDir(argv.dir)
+    if (argv.empty) {
+      await fs.emptyDir(argv.dir)
+    }
+
+    for (let i = 1; i <= latest; i++) {
+      const num = pad(i, 4)
+      const dir = join(argv.dir, num)
+
+      if (await fs.pathExists(dir)) {
+        const data = await fs.readJSON(join(dir, 'info.json'))
+        added.push({ id: i, title: data.title, num })
+        await fs.outputFile(join(dir, 'index.html'), comicPage(data))
+        continue
+      } else if (i === 404) {
+        progress(`📦 404 not found 😵`)
+        continue
+      } else {
+        progress(`📦 Fetching ${i} out of ${latest}`)
+      }
+
+      let comic = null
+
+      try {
+        comic = await getComic(i)
+      } catch (err) {
+        progress(`😢 Could not fetch ${i}, will try again later\n`)
+        errored.push(i)
+      }
+
+      await write(comic, dir)
+      added.push({
+        id: i,
+        title: comic.data.title,
+        num: num
+      })
+    }
+  } catch (err) {
+    console.log(`🐉 ${err.stack}`)
+    process.exit(1)
+  }
+
+  // TODO: redownload errored
+
+  if (errored.length === 0) {
+    progress(`📦 All comics fetched\n`)
+  } else {
+    progress(`📦 Some comics fetched\n`)
+  }
+
+  added = added.sort((a, b) => a.num - b.num)
+  await fs.copyFile(join(__dirname, '../node_modules/tachyons/css/tachyons.min.css'), join(argv.dir, 'tachyons.css'))
+  await fs.outputFile(join(argv.dir, 'index.html'), homePage(added))
+}
+/*
+
+const clone = async ({ baseDir, empty, onlyMissing }) => {
+  /*
+
+  for (const num of errored) {
+    for (let i = 0; i < 3; i++) {
+      try {
+        const comic = await getComic(i)
+        await write(comic, dir)
+        break
+      } catch (err) {
+        if (i === 2) {
+          console.log(`😢 ${num} could not be fetched: ${err.toString()}`)
+        }
+      }
+    }
+  }
+
+}
+ */
+
+run()

index.js

@@ -1,150 +0,0 @@
-const fetch = require('node-fetch')
-const fs = require('fs-extra')
-const { basename, join } = require('path')
-
-const progress = (str) => {
-  process.stdout.clearLine()
-  process.stdout.cursorTo(0)
-  process.stdout.write(str)
-}
-
-const pad = (str, max) => {
-  str = str.toString()
-  return str.length < max ? pad('0' + str, max) : str
-}
-
-const fetchLatest = () => {
-  return fetch(`https://xkcd.com/info.0.json`)
-    .then(res => res.json())
-}
-
-const fetchId = (id) => {
-  return fetch(`https://xkcd.com/${id}/info.0.json`)
-    .then(res => res.json())
-}
-
-const htmlPage = ({ alt, title, transcript, num, img }) => {
-  const btnClass = 'dib navy mh2 pa2 bg-light-blue bg-animate hover-bg-lightest-blue br2 ba bw1 b--navy no-underline'
-
-  return `<html>
-<head>
-  <title>${num} - ${title}</title>
-  <link rel="stylesheet" href="https://unpkg.com/tachyons@4.10.0/css/tachyons.min.css"/>
-</head>
-<body class="tc bg-washed-blue navy sans-serif ml-auto mr-auto mw7 w-90">
-  <h1 class="mh0 mt4 mb3 f2 small-caps tracked">${title} <span class="light-blue">#${num}</span></h1>
-
-  <nav class="mv3">
-    <a class="${btnClass}" href="../${pad(num - 1, 4)}/index.html"><span class="gray">←</span> Prev</a>
-    <a class="${btnClass}" href="../index.html">Home</a>
-    <a class="${btnClass}" href="../${pad(num + 1, 4)}/index.html">Next <span class="gray">→</span></a>
-  </nav>
-
-  <img src="./${basename(img)}" alt="${alt}">
-  <p class="dn">${transcript}</p>
-</body>
-</html>`
-}
-
-const download = async (baseDir, i) => {
-  const num = pad(i, 4)
-  const res = await fetchId(i)
-  const dir = join(baseDir, num)
-
-  try {
-    await fs.outputJSON(join(dir, 'info.json'), res, { spaces: '\t' })
-
-    const img = await fetch(res.img)
-    const dest = fs.createWriteStream(join(dir, basename(res.img)))
-
-    await fs.outputFile(join(dir, 'index.html'), htmlPage(res))
-    img.body.pipe(dest)
-  } catch (err) {
-    await fs.remove(dir)
-    throw err
-  }
-
-  return {
-    id: i,
-    title: res.title,
-    num: num
-  }
-}
-
-const homePage = (list) => `<html>
-<head>
-  <title>XKCD</title>
-  <link rel="stylesheet" href="https://unpkg.com/tachyons@4.10.0/css/tachyons.min.css"/>
-</head>
-<body class="tc bg-washed-blue navy sans-serif ml-auto mr-auto mw7 w-90">
-  <h1 class="mh0 mt4 mb3 f2 small-caps tracked">XKCD</h1>
-
-  ${list.map(({ id, title, num }) => `<a href="./${num}/index.html">${id} - ${title}</a>`).join('\n')}
-</body>
-</html>`
-
-const clone = async ({ baseDir, empty, onlyMissing }) => {
-  console.log(`😊 Going to clone XKCD to ${baseDir}`)
-
-  let errored = []
-  let added = []
-
-  try {
-    console.log(`🔍 Finding the latest comic`)
-    const latest = (await fetchLatest()).num
-    console.log(`😁 Found! Will download ${latest} comics 🥶`)
-
-    await fs.ensureDir(baseDir)
-    if (empty) await fs.emptyDir(baseDir)
-
-    let existent = []
-    if (onlyMissing) {
-      existent = fs.readdirSync(baseDir).map(v => Number(v.split('-')[0].trim()))
-    }
-
-    for (let i = 1; i <= 10 + 1; i++) {
-      if (existent.includes(i)) {
-        continue
-      } else if (i === 404) {
-        progress(`📦 404 not found 😵`)
-        continue
-      } else if (i === latest + 1) {
-        progress(`📦 All ${latest} comics fetched\n`)
-        continue
-      } else {
-        progress(`📦 Fetching ${i} out of ${latest}`)
-      }
-
-      try {
-        added.push(await download(baseDir, i))
-      } catch (err) {
-        errored.push(i)
-      }
-    }
-  } catch (err) {
-    console.log(`🐉 ${err.stack}`)
-    process.exit(1)
-  }
-
-  for (const num of errored) {
-    for (let i = 0; i < 3; i++) {
-      try {
-        added.push(await download(baseDir, i))
-        break
-      } catch (err) {
-        if (i === 2) {
-          console.log(`😢 ${num} could not be fetched: ${err.toString()}`)
-        }
-      }
-    }
-  }
-
-  added = added.sort((a, b) => a.num - b.num)
-  await fs.outputFile(join(baseDir, 'index.html'), homePage(added))
-}
-
-clone({
-  baseDir: './clone',
-  empty: true,
-  onlyMissing: false
-})

lib/helpers.js

@@ -0,0 +1,15 @@
+const progress = (str) => {
+  process.stdout.clearLine()
+  process.stdout.cursorTo(0)
+  process.stdout.write(str)
+}
+
+const pad = (str, max) => {
+  str = str.toString()
+  return str.length < max ? pad('0' + str, max) : str
+}
+
+module.exports = {
+  progress,
+  pad
+}

lib/html.js

@@ -0,0 +1,44 @@
+const { basename } = require('path')
+const { pad } = require('./helpers')
+
+const comicPage = ({ alt, title, transcript, num, img }) => {
+  const btnClass = 'dib navy mh2 pa2 bg-light-blue hover-bg-lightest-blue br2 ba bw1 b--navy no-underline'
+
+  return `<html>
+<head>
+  <title>${num} - ${title}</title>
+  <link rel="stylesheet" href="../tachyons.css"/>
+</head>
+<body class="tc bg-washed-blue navy sans-serif ml-auto mr-auto mw7 w-90">
+  <h1 class="mh0 mt4 mb3 f2 small-caps tracked">${title} <span class="light-blue">#${num}</span></h1>
+
+  <nav class="mv3">
+    <a class="${btnClass}" href="../${pad(num - 1, 4)}/index.html"><span class="gray">←</span> Prev</a>
+    <a class="${btnClass}" href="../index.html">Home</a>
+    <a class="${btnClass}" href="../${pad(num + 1, 4)}/index.html">Next <span class="gray">→</span></a>
+  </nav>
+
+  <img src="./${basename(img)}" alt="${alt}">
+  <p class="dn">${transcript}</p>
+</body>
+</html>`
+}
+
+const homePage = (list) => `<html>
+<head>
+  <title>XKCD</title>
+  <link rel="stylesheet" href="./tachyons.css"/>
+</head>
+<body class="bg-washed-blue navy sans-serif ml-auto mr-auto mw7 w-90">
+  <h1 class="tc mh0 mt4 mb3 f2 small-caps tracked">XKCD</h1>
+  
+  <ul class="list pa0 ma0">
+  ${list.map(({ id, title, num }) => `<li><a class="blue hover-dark-blue no-underline" href="./${num}/index.html">${id} - ${title}</a></li>`).join('\n')}
+  </ul>
+</body>
+</html>`
+
+module.exports = {
+  comicPage,
+  homePage
+}

lib/xkcd.js

@@ -0,0 +1,20 @@
+const fetch = require('node-fetch')
+
+async function getLatestId () {
+  const raw = await fetch(`https://xkcd.com/info.0.json`)
+  const data = await raw.json()
+  return data.num
+}
+
+async function getComic (id) {
+  const raw = await fetch(`https://xkcd.com/${id}/info.0.json`)
+  const data = await raw.json()
+  const img = await fetch(data.img)
+
+  return { data, img }
+}
+
+module.exports = {
+  getLatestId,
+  getComic
+}

package.json

@@ -2,9 +2,12 @@
   "name": "xkcd-clone",
   "version": "1.0.0",
   "description": "",
-  "main": "index.js",
+  "main": "./lib/index.js",
-  "author": "",
+  "bin": {
-  "license": "ISC",
+    "xkcd-clone": "./bin/index.js"
+  },
+  "license": "MIT",
+  "author": "Henrique Dias <hacdias@gmail.com>",
   "dependencies": {
     "fs-extra": "^7.0.1",
     "node-fetch": "^2.4.0",