diff --git a/bin/index.js b/bin/index.js new file mode 100644 index 0000000..2dd71d0 --- /dev/null +++ b/bin/index.js @@ -0,0 +1,125 @@ +#!/usr/bin/env node + +const fs = require('fs-extra') +const yargs = require('yargs') +const { basename, join } = require('path') +const { getLatestId, getComic } = require('../lib/xkcd') +const { homePage, comicPage } = require('../lib/html') +const { pad, progress } = require('../lib/helpers') + +const argv = yargs + .usage('$0', 'Clones XKCD comics. By default it only downloads the missing comics.') + .scriptName('xkcd-clone') + .option('dir', { + alias: 'd', + describe: 'Output directory', + type: 'string', + demandOption: true + }).option('empty', { + alias: 'e', + describe: 'Redownload all comics', + type: 'boolean' + }) + .help() + .argv + +async function write ({ data, img }, dir) { + try { + await fs.outputJSON(join(dir, 'info.json'), data, { spaces: '\t' }) + const dest = fs.createWriteStream(join(dir, basename(data.img))) + img.body.pipe(dest) + await fs.outputFile(join(dir, 'index.html'), comicPage(data)) + } catch (err) { + await fs.remove(dir) + throw err + } +} + +async function run () { + console.log(`😊 Going to clone XKCD to ${argv.dir}`) + + let added = [] + let errored = [] + + try { + console.log(`🔍 Finding the latest comic`) + const latest = await getLatestId() + console.log(`😁 Found! We're on comic number ${latest}!`) + + await fs.ensureDir(argv.dir) + if (argv.empty) { + await fs.emptyDir(argv.dir) + } + + for (let i = 1; i <= latest; i++) { + const num = pad(i, 4) + const dir = join(argv.dir, num) + + if (await fs.pathExists(dir)) { + const data = await fs.readJSON(join(dir, 'info.json')) + added.push({ id: i, title: data.title, num }) + await fs.outputFile(join(dir, 'index.html'), comicPage(data)) + continue + } else if (i === 404) { + progress(`📦 404 not found 😵`) + continue + } else { + progress(`📦 Fetching ${i} out of ${latest}`) + } + + let comic = null + + try { + comic = await getComic(i) + } catch (err) { + progress(`😢 Could not fetch ${i}, will try again later\n`) + errored.push(i) + } + + await write(comic, dir) + added.push({ + id: i, + title: comic.data.title, + num: num + }) + } + } catch (err) { + console.log(`🐉 ${err.stack}`) + process.exit(1) + } + + // TODO: redownload errored + + if (errored.length === 0) { + progress(`📦 All comics fetched\n`) + } else { + progress(`📦 Some comics fetched\n`) + } + + added = added.sort((a, b) => a.num - b.num) + await fs.copyFile(join(__dirname, '../node_modules/tachyons/css/tachyons.min.css'), join(argv.dir, 'tachyons.css')) + await fs.outputFile(join(argv.dir, 'index.html'), homePage(added)) +} +/* + +const clone = async ({ baseDir, empty, onlyMissing }) => { + /* + + for (const num of errored) { + for (let i = 0; i < 3; i++) { + try { + const comic = await getComic(i) + await write(comic, dir) + break + } catch (err) { + if (i === 2) { + console.log(`😢 ${num} could not be fetched: ${err.toString()}`) + } + } + } + } + +} + */ + +run() diff --git a/index.js b/index.js deleted file mode 100644 index eae555f..0000000 --- a/index.js +++ /dev/null @@ -1,150 +0,0 @@ -const fetch = require('node-fetch') -const fs = require('fs-extra') -const { basename, join } = require('path') - -const progress = (str) => { - process.stdout.clearLine() - process.stdout.cursorTo(0) - process.stdout.write(str) -} - -const pad = (str, max) => { - str = str.toString() - return str.length < max ? pad('0' + str, max) : str -} - -const fetchLatest = () => { - return fetch(`https://xkcd.com/info.0.json`) - .then(res => res.json()) -} - -const fetchId = (id) => { - return fetch(`https://xkcd.com/${id}/info.0.json`) - .then(res => res.json()) -} - -const htmlPage = ({ alt, title, transcript, num, img }) => { - const btnClass = 'dib navy mh2 pa2 bg-light-blue bg-animate hover-bg-lightest-blue br2 ba bw1 b--navy no-underline' - - return ` - - ${num} - ${title} - - - -

${title} #${num}

- - - - ${alt} -

${transcript}

- -` -} - -const download = async (baseDir, i) => { - const num = pad(i, 4) - const res = await fetchId(i) - const dir = join(baseDir, num) - - try { - await fs.outputJSON(join(dir, 'info.json'), res, { spaces: '\t' }) - - const img = await fetch(res.img) - const dest = fs.createWriteStream(join(dir, basename(res.img))) - - await fs.outputFile(join(dir, 'index.html'), htmlPage(res)) - img.body.pipe(dest) - } catch (err) { - await fs.remove(dir) - throw err - } - - return { - id: i, - title: res.title, - num: num - } -} - -const homePage = (list) => ` - - XKCD - - - -

XKCD

- - ${list.map(({ id, title, num }) => `${id} - ${title}`).join('\n')} - -` - -const clone = async ({ baseDir, empty, onlyMissing }) => { - console.log(`😊 Going to clone XKCD to ${baseDir}`) - - let errored = [] - let added = [] - - try { - console.log(`🔍 Finding the latest comic`) - const latest = (await fetchLatest()).num - console.log(`😁 Found! Will download ${latest} comics 🥶`) - - await fs.ensureDir(baseDir) - if (empty) await fs.emptyDir(baseDir) - - let existent = [] - if (onlyMissing) { - existent = fs.readdirSync(baseDir).map(v => Number(v.split('-')[0].trim())) - } - - for (let i = 1; i <= 10 + 1; i++) { - if (existent.includes(i)) { - continue - } else if (i === 404) { - progress(`📦 404 not found 😵`) - continue - } else if (i === latest + 1) { - progress(`📦 All ${latest} comics fetched\n`) - continue - } else { - progress(`📦 Fetching ${i} out of ${latest}`) - } - - try { - added.push(await download(baseDir, i)) - } catch (err) { - errored.push(i) - } - } - } catch (err) { - console.log(`🐉 ${err.stack}`) - process.exit(1) - } - - for (const num of errored) { - for (let i = 0; i < 3; i++) { - try { - added.push(await download(baseDir, i)) - break - } catch (err) { - if (i === 2) { - console.log(`😢 ${num} could not be fetched: ${err.toString()}`) - } - } - } - } - - added = added.sort((a, b) => a.num - b.num) - await fs.outputFile(join(baseDir, 'index.html'), homePage(added)) -} - -clone({ - baseDir: './clone', - empty: true, - onlyMissing: false -}) diff --git a/lib/helpers.js b/lib/helpers.js new file mode 100644 index 0000000..a137568 --- /dev/null +++ b/lib/helpers.js @@ -0,0 +1,15 @@ +const progress = (str) => { + process.stdout.clearLine() + process.stdout.cursorTo(0) + process.stdout.write(str) +} + +const pad = (str, max) => { + str = str.toString() + return str.length < max ? pad('0' + str, max) : str +} + +module.exports = { + progress, + pad +} diff --git a/lib/html.js b/lib/html.js new file mode 100644 index 0000000..26776fc --- /dev/null +++ b/lib/html.js @@ -0,0 +1,44 @@ +const { basename } = require('path') +const { pad } = require('./helpers') + +const comicPage = ({ alt, title, transcript, num, img }) => { + const btnClass = 'dib navy mh2 pa2 bg-light-blue hover-bg-lightest-blue br2 ba bw1 b--navy no-underline' + + return ` + + ${num} - ${title} + + + +

${title} #${num}

+ + + + ${alt} +

${transcript}

+ +` +} + +const homePage = (list) => ` + + XKCD + + + +

XKCD

+ + + +` + +module.exports = { + comicPage, + homePage +} diff --git a/lib/xkcd.js b/lib/xkcd.js new file mode 100644 index 0000000..cb4c9bb --- /dev/null +++ b/lib/xkcd.js @@ -0,0 +1,20 @@ +const fetch = require('node-fetch') + +async function getLatestId () { + const raw = await fetch(`https://xkcd.com/info.0.json`) + const data = await raw.json() + return data.num +} + +async function getComic (id) { + const raw = await fetch(`https://xkcd.com/${id}/info.0.json`) + const data = await raw.json() + const img = await fetch(data.img) + + return { data, img } +} + +module.exports = { + getLatestId, + getComic +} diff --git a/package.json b/package.json index 1eb7a61..7fea6c7 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,12 @@ "name": "xkcd-clone", "version": "1.0.0", "description": "", - "main": "index.js", - "author": "", - "license": "ISC", + "main": "./lib/index.js", + "bin": { + "xkcd-clone": "./bin/index.js" + }, + "license": "MIT", + "author": "Henrique Dias ", "dependencies": { "fs-extra": "^7.0.1", "node-fetch": "^2.4.0",