From 4aec99915bca75a6f9f9ed27f34d0b8693f22d12 Mon Sep 17 00:00:00 2001
From: Suzanne Soy <jsmaniac.github@suzanne.soy>
Date: Tue, 21 Nov 2023 19:20:31 +0000
Subject: [PATCH] Fixes on the deployment process
---
.github/files-to-cache.lst | 15 ++++++++++
.github/pin-using-ipfs.sh | 17 -----------
.github/warm-up-gateway-caches.sh | 30 +++++++++++++++++++
.../upload-to-ipfs-and-update-dns.yml | 23 ++++++++------
build-and-update.sh | 15 ++++++++++
deploy.sh | 12 +-------
directory_hashes.js | 2 +-
flake.nix | 2 ++
sitemap.html | 3 ++
9 files changed, 81 insertions(+), 38 deletions(-)
create mode 100644 .github/files-to-cache.lst
create mode 100755 .github/warm-up-gateway-caches.sh
create mode 100755 build-and-update.sh
diff --git a/.github/files-to-cache.lst b/.github/files-to-cache.lst
new file mode 100644
index 0000000..8b1edff
--- /dev/null
+++ b/.github/files-to-cache.lst
@@ -0,0 +1,15 @@
+codemirror-5.60.0/lib/codemirror.css
+codemirror-5.60.0/lib/codemirror.js
+codemirror-5.60.0/mode/javascript/javascript.js
+sha1.js/sha1.js
+pako/pako.min.js
+Viz.js/viz.js
+FileSaver.js/FileSaver.js
+Blob.js/Blob.js
+JSZip/jszip.min.js
+git-tutorial.css
+git-tutorial.js
+sha256.js
+micro_ipfs.js
+directory_hashes.js
+favicon.ico
\ No newline at end of file
diff --git a/.github/pin-using-ipfs.sh b/.github/pin-using-ipfs.sh
index 0c09568..b84a129 100755
--- a/.github/pin-using-ipfs.sh
+++ b/.github/pin-using-ipfs.sh
@@ -55,22 +55,5 @@ if test -n "${IPFS_REMOTE_API_ENDPOINT:-}" && test -n "${IPFS_REMOTE_TOKEN:-}" &
done)
fi
-# warm up cache, twice (a few files in the first attempt would likely fail as the DHT propagation is not instant)
-for i in `seq 2`; do
- ipfs add --progress=false --ignore-rules-path "result/www/.ipfsignore" --pin=false --hidden -r result/www \
- | cut -d ' ' -f 3- \
- | sed -e 's~^www/*~~' \
- | while read f; do
- if (printf %s\\n "$IPFS_REMOTE_API_ENDPOINT" | grep pinata) >/dev/null 2>&1; then
- printf "Warming up pinata cache for %s (attempt %d)...\n" "$f" "$i"
- wget --tries=1 --timeout=10 -O- "https://gateway.pinata.cloud/ipfs/$h/$f" > /dev/null || true
- fi
- printf "Warming up Cloudflare cache for %s (attempt %d)...\n" "$f" "$i"
- wget --tries=1 --timeout=10 -O- "https://cloudflare-ipfs.com/ipfs/$h/$f" > /dev/null || true
- printf "Warming up dweb.link cache for %s (attempt %d)...\n" "$f" "$i"
- wget --tries=1 --timeout=10 -O- "https://$h.ipfs.dweb.link/$f" > /dev/null || true
- done
-done
-
# Fail job if one of the pinning services didn't work
exit "$(cat ipfs-pin-global-exitcode)"
diff --git a/.github/warm-up-gateway-caches.sh b/.github/warm-up-gateway-caches.sh
new file mode 100755
index 0000000..072176a
--- /dev/null
+++ b/.github/warm-up-gateway-caches.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euET -o pipefail
+
+echo "Warm up cache on a couple of IPFS gateways"
+
+h="$(result/www/ipfs-add.sh --pin=true)"
+
+#wget --reject-regex ".*\?.*" -r -np --timeout=2 --tries=1 "https://cloudflare-ipfs.com/ipfs/$h" 2>&1 | grep '^--' & pid_cloudflare="$!"
+#wget --reject-regex ".*\?.*" -r -np --timeout=2 --tries=1 "https://$h.ipfs.dweb.link/" 2>&1 | grep '^--' & pid_dweb="$!"
+#wait "$pid_cloudflare" || true
+#wait "$pid_dweb" || true
+
+# Download the files, twice (a few files in the first attempt would likely fail as the DHT propagation is not instantaneous?)
+for i in `seq 2`; do
+ #ipfs add --progress=false --ignore-rules-path "result/www/.ipfsignore" --pin=false --hidden -r result/www \
+ #| cut -d ' ' -f 3- \
+ #| sed -e 's~^www/*~~' \
+ cat .github/files-to-cache.lst \
+ | while read f; do
+ if (printf %s\\n "$IPFS_REMOTE_API_ENDPOINT" | grep pinata) >/dev/null 2>&1; then
+ printf "Warming up pinata cache for %s (attempt %d)...\n" "$f" "$i"
+ wget --tries=1 --timeout=10 -O- "https://gateway.pinata.cloud/ipfs/$h/$f" > /dev/null || true
+ fi
+ printf "Warming up Cloudflare cache for %s (attempt %d)...\n" "$f" "$i"
+ wget --tries=1 --timeout=10 -O- "https://cloudflare-ipfs.com/ipfs/$h/$f" > /dev/null || true
+ printf "Warming up dweb.link cache for %s (attempt %d)...\n" "$f" "$i"
+ wget --tries=1 --timeout=10 -O- "https://$h.ipfs.dweb.link/$f" > /dev/null || true
+ done
+done
diff --git a/.github/workflows/upload-to-ipfs-and-update-dns.yml b/.github/workflows/upload-to-ipfs-and-update-dns.yml
index f31192c..59e5a6c 100644
--- a/.github/workflows/upload-to-ipfs-and-update-dns.yml
+++ b/.github/workflows/upload-to-ipfs-and-update-dns.yml
@@ -41,6 +41,17 @@ jobs:
run: ./.github/github_install_ipfs.sh
- name: Print and compare IPFS hahes
run: export PATH="/tmp/kubo:$PATH"; .github/print-and-compare-ipfs.sh
+ - name: Make tarball of website
+ run: mkdir -p "$RUNNER_TEMP" && tar --directory result/www/ -cvf "$RUNNER_TEMP/artifact.tar" .
+ - name: Upload artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: github-pages
+ path: ${{ runner.temp }}/artifact.tar
+ if-no-files-found: error
+ - name: Deploy to GitHub Pages
+ id: deployment
+ uses: actions/deploy-pages@v2
- name: Upload to IPFS
run: PATH="/tmp/kubo:$PATH" ./.github/pin-using-ipfs.sh
continue-on-error: true
@@ -50,7 +61,7 @@ jobs:
IPFS_REMOTE_TOKEN: ${{ secrets.IPFS_REMOTE_TOKEN }}
- name: Install OVH pip package
run: pip install ovh
- - name: Install OVH pip package
+ - name: Update OVH _dnslink
run: python ./.github/update-ovh.py >/dev/null 2>&1
env:
API_OVH_APPLICATION_KEY: ${{ secrets.API_OVH_APPLICATION_KEY }}
@@ -59,17 +70,11 @@ jobs:
API_OVH_ENDPOINT: ${{ secrets.API_OVH_ENDPOINT }}
OVH_DNS_DOMAIN: ${{ secrets.OVH_DNS_DOMAIN }}
OVH_DNS_RECORD_ID: ${{ secrets.OVH_DNS_RECORD_ID }}
+ - name: Warm up IPFS gateway caches
+ run: PATH="/tmp/kubo:$PATH" ./.github/warm-up-gateway-caches.sh
# - name: Update homepage URL
# run: PATH="/tmp/kubo:$PATH" ./.github/github_update_homepage.sh
# env:
# API_TOKEN_FOR_UPDATE_HOMEPAGE: ${{ secrets.API_TOKEN_FOR_UPDATE_HOMEPAGE }}
# - name: Setup Pages
# uses: actions/configure-pages@v3
-# - name: Upload artifact
-# uses: actions/upload-pages-artifact@v2
-# with:
-# # Upload entire repository
-# path: 'result/www/'
-# - name: Deploy to GitHub Pages
-# id: deployment
-# uses: actions/deploy-pages@v2
diff --git a/build-and-update.sh b/build-and-update.sh
new file mode 100755
index 0000000..53c66cc
--- /dev/null
+++ b/build-and-update.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euET -o pipefail
+
+nix build
+cp result/www/directory_hashes.js directory_hashes.js
+cp result/www/favicon.ico favicon.ico
+cp result/www/sitemap.html sitemap.html
+if test -n "$(git status --short)"; then git commit -a --amend; fi
+nix build
+diff result/www/directory_hashes.js directory_hashes.js
+diff result/www/favicon.ico favicon.ico
+diff result/www/sitemap.html sitemap.html
+
+./.github/print-and-compare-ipfs.sh
\ No newline at end of file
diff --git a/deploy.sh b/deploy.sh
index 767ce1e..ef5e2b9 100755
--- a/deploy.sh
+++ b/deploy.sh
@@ -17,17 +17,7 @@ if ! grep '<a id="this-version" href="https://github.com/jsmaniac/git-tutorial/t
exit 1
fi
-nix build
-cp result/www/directory_hashes.js directory_hashes.js
-cp result/www/favicon.ico favicon.ico
-cp result/www/sitemap.html sitemap.html
-if test -n "$(git status --short)"; then git commit -a --amend; fi
-nix build
-diff result/www/directory_hashes.js directory_hashes.js
-diff result/www/favicon.ico favicon.ico
-diff result/www/sitemap.html sitemap.html
-
-./.github/print-and-compare-ipfs.sh
+./build-and-update.sh
# Add to IPFS and get the hash
ipfs_hash="$(./result/www/ipfs-add.sh --pin=true)"
diff --git a/directory_hashes.js b/directory_hashes.js
index b9a20ca..7b02054 100644
--- a/directory_hashes.js
+++ b/directory_hashes.js
@@ -1 +1 @@
-jsonp_ipfs_directory_hashes({"vanity_text":"soy","vanity_number":2035,"tree":{"Links":[{"Name":".github","Hash":"QmPVN8KzBJhCezPsFbNHfJ38LRJFyWhVwkpGQdWE6nSRh3","Size":9433},{"Name":".gitignore","Hash":"QmW9iMXzmPqLSnzL4p6DKsvsL3nC1xKS3teRB4SRdukfrz","Size":16},{"Name":".ipfsignore","Hash":"QmPpQN29FbeaNwGsXbebbv588UZtSLCGRffa3Zrz68RAMp","Size":22},{"Name":".nojekyll","Hash":"QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH","Size":6},{"Name":"Blob.js","Hash":"QmSxKmtcBxBRkmkLGpnXAGrRc16kPrJx5Hmdsvt6LRWeSG","Size":21833},{"Name":"CNAME","Hash":"QmT3mZdxv3tQQGq9mwMmjnhXDaDAnkrcwmh2Hvy8gQhRyr","Size":32},{"Name":"FileSaver.js","Hash":"QmUgg2HLo4W9bpf92CkHH6WWVFfg2DmNqJrz2Z46L7VmUq","Size":7367},{"Name":"JSZip","Hash":"QmWW2hDPrMU5e5KgSAMiqfM2YW5RSiZzWiNJSQ7w63ngiL","Size":422094},{"Name":"README","Hash":"QmVPSdJVXbYuQSN5v3VXqZcYKJuuzvv4ZArg8S7u1MF85F","Size":810},{"Name":"Viz.js","Hash":"QmaxUCu1gnFwTTpDoTAPB3fMQQav1NJZrZ7LGqLXECidKj","Size":3564410},{"Name":"codemirror-5.60.0","Hash":"QmXPbArMAid8MbC5G7HCyWz2PUkfSMWZaUQpnq63x8Dw2y","Size":4669604},{"Name":"deploy.sh","Hash":"QmedckMFRS5rBkuv4DVqHxff11ouWZSitr2SNKvnYot56j","Size":1273},{"Name":"directory_hashes.js","Hash":"","Size":0},{"Name":"favicon.ico","Hash":"QmUq6pQamF58ZDNpPSvF3C2bcCWEJSjx3dFZZLjkSCrYpi","Size":32052},{"Name":"favicon.svg","Hash":"QmesnKGtStCZGpiTjoAcAETdSZgUUQ3wzekn1LSQMFtbgn","Size":3272},{"Name":"flake.lock","Hash":"QmdkX8PkV6j2sLH1JSPD1z4533rEGTa6JKSfsJcYAGSrvx","Size":1475},{"Name":"flake.nix","Hash":"Qmf28V3ScemKqCCq2x7uAYPSikHv9tYbWwo78MsxqvKaZf","Size":1556},{"Name":"git-tutorial.css","Hash":"QmdsWg4RVZR3kRA7xFchoWLEQQzLpzVyspTtKwa9qttDMF","Size":10842},{"Name":"git-tutorial.js","Hash":"QmTpny5DSeUzCULRtYH2YJSgLx57KuYPr2LRR7N2A2K4Qg","Size":47919},{"Name":"index.html","Hash":"QmSAqYiSkbR1xhZU6FLMUFTtfXBNXrbqtYb6hJ78Zq1ibB","Size":118368},{"Name":"ipfs-add.sh","Hash":"QmXSLYLy13efSFVEN3Ej3A3vyimH618Vrt82hoBKeKYgDB","Size":473},{"Name":"micro_ipfs.js","Hash":"QmeWPj4vzN66eCUwQkjjzTgfciBLBzNjQQdvqEBL8x1pmh","Size":16738},{"Name":"pako","Hash":"QmRtJhu2rJCe59JPS9UiyAja5iUZNmJ8nyBijdZpLLEgG9","Size":178431},{"Name":"sha1.js","Hash":"QmP7HPPYQqwKXYyDrkDm9vKt8FZE1WsDUJG8cLnjFf4a11","Size":7966},{"Name":"sha256.js","Hash":"QmRhgx5Fq4JqfCgsPcMxNSYwt8M9WRBkec9omPWzJ7gdwL","Size":8553},{"Name":"sitemap.html","Hash":"QmWgCZfJ7TkX4hpXj71Q1FnZ5ccZeKSik3oXSnXein4XW4","Size":71584}],"Data":"\b\u0001"}});
+jsonp_ipfs_directory_hashes({"vanity_text":"soy","vanity_number":862,"tree":{"Links":[{"Name":".github","Hash":"QmVQymgJYtsWZiVSaxdXxCrovGsdsxTPNWW9K2LXmNWmrk","Size":10687},{"Name":".gitignore","Hash":"QmW9iMXzmPqLSnzL4p6DKsvsL3nC1xKS3teRB4SRdukfrz","Size":16},{"Name":".ipfsignore","Hash":"QmPpQN29FbeaNwGsXbebbv588UZtSLCGRffa3Zrz68RAMp","Size":22},{"Name":".nojekyll","Hash":"QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH","Size":6},{"Name":"Blob.js","Hash":"QmSxKmtcBxBRkmkLGpnXAGrRc16kPrJx5Hmdsvt6LRWeSG","Size":21833},{"Name":"CNAME","Hash":"QmT3mZdxv3tQQGq9mwMmjnhXDaDAnkrcwmh2Hvy8gQhRyr","Size":32},{"Name":"FileSaver.js","Hash":"QmUgg2HLo4W9bpf92CkHH6WWVFfg2DmNqJrz2Z46L7VmUq","Size":7367},{"Name":"JSZip","Hash":"QmWW2hDPrMU5e5KgSAMiqfM2YW5RSiZzWiNJSQ7w63ngiL","Size":422094},{"Name":"README","Hash":"QmVPSdJVXbYuQSN5v3VXqZcYKJuuzvv4ZArg8S7u1MF85F","Size":810},{"Name":"Viz.js","Hash":"QmaxUCu1gnFwTTpDoTAPB3fMQQav1NJZrZ7LGqLXECidKj","Size":3564410},{"Name":"build-and-update.sh","Hash":"QmRypwTrHQq6Ftrb3ygJZ2E4haqqvNZxxZrm4sVnbJuaHH","Size":448},{"Name":"codemirror-5.60.0","Hash":"QmXPbArMAid8MbC5G7HCyWz2PUkfSMWZaUQpnq63x8Dw2y","Size":4669604},{"Name":"deploy.sh","Hash":"QmdgLSVN4UedbKuyAbsCQhiHjxdWpyxMqjdW4cDsvfzDNF","Size":901},{"Name":"directory_hashes.js","Hash":"","Size":0},{"Name":"favicon.ico","Hash":"QmUq6pQamF58ZDNpPSvF3C2bcCWEJSjx3dFZZLjkSCrYpi","Size":32052},{"Name":"favicon.svg","Hash":"QmesnKGtStCZGpiTjoAcAETdSZgUUQ3wzekn1LSQMFtbgn","Size":3272},{"Name":"flake.lock","Hash":"QmdkX8PkV6j2sLH1JSPD1z4533rEGTa6JKSfsJcYAGSrvx","Size":1475},{"Name":"flake.nix","Hash":"Qmaa8dpTGb7T2KJLiPxjR4HrmGmffDWHrCn2G5FmhjhZXb","Size":1642},{"Name":"git-tutorial.css","Hash":"QmdsWg4RVZR3kRA7xFchoWLEQQzLpzVyspTtKwa9qttDMF","Size":10842},{"Name":"git-tutorial.js","Hash":"QmTpny5DSeUzCULRtYH2YJSgLx57KuYPr2LRR7N2A2K4Qg","Size":47919},{"Name":"index.html","Hash":"QmSAqYiSkbR1xhZU6FLMUFTtfXBNXrbqtYb6hJ78Zq1ibB","Size":118368},{"Name":"ipfs-add.sh","Hash":"QmXSLYLy13efSFVEN3Ej3A3vyimH618Vrt82hoBKeKYgDB","Size":473},{"Name":"micro_ipfs.js","Hash":"QmeWPj4vzN66eCUwQkjjzTgfciBLBzNjQQdvqEBL8x1pmh","Size":16738},{"Name":"pako","Hash":"QmRtJhu2rJCe59JPS9UiyAja5iUZNmJ8nyBijdZpLLEgG9","Size":178431},{"Name":"sha1.js","Hash":"QmP7HPPYQqwKXYyDrkDm9vKt8FZE1WsDUJG8cLnjFf4a11","Size":7966},{"Name":"sha256.js","Hash":"QmRhgx5Fq4JqfCgsPcMxNSYwt8M9WRBkec9omPWzJ7gdwL","Size":8553},{"Name":"sitemap.html","Hash":"QmdxctnkjqpH4JrwhnxA8NHuGK3QqUTe2nnGXuWKc6pM3k","Size":71800}],"Data":"\b\u0001"}});
diff --git a/flake.nix b/flake.nix
index 4ae1980..4316cd4 100644
--- a/flake.nix
+++ b/flake.nix
@@ -10,6 +10,8 @@
src = self;
buildInputs = with pkgs; [kubo jq nodejs-slim imagemagick];
buildPhase = ''
+ # TODO: remove files ignored by .ipfsignore during build process
+
convert -background none favicon.svg -define icon:auto-resize=64,48,32,16 favicon.ico
mkdir "$out"
diff --git a/sitemap.html b/sitemap.html
index 336ae5b..b99fbe3 100644
--- a/sitemap.html
+++ b/sitemap.html
@@ -1,11 +1,13 @@
<!DOCTYPE html><html><head><title>Sitemap</title></head><body>
<a href=".">.</a>
<a href="./.github">./.github</a>
+<a href="./.github/files-to-cache.lst">./.github/files-to-cache.lst</a>
<a href="./.github/github_install_ipfs.sh">./.github/github_install_ipfs.sh</a>
<a href="./.github/github_update_homepage.sh">./.github/github_update_homepage.sh</a>
<a href="./.github/pin-using-ipfs.sh">./.github/pin-using-ipfs.sh</a>
<a href="./.github/print-and-compare-ipfs.sh">./.github/print-and-compare-ipfs.sh</a>
<a href="./.github/update-ovh.py">./.github/update-ovh.py</a>
+<a href="./.github/warm-up-gateway-caches.sh">./.github/warm-up-gateway-caches.sh</a>
<a href="./.github/workflows">./.github/workflows</a>
<a href="./.github/workflows/upload-to-ipfs-and-update-dns.yml">./.github/workflows/upload-to-ipfs-and-update-dns.yml</a>
<a href="./.gitignore">./.gitignore</a>
@@ -27,6 +29,7 @@
<a href="./Viz.js/LICENSE">./Viz.js/LICENSE</a>
<a href="./Viz.js/viz-lite.js">./Viz.js/viz-lite.js</a>
<a href="./Viz.js/viz.js">./Viz.js/viz.js</a>
+<a href="./build-and-update.sh">./build-and-update.sh</a>
<a href="./codemirror-5.60.0">./codemirror-5.60.0</a>
<a href="./codemirror-5.60.0/.editorconfig">./codemirror-5.60.0/.editorconfig</a>
<a href="./codemirror-5.60.0/.gitattributes">./codemirror-5.60.0/.gitattributes</a>