Fixes on the deployment process

2023-11-21 19:20:31 +00:00 · 2023-11-21 19:20:31 +00:00 · 4aec99915b
commit 4aec99915b
parent 7cd1380d06
9 changed files with 81 additions and 38 deletions
--- a/.github/files-to-cache.lst
+++ b/.github/files-to-cache.lst
@ -0,0 +1,15 @@
+codemirror-5.60.0/lib/codemirror.css
+codemirror-5.60.0/lib/codemirror.js
+codemirror-5.60.0/mode/javascript/javascript.js
+sha1.js/sha1.js
+pako/pako.min.js
+Viz.js/viz.js
+FileSaver.js/FileSaver.js
+Blob.js/Blob.js
+JSZip/jszip.min.js
+git-tutorial.css
+git-tutorial.js
+sha256.js
+micro_ipfs.js
+directory_hashes.js
+favicon.ico
--- a/.github/pin-using-ipfs.sh
+++ b/.github/pin-using-ipfs.sh
@ -55,22 +55,5 @@ if test -n "${IPFS_REMOTE_API_ENDPOINT:-}" && test -n "${IPFS_REMOTE_TOKEN:-}" &
  done)
 fi

-# warm up cache, twice (a few files in the first attempt would likely fail as the DHT propagation is not instant)
-for i in `seq 2`; do
-  ipfs add --progress=false --ignore-rules-path "result/www/.ipfsignore" --pin=false --hidden -r result/www \
-  | cut -d ' ' -f 3- \
-  | sed -e 's~^www/*~~' \
-  | while read f; do
-    if (printf %s\\n "$IPFS_REMOTE_API_ENDPOINT" | grep pinata) >/dev/null 2>&1; then
-      printf "Warming up pinata cache for %s (attempt %d)...\n" "$f" "$i"
-      wget --tries=1 --timeout=10 -O- "https://gateway.pinata.cloud/ipfs/$h/$f" > /dev/null || true
-    fi
-    printf "Warming up Cloudflare cache for %s (attempt %d)...\n" "$f" "$i"
-    wget --tries=1 --timeout=10 -O- "https://cloudflare-ipfs.com/ipfs/$h/$f" > /dev/null || true
-    printf "Warming up dweb.link cache for %s (attempt %d)...\n" "$f" "$i"
-    wget --tries=1 --timeout=10 -O- "https://$h.ipfs.dweb.link/$f" > /dev/null || true
-  done
-done
-
 # Fail job if one of the pinning services didn't work
 exit "$(cat ipfs-pin-global-exitcode)"
--- a/.github/warm-up-gateway-caches.sh
+++ b/.github/warm-up-gateway-caches.sh
@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euET -o pipefail
+
+echo "Warm up cache on a couple of IPFS gateways"
+
+h="$(result/www/ipfs-add.sh --pin=true)"
+
+#wget --reject-regex ".*\?.*" -r -np --timeout=2 --tries=1 "https://cloudflare-ipfs.com/ipfs/$h" 2>&1 | grep '^--' & pid_cloudflare="$!"
+#wget --reject-regex ".*\?.*" -r -np --timeout=2 --tries=1 "https://$h.ipfs.dweb.link/" 2>&1 | grep '^--' & pid_dweb="$!"
+#wait "$pid_cloudflare" || true
+#wait "$pid_dweb" || true
+
+# Download the files, twice (a few files in the first attempt would likely fail as the DHT propagation is not instantaneous?)
+for i in `seq 2`; do
+  #ipfs add --progress=false --ignore-rules-path "result/www/.ipfsignore" --pin=false --hidden -r result/www \
+  #| cut -d ' ' -f 3- \
+  #| sed -e 's~^www/*~~' \
+  cat .github/files-to-cache.lst \
+  | while read f; do
+    if (printf %s\\n "$IPFS_REMOTE_API_ENDPOINT" | grep pinata) >/dev/null 2>&1; then
+      printf "Warming up pinata cache for %s (attempt %d)...\n" "$f" "$i"
+      wget --tries=1 --timeout=10 -O- "https://gateway.pinata.cloud/ipfs/$h/$f" > /dev/null || true
+    fi
+    printf "Warming up Cloudflare cache for %s (attempt %d)...\n" "$f" "$i"
+    wget --tries=1 --timeout=10 -O- "https://cloudflare-ipfs.com/ipfs/$h/$f" > /dev/null || true
+    printf "Warming up dweb.link cache for %s (attempt %d)...\n" "$f" "$i"
+    wget --tries=1 --timeout=10 -O- "https://$h.ipfs.dweb.link/$f" > /dev/null || true
+  done
+done
--- a/.github/workflows/upload-to-ipfs-and-update-dns.yml
+++ b/.github/workflows/upload-to-ipfs-and-update-dns.yml
@ -41,6 +41,17 @@ jobs:
        run: ./.github/github_install_ipfs.sh
      - name: Print and compare IPFS hahes
        run: export PATH="/tmp/kubo:$PATH"; .github/print-and-compare-ipfs.sh
+      - name: Make tarball of website
+        run: mkdir -p "$RUNNER_TEMP" && tar --directory result/www/ -cvf "$RUNNER_TEMP/artifact.tar" .
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: github-pages
+          path: ${{ runner.temp }}/artifact.tar
+          if-no-files-found: error
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
      - name: Upload to IPFS
        run: PATH="/tmp/kubo:$PATH" ./.github/pin-using-ipfs.sh
        continue-on-error: true
@ -50,7 +61,7 @@ jobs:
          IPFS_REMOTE_TOKEN: ${{ secrets.IPFS_REMOTE_TOKEN }}
      - name: Install OVH pip package
        run: pip install ovh
-      - name: Install OVH pip package
+      - name: Update OVH _dnslink
        run: python ./.github/update-ovh.py >/dev/null 2>&1
        env:
          API_OVH_APPLICATION_KEY: ${{ secrets.API_OVH_APPLICATION_KEY }}
@ -59,17 +70,11 @@ jobs:
          API_OVH_ENDPOINT: ${{ secrets.API_OVH_ENDPOINT }}
          OVH_DNS_DOMAIN: ${{ secrets.OVH_DNS_DOMAIN }}
          OVH_DNS_RECORD_ID: ${{ secrets.OVH_DNS_RECORD_ID }}
+      - name: Warm up IPFS gateway caches
+        run: PATH="/tmp/kubo:$PATH" ./.github/warm-up-gateway-caches.sh
 #      - name: Update homepage URL
 #        run: PATH="/tmp/kubo:$PATH" ./.github/github_update_homepage.sh
 #        env:
 #          API_TOKEN_FOR_UPDATE_HOMEPAGE: ${{ secrets.API_TOKEN_FOR_UPDATE_HOMEPAGE }}
 #      - name: Setup Pages
 #        uses: actions/configure-pages@v3
-#      - name: Upload artifact
-#        uses: actions/upload-pages-artifact@v2
-#        with:
-#          # Upload entire repository
-#          path: 'result/www/'
-#      - name: Deploy to GitHub Pages
-#        id: deployment
-#        uses: actions/deploy-pages@v2
--- a/build-and-update.sh
+++ b/build-and-update.sh
@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euET -o pipefail
+
+nix build
+cp result/www/directory_hashes.js directory_hashes.js
+cp result/www/favicon.ico favicon.ico
+cp result/www/sitemap.html sitemap.html
+if test -n "$(git status --short)"; then git commit -a --amend; fi
+nix build
+diff result/www/directory_hashes.js directory_hashes.js
+diff result/www/favicon.ico favicon.ico
+diff result/www/sitemap.html sitemap.html
+
+./.github/print-and-compare-ipfs.sh
--- a/deploy.sh
+++ b/deploy.sh
@ -17,17 +17,7 @@ if ! grep '<a id="this-version" href="https://github.com/jsmaniac/git-tutorial/t
  exit 1
 fi

-nix build
-cp result/www/directory_hashes.js directory_hashes.js
-cp result/www/favicon.ico favicon.ico
-cp result/www/sitemap.html sitemap.html
-if test -n "$(git status --short)"; then git commit -a --amend; fi
-nix build
-diff result/www/directory_hashes.js directory_hashes.js
-diff result/www/favicon.ico favicon.ico
-diff result/www/sitemap.html sitemap.html
-
-./.github/print-and-compare-ipfs.sh
+./build-and-update.sh

 # Add to IPFS and get the hash
 ipfs_hash="$(./result/www/ipfs-add.sh --pin=true)"
--- a/directory_hashes.js
+++ b/directory_hashes.js
@ -1 +1 @@
-jsonp_ipfs_directory_hashes({"vanity_text":"soy","vanity_number":2035,"tree":{"Links":[{"Name":".github","Hash":"QmPVN8KzBJhCezPsFbNHfJ38LRJFyWhVwkpGQdWE6nSRh3","Size":9433},{"Name":".gitignore","Hash":"QmW9iMXzmPqLSnzL4p6DKsvsL3nC1xKS3teRB4SRdukfrz","Size":16},{"Name":".ipfsignore","Hash":"QmPpQN29FbeaNwGsXbebbv588UZtSLCGRffa3Zrz68RAMp","Size":22},{"Name":".nojekyll","Hash":"QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH","Size":6},{"Name":"Blob.js","Hash":"QmSxKmtcBxBRkmkLGpnXAGrRc16kPrJx5Hmdsvt6LRWeSG","Size":21833},{"Name":"CNAME","Hash":"QmT3mZdxv3tQQGq9mwMmjnhXDaDAnkrcwmh2Hvy8gQhRyr","Size":32},{"Name":"FileSaver.js","Hash":"QmUgg2HLo4W9bpf92CkHH6WWVFfg2DmNqJrz2Z46L7VmUq","Size":7367},{"Name":"JSZip","Hash":"QmWW2hDPrMU5e5KgSAMiqfM2YW5RSiZzWiNJSQ7w63ngiL","Size":422094},{"Name":"README","Hash":"QmVPSdJVXbYuQSN5v3VXqZcYKJuuzvv4ZArg8S7u1MF85F","Size":810},{"Name":"Viz.js","Hash":"QmaxUCu1gnFwTTpDoTAPB3fMQQav1NJZrZ7LGqLXECidKj","Size":3564410},{"Name":"codemirror-5.60.0","Hash":"QmXPbArMAid8MbC5G7HCyWz2PUkfSMWZaUQpnq63x8Dw2y","Size":4669604},{"Name":"deploy.sh","Hash":"QmedckMFRS5rBkuv4DVqHxff11ouWZSitr2SNKvnYot56j","Size":1273},{"Name":"directory_hashes.js","Hash":"","Size":0},{"Name":"favicon.ico","Hash":"QmUq6pQamF58ZDNpPSvF3C2bcCWEJSjx3dFZZLjkSCrYpi","Size":32052},{"Name":"favicon.svg","Hash":"QmesnKGtStCZGpiTjoAcAETdSZgUUQ3wzekn1LSQMFtbgn","Size":3272},{"Name":"flake.lock","Hash":"QmdkX8PkV6j2sLH1JSPD1z4533rEGTa6JKSfsJcYAGSrvx","Size":1475},{"Name":"flake.nix","Hash":"Qmf28V3ScemKqCCq2x7uAYPSikHv9tYbWwo78MsxqvKaZf","Size":1556},{"Name":"git-tutorial.css","Hash":"QmdsWg4RVZR3kRA7xFchoWLEQQzLpzVyspTtKwa9qttDMF","Size":10842},{"Name":"git-tutorial.js","Hash":"QmTpny5DSeUzCULRtYH2YJSgLx57KuYPr2LRR7N2A2K4Qg","Size":47919},{"Name":"index.html","Hash":"QmSAqYiSkbR1xhZU6FLMUFTtfXBNXrbqtYb6hJ78Zq1ibB","Size":118368},{"Name":"ipfs-add.sh","Hash":"QmXSLYLy13efSFVEN3Ej3A3vyimH618Vrt82hoBKeKYgDB","Size":473},{"Name":"micro_ipfs.js","Hash":"QmeWPj4vzN66eCUwQkjjzTgfciBLBzNjQQdvqEBL8x1pmh","Size":16738},{"Name":"pako","Hash":"QmRtJhu2rJCe59JPS9UiyAja5iUZNmJ8nyBijdZpLLEgG9","Size":178431},{"Name":"sha1.js","Hash":"QmP7HPPYQqwKXYyDrkDm9vKt8FZE1WsDUJG8cLnjFf4a11","Size":7966},{"Name":"sha256.js","Hash":"QmRhgx5Fq4JqfCgsPcMxNSYwt8M9WRBkec9omPWzJ7gdwL","Size":8553},{"Name":"sitemap.html","Hash":"QmWgCZfJ7TkX4hpXj71Q1FnZ5ccZeKSik3oXSnXein4XW4","Size":71584}],"Data":"\b\u0001"}});
+jsonp_ipfs_directory_hashes({"vanity_text":"soy","vanity_number":862,"tree":{"Links":[{"Name":".github","Hash":"QmVQymgJYtsWZiVSaxdXxCrovGsdsxTPNWW9K2LXmNWmrk","Size":10687},{"Name":".gitignore","Hash":"QmW9iMXzmPqLSnzL4p6DKsvsL3nC1xKS3teRB4SRdukfrz","Size":16},{"Name":".ipfsignore","Hash":"QmPpQN29FbeaNwGsXbebbv588UZtSLCGRffa3Zrz68RAMp","Size":22},{"Name":".nojekyll","Hash":"QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH","Size":6},{"Name":"Blob.js","Hash":"QmSxKmtcBxBRkmkLGpnXAGrRc16kPrJx5Hmdsvt6LRWeSG","Size":21833},{"Name":"CNAME","Hash":"QmT3mZdxv3tQQGq9mwMmjnhXDaDAnkrcwmh2Hvy8gQhRyr","Size":32},{"Name":"FileSaver.js","Hash":"QmUgg2HLo4W9bpf92CkHH6WWVFfg2DmNqJrz2Z46L7VmUq","Size":7367},{"Name":"JSZip","Hash":"QmWW2hDPrMU5e5KgSAMiqfM2YW5RSiZzWiNJSQ7w63ngiL","Size":422094},{"Name":"README","Hash":"QmVPSdJVXbYuQSN5v3VXqZcYKJuuzvv4ZArg8S7u1MF85F","Size":810},{"Name":"Viz.js","Hash":"QmaxUCu1gnFwTTpDoTAPB3fMQQav1NJZrZ7LGqLXECidKj","Size":3564410},{"Name":"build-and-update.sh","Hash":"QmRypwTrHQq6Ftrb3ygJZ2E4haqqvNZxxZrm4sVnbJuaHH","Size":448},{"Name":"codemirror-5.60.0","Hash":"QmXPbArMAid8MbC5G7HCyWz2PUkfSMWZaUQpnq63x8Dw2y","Size":4669604},{"Name":"deploy.sh","Hash":"QmdgLSVN4UedbKuyAbsCQhiHjxdWpyxMqjdW4cDsvfzDNF","Size":901},{"Name":"directory_hashes.js","Hash":"","Size":0},{"Name":"favicon.ico","Hash":"QmUq6pQamF58ZDNpPSvF3C2bcCWEJSjx3dFZZLjkSCrYpi","Size":32052},{"Name":"favicon.svg","Hash":"QmesnKGtStCZGpiTjoAcAETdSZgUUQ3wzekn1LSQMFtbgn","Size":3272},{"Name":"flake.lock","Hash":"QmdkX8PkV6j2sLH1JSPD1z4533rEGTa6JKSfsJcYAGSrvx","Size":1475},{"Name":"flake.nix","Hash":"Qmaa8dpTGb7T2KJLiPxjR4HrmGmffDWHrCn2G5FmhjhZXb","Size":1642},{"Name":"git-tutorial.css","Hash":"QmdsWg4RVZR3kRA7xFchoWLEQQzLpzVyspTtKwa9qttDMF","Size":10842},{"Name":"git-tutorial.js","Hash":"QmTpny5DSeUzCULRtYH2YJSgLx57KuYPr2LRR7N2A2K4Qg","Size":47919},{"Name":"index.html","Hash":"QmSAqYiSkbR1xhZU6FLMUFTtfXBNXrbqtYb6hJ78Zq1ibB","Size":118368},{"Name":"ipfs-add.sh","Hash":"QmXSLYLy13efSFVEN3Ej3A3vyimH618Vrt82hoBKeKYgDB","Size":473},{"Name":"micro_ipfs.js","Hash":"QmeWPj4vzN66eCUwQkjjzTgfciBLBzNjQQdvqEBL8x1pmh","Size":16738},{"Name":"pako","Hash":"QmRtJhu2rJCe59JPS9UiyAja5iUZNmJ8nyBijdZpLLEgG9","Size":178431},{"Name":"sha1.js","Hash":"QmP7HPPYQqwKXYyDrkDm9vKt8FZE1WsDUJG8cLnjFf4a11","Size":7966},{"Name":"sha256.js","Hash":"QmRhgx5Fq4JqfCgsPcMxNSYwt8M9WRBkec9omPWzJ7gdwL","Size":8553},{"Name":"sitemap.html","Hash":"QmdxctnkjqpH4JrwhnxA8NHuGK3QqUTe2nnGXuWKc6pM3k","Size":71800}],"Data":"\b\u0001"}});
--- a/flake.nix
+++ b/flake.nix
@ -10,6 +10,8 @@
        src = self;
        buildInputs = with pkgs; [kubo jq nodejs-slim imagemagick];
        buildPhase = ''
+          # TODO: remove files ignored by .ipfsignore during build process
+          
          convert -background none favicon.svg -define icon:auto-resize=64,48,32,16 favicon.ico

          mkdir "$out"
--- a/sitemap.html
+++ b/sitemap.html
@ -1,11 +1,13 @@
 <!DOCTYPE html><html><head><title>Sitemap</title></head><body>
 <a href=".">.</a>
 <a href="./.github">./.github</a>
+<a href="./.github/files-to-cache.lst">./.github/files-to-cache.lst</a>
 <a href="./.github/github_install_ipfs.sh">./.github/github_install_ipfs.sh</a>
 <a href="./.github/github_update_homepage.sh">./.github/github_update_homepage.sh</a>
 <a href="./.github/pin-using-ipfs.sh">./.github/pin-using-ipfs.sh</a>
 <a href="./.github/print-and-compare-ipfs.sh">./.github/print-and-compare-ipfs.sh</a>
 <a href="./.github/update-ovh.py">./.github/update-ovh.py</a>
+<a href="./.github/warm-up-gateway-caches.sh">./.github/warm-up-gateway-caches.sh</a>
 <a href="./.github/workflows">./.github/workflows</a>
 <a href="./.github/workflows/upload-to-ipfs-and-update-dns.yml">./.github/workflows/upload-to-ipfs-and-update-dns.yml</a>
 <a href="./.gitignore">./.gitignore</a>
@ -27,6 +29,7 @@
 <a href="./Viz.js/LICENSE">./Viz.js/LICENSE</a>
 <a href="./Viz.js/viz-lite.js">./Viz.js/viz-lite.js</a>
 <a href="./Viz.js/viz.js">./Viz.js/viz.js</a>
+<a href="./build-and-update.sh">./build-and-update.sh</a>
 <a href="./codemirror-5.60.0">./codemirror-5.60.0</a>
 <a href="./codemirror-5.60.0/.editorconfig">./codemirror-5.60.0/.editorconfig</a>
 <a href="./codemirror-5.60.0/.gitattributes">./codemirror-5.60.0/.gitattributes</a>