Finish static content replication

README.md

@@ -30,6 +30,19 @@ Keys useful for deployment:
  - *recent-seconds*: number, in seconds; default 172800. Packages
    modified fewer than this many seconds ago are considered "recent",
    and displayed as such in the UI.
+ - *static-content-target-directory*: either `#f` or a string denoting
+   a path to a folder to which the static content of the site will be
+   copied.
+ - *static-content-update-hook*: either `#f`, or a string containing a
+   shell command to invoke every time files are updated in
+   *static-content-target-directory*.
+ - *dynamic-urlprefix*: string; absolute or relative URL, prepended to
+   URLs targetting dynamic content on the site.
+ - *static-urlprefix*: string; absolute or relative URL, prepended to
+   relative URLs referring to static HTML files placed in
+   `static-generated-directory`.
+ - *extra-static-content-directories*: list of strings; defaults to
+   the empty list.
 
 Keys useful for development:
 
@@ -40,11 +53,6 @@ Keys useful for development:
  - *static-generated-directory*: string; names a directory relative to
    `src/` within which generated static HTML files are to be placed.
    Must be writable by the user running the server.
- - *static-urlprefix*: string; absolute or relative URL, prepended to
-   relative URLs referring to static HTML files placed in
-   `static-generated-directory`.
- - *dynamic-urlprefix*: string; absolute or relative URL, prepended to
-   URLs targetting dynamic content on the site.
  - *disable-cache?*: boolean; default `#f`.
  - *backend-baseurl*: string; default `https://pkgd.racket-lang.org`.
    Must point to the backend package server API root, such that (for
@@ -83,6 +91,31 @@ compile` at all previously.
 
 ## Deployment
 
+### Static Content
+
+The site can be set up to run either
+
+ 0. entirely dynamically, generating package pages on-the-fly for each
+    request;
+ 0. both statically and dynamically, with HTML renderings of package
+    pages stored on and served from disk like other static resources
+    such as Javascript and CSS; or
+ 0. both statically and dynamically, as the previous option, but
+    additionally replicating both static and generated content to a
+    local file-system directory and invoking an optional update hook
+    that can be used to further replicate the content to S3 or a
+    remote host.
+
+The default is mixed static/dynamic, with no additional replication.
+
+For a fully dynamic site, set configuration variable `disable-cache?`
+to `#t`.
+
+To enable replication, set configuration variable
+`static-content-target-directory` to a non-`#f` value, and optionally
+set `static-content-update-hook` to a string containing a shell
+command to execute every time the static content is updated.
+
 ### Supervision
 
 Startable using djb's [daemontools](http://cr.yp.to/daemontools.html);

configs/tonyg.rkt

@@ -3,11 +3,12 @@
 (require "../src/main.rkt")
 (main (hash 'port 8444
             'reloadable? #t
-            'package-index-url "http://localhost/~tonyg/pkg-catalog-static/pkgs-all.json.gz"
-            'static-content-target-type 'directory
-            'static-content-target-location (build-path (find-system-path 'home-dir)
-                                                        "public_html/pkg-catalog-static")
-            'static-urlprefix "http://localhost/~tonyg/pkg-catalog-static"
+            'package-index-url "https://localhost:8444/pkgs-all.json.gz"
+            'static-content-target-directory (build-path (find-system-path 'home-dir)
+                                                         "public_html/pkg-catalog-static")
+            'static-urlprefix "https://localhost/~tonyg/pkg-catalog-static"
             'dynamic-urlprefix "https://localhost:8444"
             'backend-baseurl "https://localhost:8445"
+            'extra-static-content-directories (list (build-path (find-system-path 'home-dir)
+                                                                "public_html/pkg-index-static"))
             ))

src/site.rkt

@@ -44,17 +44,10 @@
 
 (define navbar-header
   `(a ((href "http://www.racket-lang.org/"))
-    (img ((src "/logo-and-text.png")
+    (img ((src ,(string-append static-urlprefix "/logo-and-text.png"))
           (height "60")
           (alt "Racket Package Index")))))
 
-(define navigation `((,nav-index "/")
-                     (,nav-search "/search")
-                     ;; ((div ,(glyphicon 'download-alt)
-                     ;;       " Download")
-                     ;;  "http://download.racket-lang.org/")
-                     ))
-
 (define backend-baseurl
   (or (@ (config) backend-baseurl)
       "https://pkgd.racket-lang.org"))
@@ -107,7 +100,12 @@
 
 (define-syntax-rule (with-site-config body ...)
   (parameterize ((bootstrap-navbar-header navbar-header)
-                 (bootstrap-navigation navigation)
+                 (bootstrap-navigation `((,nav-index ,(main-page-url))
+                                         (,nav-search ,(named-url search-page))
+                                         ;; ((div ,(glyphicon 'download-alt)
+                                         ;;       " Download")
+                                         ;;  "http://download.racket-lang.org/")
+                                         ))
                  (bootstrap-static-urlprefix static-urlprefix)
                  (jsonp-baseurl backend-baseurl))
     body ...))
@@ -423,7 +421,7 @@
 (define (view-package-url package-name)
   (define package-name-str (~a package-name))
   (if (use-cache?)
-      (format "~a~a" static-urlprefix (named-url package-page package-name-str))
+      (format "~a~a" static-urlprefix (relative-named-url package-page package-name-str))
       (named-url package-page package-name-str)))
 
 (define (package-link package-name)
@@ -1295,7 +1293,7 @@
   (sync/timeout (and index-rerender-needed?
                      (lambda ()
                        (static-render! relative-named-url main-page #:filename "/index.html")
-                       ;; TODO: copy static files to target
+                       (finish-static-update!)
                        (for ((completion-ch pending-completions))
                          (channel-put completion-ch (void)))
                        (package-change-handler #f '())))

src/static.rkt

@@ -3,8 +3,10 @@
 (provide static-generated-directory
          rendering-static-page?
          static-render!
+         finish-static-update!
          extra-files-paths)
 
+(require racket/system)
 (require racket/promise)
 (require racket/file)
 (require web-server/private/servlet)
@@ -18,6 +20,17 @@
   (config-path (or (@ (config) static-generated-directory)
                    (build-path (var-path) "generated-htdocs"))))
 
+(define static-content-target-directory
+  (let ((p (@ (config) static-content-target-directory)))
+    (and p (config-path p))))
+
+(define static-content-update-hook (@ (config) static-content-update-hook))
+
+(define extra-static-content-directories
+  (map config-path
+       (or (@ (config) extra-static-content-directories)
+           '())))
+
 (define rendering-static-page? (make-parameter #f))
 
 (define (static-render! #:filename [base-filename #f]
@@ -60,6 +73,24 @@
                  (response-code response)
                  (cons handler named-url-args))]))
 
+(define (finish-static-update!)
+  (when static-content-target-directory
+    (make-directory* static-content-target-directory)
+    (define command
+      (append (list (path->string (find-executable-path "rsync"))
+                    "-a"
+                    "--delete"
+                    (path->string (build-path static-generated-directory "."))
+                    (path->string (build-path (config-path "../static") ".")))
+              (for/list [(dir extra-static-content-directories)]
+                (path->string (build-path dir ".")))
+              (list (path->string (build-path static-content-target-directory ".")))))
+    (log-info "Executing rsync to replicate static content; argv: ~v" command)
+    (apply system* command))
+  (when static-content-update-hook
+    (system static-content-update-hook)))
+
 (define (extra-files-paths)
-  (list (config-path static-generated-directory)
-        (config-path "../static")))
+  (list* static-generated-directory
+         (config-path "../static")
+         extra-static-content-directories))