make doc-search URL configurable for installer builds

Also, add an initial-catalogs configuration to clients and
`farm' builds.

INSTALL.txt

@@ -183,13 +183,14 @@ on to your farm-configuration module (accessible via the
 default `#:clean?' configration for a client #t instead of #f.
 
 A configuration file can specify the packages to include, host address
-of the server, distribution name, and installer directory, but defaults
+of the server, distribution name, installer directory, and
-can be provided as `make' arguments via `PKGS', `SERVER', `DIST_NAME',
+documentation search URL, but defaults can be provided as `make'
-`DIST_BASE', and `DIST_DIR', respectively. Note that a sets of
+arguments via `PKGS', `SERVER', `DIST_NAME', `DIST_BASE', and
+`DIST_DIR', `DOC_SEARCH', respectively. Note that a sets of
 packages specified in a configure file affects the clients, but it
-does not affect the packages prepared by the server; only `PKGS'
+does not affect the packages prepared by the server; only `PKGS' and
-affects the server (and the client's packages must be a subset of the
+`DOC_SEARCH' affect the server (and the client's packages must be
-server's packages).
+a subset of the server's packages).
 
 For each installer written to "build/installers", the installer's name
 is
@@ -215,11 +216,18 @@ Roughly, the steps are
  1. On the server machine:
       make server PKGS="..."
 
+    Add `DOC_SEARCH="..."' to the `server' line to build documentation
+    so that it redirects to the given URL when a remote search is
+    necessary.
+
  2. On each client machine:
       make client SERVER=... PKGS="..."
     or
       nmake win32-client SERVER=... PKGS="..."
 
+    Add `DOC_SEARCH="..."' to the `client' line, if needed;
+    normally, it should be the same as for the `server' line.
+
     Add `RELEASE_MODE=--release' to the `client' line to build
     a "release" installer, as opposed to a snapshot installer.
 
@@ -241,6 +249,11 @@ Roughly, the steps are
     description, which is used as a key in the generated table of
     installer files.
 
+    Add `DIST_CATALOGS_q='...'' to the `client' line to declare a
+    space-separated sequence of catalog URLs to set an installation's
+    initial package catalog URLs. Use the empty string in place of a
+    URL to indicate that te default path should be spliced.
+
 In more detail:
 
  1a. Build "racket" on a server.
@@ -259,6 +272,10 @@ In more detail:
      The `PKGS' variable of the makefile determines which packages are
      built for potential inclusion in a distribution.
 
+     The `DOC_SEARCH' variable of the makefile determine a URL that is
+     embedded in rendered documentation for cases where a remote
+     search is needed (because other documentation is not installed).
+
      The `SRC_CATALOG' variable determines the catalog that is used to
      get package sources and native-library packages, but a value of
      "local" triggers a bootstrap mode where native libraries are
@@ -280,7 +297,10 @@ In more detail:
      The `client' (or `win32-client') target of the makefile will do
      that. Provide `SERVER' as the hostname of the server machine, and
      provide the same `PKGS' (or a subset) as in step 1b if you want a
-     different set than the ones listed in the makefile.
+     different set than the ones listed in the makefile. Similarly,
+     `DOC_SEARCH' normally should be the same as in step 1b, but for a
+     client, it affects future documentation builds in the
+     installation.
 
      Alternatively, use the `client' target, which combines `core' and
      `client-from-core' (i.e., steps 2a and 2b).
@@ -314,6 +334,14 @@ In more detail:
      `DIST_DESC' to `make'. The description string is recorded
      alongside the installer.
 
+     To set the initial package catalogs URLs for an installation,
+     provide `DIST_CATALOGS_q' to `make'. Separate multiple URLs with
+     a space, and use an empty string in place of a URL to indicate
+     that the default catalogs should be used. The "_q" in the
+     variable name indicates that its value can include double quotes
+     (but not single quotes) --- which are needed to specify an empty
+     string, for example.
+
 On each client, step 2b produces a "bundle/installer.txt" file that
 contains the path to the generated installer on one line, followed by
 the description on a second line. The installer is also uploaded to

Makefile

@@ -82,6 +82,11 @@ PKGS = main-distribution plt-services
 # a URL (spaces allowed).
 SRC_CATALOG = local
 
+# A URL embedded in documentation for remote searches, where a Racket
+# version and search key are added as query fields to the URL, and ""
+# is replaced by default:
+DOC_SEARCH = 
+
 # Server for built packages (i.e., the host where you'll run the
 # server):
 SERVER = localhost
@@ -102,6 +107,11 @@ DIST_SUFFIX =
 # installer, usually describing a platform:
 DIST_DESC =
 
+# Package catalog URLs (individually quoted as needed, separated by
+# spaces) to install as the initial configuration in generated
+# installers, where "" is replaced by the default configuration:
+DIST_CATALOGS_q = ""
+
 # Configuration of clients to run for a build farm, normally
 # implemented with `#lang distro-build/farm':
 FARM_CONFIG = build/farm-config.rkt
@@ -142,6 +152,7 @@ LOCAL_USER_AUTO = --catalog build/local/catalog $(USER_AUTO_OPTIONS)
 SOURCE_USER_AUTO_q = --catalog "$(SRC_CATALOG)" $(USER_AUTO_OPTIONS)
 REMOTE_USER_AUTO = --catalog http://$(SERVER):9440/ $(USER_AUTO_OPTIONS)
 REMOTE_INST_AUTO = --catalog http://$(SERVER):9440/ --scope installation --deps search-auto
+BUNDLE_CONFIG = bundle/racket/etc/config.rktd
 
 # ------------------------------------------------------------
 # Linking all packages (development mode; not an installer build)
@@ -193,9 +204,13 @@ local-source-catalog:
 # Clear out a package build in "build/user", and then install
 # packages:
 local-build:
-	rm -rf build/user
+	$(MAKE) fresh-user
 	$(MAKE) packages-from-local
 
+fresh-user:
+	rm -rf build/user
+	$(RACKET) $(DISTBLD)/set-config.rkt racket/etc/config.rktd "$(DOC_SEARCH)" ""
+
 # Install packages from the source copies in this directory. The
 # packages are installed in user scope, but we set the add-on
 # directory to "build/user", so that we don't affect the actual
@@ -209,6 +224,7 @@ packages-from-local:
 # `build-from-local'), where the source catalog is specified as
 # `SRC_CATALOG':
 build-from-catalog:
+	$(MAKE) fresh-user
 	$(RACO) pkg install $(SOURCE_USER_AUTO_q) $(PKGS) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS)
 	$(RACO) setup --avoid-main
 
@@ -283,6 +299,7 @@ bundle-from-server:
 	$(RACKET) -l setup/unixstyle-install bundle racket bundle/racket
 	$(RACKET) -l distro-build/unpack-collects http://$(SERVER):9440/
 	bundle/racket/bin/raco pkg install $(REMOTE_INST_AUTO) $(PKGS) $(REQUIRED_PKGS)
+	$(RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) "$(DOC_SEARCH)" $(DIST_CATALOGS_q)
 
 UPLOAD_q = --upload http://$(SERVER):9440/ --desc "$(DIST_DESC)"
 DIST_ARGS_q = $(UPLOAD_q) $(RELEASE_MODE) "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR) "$(DIST_SUFFIX)"
@@ -307,6 +324,7 @@ win32-bundle-from-server:
 	$(WIN32_RACKET) -l distro-build/unpack-collects http://$(SERVER):9440/
 	bundle\racket\raco pkg install $(REMOTE_INST_AUTO) $(REQUIRED_PKGS)
 	bundle\racket\raco pkg install $(REMOTE_INST_AUTO) $(PKGS)
+	$(WIN32_RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) "$(DOC_SEARCH)" $(DIST_CATALOGS_q)
 
 win32-installer-from-bundle:
 	$(WIN32_RACKET) -l- distro-build/installer $(DIST_ARGS_q)
@@ -314,7 +332,8 @@ win32-installer-from-bundle:
 # ------------------------------------------------------------
 # Drive installer build:
 
-DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(FARM_CONFIG)" "$(FARM_MODE)" $(SERVER) "$(PKGS)" "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR)
+DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(FARM_CONFIG)" "$(FARM_MODE)" \
+               $(SERVER) "$(PKGS)" "$(DOC_SEARCH)" "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR)
 DRIVE_CMD_q = $(RACKET) -l- distro-build/drive-clients $(DRIVE_ARGS_q)
 
 # Full server build and clients drive, based on `FARM_CONFIG':

pkgs/distro-build/drive-clients.rkt

@@ -18,7 +18,7 @@
 (define default-clean? #f)
 
 (define-values (config-file config-mode
-                            default-server default-pkgs 
+                            default-server default-pkgs default-doc-search
                             default-dist-name default-dist-base default-dist-dir)
   (command-line
    #:once-each
@@ -27,10 +27,10 @@
    [("--clean") "Erase client directories before building"
     (set! default-clean? #t)]
    #:args (config-file config-mode 
-                       server pkgs 
+                       server pkgs doc-search
                        dist-name dist-base dist-dir)
-   (values config-file config-mode 
+   (values config-file config-mode
-           server pkgs 
+           server pkgs doc-search
            dist-name dist-base dist-dir)))
 
 (define config (parameterize ([current-mode config-mode])
@@ -168,18 +168,55 @@
 (define (q s)
   (~a "\"" s "\""))
 
-(define (client-args desc server pkgs dist-name dist-base dist-dir dist-suffix)
+(define (qq l kind)
+  (case kind
+    [(unix) (~a "'"
+                (apply ~a #:separator " " (map q l))
+                "'")]
+    [(windows) (~a "\""
+                   (apply 
+                    ~a #:separator " " 
+                    (for/list ([i (in-list l)])
+                      (~a "\\\""
+                          i
+                          ;; A backslash is literal unless followed by a
+                          ;; quote. If `i' ends in backslashes, they
+                          ;; must be doubled, because the \" added to
+                          ;; the end will make them treated as escapes.
+                          (let ([m (regexp-match #rx"\\\\*$" i)])
+                            (car m))
+                          "\\\"")))
+                   "\"")]))
+
+(define (client-args c server kind)
+  (define desc (client-name c))
+  (define pkgs (let ([l (get-opt c '#:pkgs)])
+                 (if l
+                     (apply ~a #:separator " " l)
+                     default-pkgs)))
+  (define doc-search (get-opt c '#:doc-search 
+                              default-doc-search))
+  (define dist-name (or (get-opt c '#:dist-name)
+                        default-dist-name))
+  (define dist-base (or (get-opt c '#:dist-base)
+                        default-dist-base))
+  (define dist-dir (or (get-opt c '#:dist-dir)
+                       default-dist-dir))
+  (define dist-suffix (get-opt c '#:dist-suffix ""))
+  (define dist-catalogs (get-opt c '#:dist-catalogs '("")))
+  (define pull? (get-opt c '#:pull? #t))
   (~a " SERVER=" server
       " PKGS=" (q pkgs)
+      " DOC_SEARCH=" (q doc-search)
       " DIST_DESC=" (q desc)
       " DIST_NAME=" (q dist-name)
       " DIST_BASE=" dist-base
       " DIST_DIR=" dist-dir
       " DIST_SUFFIX=" (q dist-suffix)
+      " DIST_CATALOGS_q=" (qq dist-catalogs kind)
       " RELEASE_MODE=" (if release? "--release" (q ""))))
 
-(define (unix-build c host port user server repo clean? pull?
+(define (unix-build c host port user server repo clean? pull?)
-                    pkgs dist-name dist-base dist-dir dist-suffix)
   (define dir (or (get-opt c '#:dir)
                   "build/plt"))
   (define (sh . args) 
@@ -197,14 +234,11 @@
             "git pull"))
    (sh "cd " (q dir) " ; "
        "make -j " j " client"
-       (client-args (client-name c)
+       (client-args c server 'unix)
-                    server pkgs 
-                    dist-name dist-base dist-dir dist-suffix)
        " CORE_CONFIGURE_ARGS=" (q (apply ~a #:separator " "
                                          (get-opt c '#:configure null))))))
 
-(define (windows-build c host port user server repo clean? pull?
+(define (windows-build c host port user server repo clean? pull?)
-                       pkgs dist-name dist-base dist-dir dist-suffix)
   (define dir (or (get-opt c '#:dir)
                   "build\\plt"))
   (define bits (or (get-opt c '#:bits) 64))
@@ -226,9 +260,7 @@
         " && \"c:\\Program Files" (if (= bits 64) " (x86)" "") "\\Microsoft Visual Studio 9.0\\vc\\vcvarsall.bat\""
         " " vc
         " && nmake win32-client" 
-        (client-args (client-name c)
+        (client-args c server 'windows))))
-                     server pkgs 
-                     dist-name dist-base dist-dir dist-suffix))))
 
 (define (client-build c)
   (define host (or (get-opt c '#:host)
@@ -238,17 +270,6 @@
   (define user (get-opt c '#:user))
   (define server (or (get-opt c '#:server)
                      default-server))
-  (define pkgs (let ([l (get-opt c '#:pkgs)])
-                 (if l
-                     (apply ~a #:separator " " l)
-                     default-pkgs)))
-  (define dist-name (or (get-opt c '#:dist-name)
-                        default-dist-name))
-  (define dist-base (or (get-opt c '#:dist-base)
-                        default-dist-base))
-  (define dist-dir (or (get-opt c '#:dist-dir)
-                       default-dist-dir))
-  (define dist-suffix (get-opt c '#:dist-suffix ""))
   (define repo (or (get-opt c '#:repo)
                    (~a "http://" server ":9440/.git")))
   (define clean? (let ([v (get-opt c '#:clean? 'none)])
@@ -259,8 +280,7 @@
   ((case (or (get-opt c '#:platform) 'unix)
      [(unix) unix-build]
      [else windows-build])
-   c host port user server repo clean? pull?
+   c host port user server repo clean? pull?))
-   pkgs dist-name dist-base dist-dir dist-suffix))
 
 ;; ----------------------------------------
 

pkgs/distro-build/farm.rkt

@@ -93,6 +93,12 @@
 ;;                              `PKGS' in the makfile (or, more genereally,
 ;;                              the `pkgs' command-line argument to
 ;;                              `distro-build/drive-clients')
+;;   #:doc-search <string> --- URL to install as the configuration
+;;                             for remote documentation searches in
+;;                             generated installers; "" is replaced
+;;                             with the PLT default; defaults to the
+;;                             `DOC_SEARCH' makefile variable or the
+;;                             `doc-search' argument
 ;;   #:dist-name <string> --- the distribution name; defaults to the
 ;;                            `DIST_NAME' makefile variable or `dist-name'
 ;;                            command-line argument
@@ -106,6 +112,11 @@
 ;;                               used for an OS variant; defaults to the
 ;;                               `DIST_SUFFIX' makefile variable or the
 ;;                               `dist-suffix' command-line argument
+;;   #:dist-catalogs '(<string> ...) --- catalog URLs to install as the
+;;                                       initial catalog configuration in
+;;                                       generated installed, where ""
+;;                                       is replaced with the PLT default
+;;                                       catalogs
 ;;   #:max-vm <real> --- max number of VMs allowed to run with this
 ;;                       machine, counting the machine; defaults to 1
 ;;   #:vbox <string> --- Virtual Box machine name; if provided the
@@ -284,10 +295,12 @@
 (define (check-group-keyword kw val)
   (case kw
     [(#:pkgs) (and (list? val) (andmap simple-string? val))]
+    [(#:doc-search) (string? val)]
     [(#:dist-name) (string? val)]
     [(#:dist-base) (simple-string? val)]
     [(#:dist-dir) (simple-string? val)]
     [(#:dist-suffix) (simple-string? val)]
+    [(#:dist-catalogs) (and (list? val) (andmap string? val))]
     [(#:max-vm) (real? val)]
     [(#:server) (simple-string? val)]
     [(#:host) (simple-string? val)]

pkgs/distro-build/set-config.rkt

@@ -0,0 +1,35 @@
+#lang racket/base
+(require racket/cmdline
+         racket/file
+         racket/path)
+
+(define-values (config-file doc-search catalogs)
+  (command-line
+   #:args
+   (config-file doc-search . catalog)
+   (values config-file doc-search catalog)))
+
+(define orig
+  (if (file-exists? config-file)
+      (call-with-input-file* config-file read)
+      (hash)))
+
+(let* ([table orig]
+       [table
+        (if (equal? doc-search "")
+            table
+            (hash-set table 'doc-search-url doc-search))]
+       [table (if (equal? catalogs '(""))
+                  table
+                  (hash-set table 'catalogs 
+                            (for/list ([c (in-list catalogs)])
+                              (if (equal? c "")
+                                  #f
+                                  c))))])
+  (unless (equal? table orig)
+    (make-directory* (path-only config-file))
+    (call-with-output-file config-file
+      #:exists 'truncate
+      (lambda (o)
+        (write table o)
+        (newline o)))))

pkgs/racket-pkgs/racket-doc/scribblings/raco/config.scrbl

@@ -75,7 +75,15 @@ directory:
 
  @item{@racket['include-search-dirs] --- like
        @racket[doc-search-dirs], but for directories containing C
-       header files}
+       header files.}
+
+ @item{@racket['doc-search-url] --- a URL string that is augmented
+       with version and search-tag queries to form a remote
+       documentation reference.}
+
+ @item{@racket['catalogs] --- a list of URL strings used as the search
+       path for resolving package names; an @racket[#f] in the list
+       is replaced with the default search path.}
 
  @item{@racket['absolute-installation?] --- a boolean that is
        @racket[#t] if the installation uses absolute path names,

pkgs/racket-pkgs/racket-doc/scribblings/raco/setup.scrbl

@@ -1062,6 +1062,10 @@ v
   Returns a path to the user's man-page directory; the directory
   indicated by the returned path may or may not exist.}
 
+@defproc[(get-doc-search-url) string?]{
+  Returns a string that is used by the documentation system, augmented
+  with a version and search-key query, for remote documentation links.}
+
 @defthing[absolute-installation? boolean?]{
   A binary boolean flag that is true if this installation is using
   absolute path names.}

pkgs/racket-pkgs/racket-index/setup/scribble.rkt

@@ -28,7 +28,7 @@
          racket/place
          pkg/lib
          pkg/strip
-         (only-in net/url url->string path->url)
+         (prefix-in u: net/url)
          (prefix-in html: scribble/html-render)
          (prefix-in latex: scribble/latex-render)
          (prefix-in contract: scribble/contract-render))
@@ -614,7 +614,7 @@
                                         (std-path "scribble-common.js")
                                         (cons local-redirect-file "../local-redirect/local-redirect.js")))
                                 (list (cons local-redirect-file 
-                                            (url->string (path->url local-redirect-file)))))]
+                                            (u:url->string (u:path->url local-redirect-file)))))]
                ;; For main-directory, non-start files, up-path is #t, which makes the
                ;; "up" link go to the (user's) start page using cookies. For other files,
                ;; 
@@ -635,7 +635,14 @@
           ;; for all links external to the document, but also install the
           ;; "local-redirect.js" hook:
           (send r set-external-tag-path 
-                (format "http://pkg-docs.racket-lang.org?version=~a" (version)))
+                (u:url->string
+                 (let ([u (u:string->url (get-doc-search-url))])
+                   (struct-copy
+                    u:url
+                    u
+                    [query
+                     (cons (cons 'version (version))
+                           (u:url-query u))]))))
           (send r add-extra-script-file local-redirect-file))
         ;; Result is the renderer:
         r)))
@@ -1011,8 +1018,8 @@
     ;; and fix up the path if there is a reference:
     (define js-path (if (doc-under-main? doc)
                         "../local-redirect"
-                        (url->string (path->url (build-path (find-user-doc-dir)
+                        (u:url->string (u:path->url (build-path (find-user-doc-dir)
-                                                            "local-redirect")))))
+                                                                "local-redirect")))))
     (for ([p (in-directory dest-dir)])
       (when (regexp-match? #rx#"[.]html$" (path->bytes p))
         (fixup-local-redirect-reference p js-path)))

racket/lib/collects/pkg/lib.rkt

@@ -301,20 +301,29 @@
 
 (define (read-pkg-cfg/def k)
   (define c (read-pkg-cfg))
-  (hash-ref c k
+  (define (get-default)
-            (λ ()
+    (match k
-              (match k
+      ['catalogs
-                ["catalogs"
+       (list "https://pkg.racket-lang.org"
-                 (list "https://pkg.racket-lang.org"
+             "https://planet-compat.racket-lang.org")]
-                       "https://planet-compat.racket-lang.org")]))))
+      [_ #f]))
+  (define v (hash-ref c k get-default))
+  (match k
+    ['catalogs
+     ;; Replace "" with default URLs:
+     (apply append (for/list ([i (in-list v)])
+                     (if (not i)
+                         (get-default)
+                         (list i))))]
+    [_ v]))
 
 (define (pkg-config-catalogs)
   (with-pkg-lock/read-only
-   (read-pkg-cfg/def "catalogs")))
+   (read-pkg-cfg/def 'catalogs)))
 
 (define (pkg-catalogs)
   (or (current-pkg-catalogs)
-      (map string->url (read-pkg-cfg/def "catalogs"))))
+      (map string->url (read-pkg-cfg/def 'catalogs))))
 
 (define (db-path? p)
   (regexp-match? #rx"[.]sqlite$" (path->bytes p)))
@@ -540,7 +549,7 @@
   (parameterize ([current-pkg-scope 'installation])
     (with-pkg-lock/read-only
      (define cfg (read-pkg-cfg))
-     (hash-ref cfg "default-scope" "user"))))
+     (hash-ref cfg 'default-scope "user"))))
 
 (struct pkg-info (orig-pkg checksum auto?) #:prefab)
 (struct sc-pkg-info pkg-info (collect) #:prefab) ; a pkg with a single collection
@@ -1588,7 +1597,7 @@
     [config:set
      (match key+vals
        [(list* (and key "catalogs") val)
-        (update-pkg-cfg! "catalogs" val)]
+        (update-pkg-cfg! 'catalogs val)]
        [(list (and key "default-scope") val)
         (unless (member val '("installation" "user" "shared"))
           (pkg-error (~a "invliad value for config key\n"
@@ -1598,7 +1607,7 @@
                      key
                      val))
         (if (eq? 'installation (current-pkg-scope))
-            (update-pkg-cfg! "default-scope" val)
+            (update-pkg-cfg! 'default-scope val)
             (pkg-error (~a "config key makes sense only with --installation/-i\n"
                            "  config key: ~a\n"
                            "  given value: ~a")
@@ -1613,7 +1622,7 @@
        [(list key)
         (match key
           ["catalogs"
-           (for ([s (in-list (read-pkg-cfg/def "catalogs"))])
+           (for ([s (in-list (read-pkg-cfg/def 'catalogs))])
              (printf "~a\n" s))]
           ["default-scope"
            (if (eq? 'installation (current-pkg-scope))

racket/lib/collects/setup/dirs.rkt

@@ -73,14 +73,18 @@
 (define-config config:cgc-suffix 'cgc-suffix values)
 (define-config config:3m-suffix '3m-suffix values)
 (define-config config:absolute-installation? 'absolute-installation? (lambda (x) (and x #t)))
+(define-config config:doc-search-url 'doc-search-url values)
 
 (provide get-absolute-installation?
          get-cgc-suffix
-         get-3m-suffix)
+         get-3m-suffix
+         get-doc-search-url)
 
 (define (get-absolute-installation?) (force config:absolute-installation?))
 (define (get-cgc-suffix) (force config:cgc-suffix))
 (define (get-3m-suffix) (force config:3m-suffix))
+(define (get-doc-search-url) (or (force config:doc-search-url)
+                                 "http://docs.racket-lang.org"))
 
 ;; ----------------------------------------
 ;;  "collects"