make doc-search URL configurable for installer builds

Also, add an initial-catalogs configuration to clients and
`farm' builds.
This commit is contained in:
Matthew Flatt 2013-07-01 17:18:45 -06:00
parent 349072c623
commit 00a4cb611b
10 changed files with 203 additions and 56 deletions

View File

@ -183,13 +183,14 @@ on to your farm-configuration module (accessible via the
default `#:clean?' configration for a client #t instead of #f.
A configuration file can specify the packages to include, host address
of the server, distribution name, and installer directory, but defaults
can be provided as `make' arguments via `PKGS', `SERVER', `DIST_NAME',
`DIST_BASE', and `DIST_DIR', respectively. Note that a sets of
of the server, distribution name, installer directory, and
documentation search URL, but defaults can be provided as `make'
arguments via `PKGS', `SERVER', `DIST_NAME', `DIST_BASE', and
`DIST_DIR', `DOC_SEARCH', respectively. Note that a sets of
packages specified in a configure file affects the clients, but it
does not affect the packages prepared by the server; only `PKGS'
affects the server (and the client's packages must be a subset of the
server's packages).
does not affect the packages prepared by the server; only `PKGS' and
`DOC_SEARCH' affect the server (and the client's packages must be
a subset of the server's packages).
For each installer written to "build/installers", the installer's name
is
@ -215,11 +216,18 @@ Roughly, the steps are
1. On the server machine:
make server PKGS="..."
Add `DOC_SEARCH="..."' to the `server' line to build documentation
so that it redirects to the given URL when a remote search is
necessary.
2. On each client machine:
make client SERVER=... PKGS="..."
or
nmake win32-client SERVER=... PKGS="..."
Add `DOC_SEARCH="..."' to the `client' line, if needed;
normally, it should be the same as for the `server' line.
Add `RELEASE_MODE=--release' to the `client' line to build
a "release" installer, as opposed to a snapshot installer.
@ -241,6 +249,11 @@ Roughly, the steps are
description, which is used as a key in the generated table of
installer files.
Add `DIST_CATALOGS_q='...'' to the `client' line to declare a
space-separated sequence of catalog URLs to set an installation's
initial package catalog URLs. Use the empty string in place of a
URL to indicate that te default path should be spliced.
In more detail:
1a. Build "racket" on a server.
@ -259,6 +272,10 @@ In more detail:
The `PKGS' variable of the makefile determines which packages are
built for potential inclusion in a distribution.
The `DOC_SEARCH' variable of the makefile determine a URL that is
embedded in rendered documentation for cases where a remote
search is needed (because other documentation is not installed).
The `SRC_CATALOG' variable determines the catalog that is used to
get package sources and native-library packages, but a value of
"local" triggers a bootstrap mode where native libraries are
@ -280,7 +297,10 @@ In more detail:
The `client' (or `win32-client') target of the makefile will do
that. Provide `SERVER' as the hostname of the server machine, and
provide the same `PKGS' (or a subset) as in step 1b if you want a
different set than the ones listed in the makefile.
different set than the ones listed in the makefile. Similarly,
`DOC_SEARCH' normally should be the same as in step 1b, but for a
client, it affects future documentation builds in the
installation.
Alternatively, use the `client' target, which combines `core' and
`client-from-core' (i.e., steps 2a and 2b).
@ -314,6 +334,14 @@ In more detail:
`DIST_DESC' to `make'. The description string is recorded
alongside the installer.
To set the initial package catalogs URLs for an installation,
provide `DIST_CATALOGS_q' to `make'. Separate multiple URLs with
a space, and use an empty string in place of a URL to indicate
that the default catalogs should be used. The "_q" in the
variable name indicates that its value can include double quotes
(but not single quotes) --- which are needed to specify an empty
string, for example.
On each client, step 2b produces a "bundle/installer.txt" file that
contains the path to the generated installer on one line, followed by
the description on a second line. The installer is also uploaded to

View File

@ -82,6 +82,11 @@ PKGS = main-distribution plt-services
# a URL (spaces allowed).
SRC_CATALOG = local
# A URL embedded in documentation for remote searches, where a Racket
# version and search key are added as query fields to the URL, and ""
# is replaced by default:
DOC_SEARCH =
# Server for built packages (i.e., the host where you'll run the
# server):
SERVER = localhost
@ -102,6 +107,11 @@ DIST_SUFFIX =
# installer, usually describing a platform:
DIST_DESC =
# Package catalog URLs (individually quoted as needed, separated by
# spaces) to install as the initial configuration in generated
# installers, where "" is replaced by the default configuration:
DIST_CATALOGS_q = ""
# Configuration of clients to run for a build farm, normally
# implemented with `#lang distro-build/farm':
FARM_CONFIG = build/farm-config.rkt
@ -142,6 +152,7 @@ LOCAL_USER_AUTO = --catalog build/local/catalog $(USER_AUTO_OPTIONS)
SOURCE_USER_AUTO_q = --catalog "$(SRC_CATALOG)" $(USER_AUTO_OPTIONS)
REMOTE_USER_AUTO = --catalog http://$(SERVER):9440/ $(USER_AUTO_OPTIONS)
REMOTE_INST_AUTO = --catalog http://$(SERVER):9440/ --scope installation --deps search-auto
BUNDLE_CONFIG = bundle/racket/etc/config.rktd
# ------------------------------------------------------------
# Linking all packages (development mode; not an installer build)
@ -193,9 +204,13 @@ local-source-catalog:
# Clear out a package build in "build/user", and then install
# packages:
local-build:
rm -rf build/user
$(MAKE) fresh-user
$(MAKE) packages-from-local
fresh-user:
rm -rf build/user
$(RACKET) $(DISTBLD)/set-config.rkt racket/etc/config.rktd "$(DOC_SEARCH)" ""
# Install packages from the source copies in this directory. The
# packages are installed in user scope, but we set the add-on
# directory to "build/user", so that we don't affect the actual
@ -209,6 +224,7 @@ packages-from-local:
# `build-from-local'), where the source catalog is specified as
# `SRC_CATALOG':
build-from-catalog:
$(MAKE) fresh-user
$(RACO) pkg install $(SOURCE_USER_AUTO_q) $(PKGS) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS)
$(RACO) setup --avoid-main
@ -283,6 +299,7 @@ bundle-from-server:
$(RACKET) -l setup/unixstyle-install bundle racket bundle/racket
$(RACKET) -l distro-build/unpack-collects http://$(SERVER):9440/
bundle/racket/bin/raco pkg install $(REMOTE_INST_AUTO) $(PKGS) $(REQUIRED_PKGS)
$(RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) "$(DOC_SEARCH)" $(DIST_CATALOGS_q)
UPLOAD_q = --upload http://$(SERVER):9440/ --desc "$(DIST_DESC)"
DIST_ARGS_q = $(UPLOAD_q) $(RELEASE_MODE) "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR) "$(DIST_SUFFIX)"
@ -307,6 +324,7 @@ win32-bundle-from-server:
$(WIN32_RACKET) -l distro-build/unpack-collects http://$(SERVER):9440/
bundle\racket\raco pkg install $(REMOTE_INST_AUTO) $(REQUIRED_PKGS)
bundle\racket\raco pkg install $(REMOTE_INST_AUTO) $(PKGS)
$(WIN32_RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) "$(DOC_SEARCH)" $(DIST_CATALOGS_q)
win32-installer-from-bundle:
$(WIN32_RACKET) -l- distro-build/installer $(DIST_ARGS_q)
@ -314,7 +332,8 @@ win32-installer-from-bundle:
# ------------------------------------------------------------
# Drive installer build:
DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(FARM_CONFIG)" "$(FARM_MODE)" $(SERVER) "$(PKGS)" "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR)
DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(FARM_CONFIG)" "$(FARM_MODE)" \
$(SERVER) "$(PKGS)" "$(DOC_SEARCH)" "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR)
DRIVE_CMD_q = $(RACKET) -l- distro-build/drive-clients $(DRIVE_ARGS_q)
# Full server build and clients drive, based on `FARM_CONFIG':

View File

@ -18,7 +18,7 @@
(define default-clean? #f)
(define-values (config-file config-mode
default-server default-pkgs
default-server default-pkgs default-doc-search
default-dist-name default-dist-base default-dist-dir)
(command-line
#:once-each
@ -27,10 +27,10 @@
[("--clean") "Erase client directories before building"
(set! default-clean? #t)]
#:args (config-file config-mode
server pkgs
server pkgs doc-search
dist-name dist-base dist-dir)
(values config-file config-mode
server pkgs
(values config-file config-mode
server pkgs doc-search
dist-name dist-base dist-dir)))
(define config (parameterize ([current-mode config-mode])
@ -168,18 +168,55 @@
(define (q s)
(~a "\"" s "\""))
(define (client-args desc server pkgs dist-name dist-base dist-dir dist-suffix)
(define (qq l kind)
(case kind
[(unix) (~a "'"
(apply ~a #:separator " " (map q l))
"'")]
[(windows) (~a "\""
(apply
~a #:separator " "
(for/list ([i (in-list l)])
(~a "\\\""
i
;; A backslash is literal unless followed by a
;; quote. If `i' ends in backslashes, they
;; must be doubled, because the \" added to
;; the end will make them treated as escapes.
(let ([m (regexp-match #rx"\\\\*$" i)])
(car m))
"\\\"")))
"\"")]))
(define (client-args c server kind)
(define desc (client-name c))
(define pkgs (let ([l (get-opt c '#:pkgs)])
(if l
(apply ~a #:separator " " l)
default-pkgs)))
(define doc-search (get-opt c '#:doc-search
default-doc-search))
(define dist-name (or (get-opt c '#:dist-name)
default-dist-name))
(define dist-base (or (get-opt c '#:dist-base)
default-dist-base))
(define dist-dir (or (get-opt c '#:dist-dir)
default-dist-dir))
(define dist-suffix (get-opt c '#:dist-suffix ""))
(define dist-catalogs (get-opt c '#:dist-catalogs '("")))
(define pull? (get-opt c '#:pull? #t))
(~a " SERVER=" server
" PKGS=" (q pkgs)
" DOC_SEARCH=" (q doc-search)
" DIST_DESC=" (q desc)
" DIST_NAME=" (q dist-name)
" DIST_BASE=" dist-base
" DIST_DIR=" dist-dir
" DIST_SUFFIX=" (q dist-suffix)
" DIST_CATALOGS_q=" (qq dist-catalogs kind)
" RELEASE_MODE=" (if release? "--release" (q ""))))
(define (unix-build c host port user server repo clean? pull?
pkgs dist-name dist-base dist-dir dist-suffix)
(define (unix-build c host port user server repo clean? pull?)
(define dir (or (get-opt c '#:dir)
"build/plt"))
(define (sh . args)
@ -197,14 +234,11 @@
"git pull"))
(sh "cd " (q dir) " ; "
"make -j " j " client"
(client-args (client-name c)
server pkgs
dist-name dist-base dist-dir dist-suffix)
(client-args c server 'unix)
" CORE_CONFIGURE_ARGS=" (q (apply ~a #:separator " "
(get-opt c '#:configure null))))))
(define (windows-build c host port user server repo clean? pull?
pkgs dist-name dist-base dist-dir dist-suffix)
(define (windows-build c host port user server repo clean? pull?)
(define dir (or (get-opt c '#:dir)
"build\\plt"))
(define bits (or (get-opt c '#:bits) 64))
@ -226,9 +260,7 @@
" && \"c:\\Program Files" (if (= bits 64) " (x86)" "") "\\Microsoft Visual Studio 9.0\\vc\\vcvarsall.bat\""
" " vc
" && nmake win32-client"
(client-args (client-name c)
server pkgs
dist-name dist-base dist-dir dist-suffix))))
(client-args c server 'windows))))
(define (client-build c)
(define host (or (get-opt c '#:host)
@ -238,17 +270,6 @@
(define user (get-opt c '#:user))
(define server (or (get-opt c '#:server)
default-server))
(define pkgs (let ([l (get-opt c '#:pkgs)])
(if l
(apply ~a #:separator " " l)
default-pkgs)))
(define dist-name (or (get-opt c '#:dist-name)
default-dist-name))
(define dist-base (or (get-opt c '#:dist-base)
default-dist-base))
(define dist-dir (or (get-opt c '#:dist-dir)
default-dist-dir))
(define dist-suffix (get-opt c '#:dist-suffix ""))
(define repo (or (get-opt c '#:repo)
(~a "http://" server ":9440/.git")))
(define clean? (let ([v (get-opt c '#:clean? 'none)])
@ -259,8 +280,7 @@
((case (or (get-opt c '#:platform) 'unix)
[(unix) unix-build]
[else windows-build])
c host port user server repo clean? pull?
pkgs dist-name dist-base dist-dir dist-suffix))
c host port user server repo clean? pull?))
;; ----------------------------------------

View File

@ -93,6 +93,12 @@
;; `PKGS' in the makfile (or, more genereally,
;; the `pkgs' command-line argument to
;; `distro-build/drive-clients')
;; #:doc-search <string> --- URL to install as the configuration
;; for remote documentation searches in
;; generated installers; "" is replaced
;; with the PLT default; defaults to the
;; `DOC_SEARCH' makefile variable or the
;; `doc-search' argument
;; #:dist-name <string> --- the distribution name; defaults to the
;; `DIST_NAME' makefile variable or `dist-name'
;; command-line argument
@ -106,6 +112,11 @@
;; used for an OS variant; defaults to the
;; `DIST_SUFFIX' makefile variable or the
;; `dist-suffix' command-line argument
;; #:dist-catalogs '(<string> ...) --- catalog URLs to install as the
;; initial catalog configuration in
;; generated installed, where ""
;; is replaced with the PLT default
;; catalogs
;; #:max-vm <real> --- max number of VMs allowed to run with this
;; machine, counting the machine; defaults to 1
;; #:vbox <string> --- Virtual Box machine name; if provided the
@ -284,10 +295,12 @@
(define (check-group-keyword kw val)
(case kw
[(#:pkgs) (and (list? val) (andmap simple-string? val))]
[(#:doc-search) (string? val)]
[(#:dist-name) (string? val)]
[(#:dist-base) (simple-string? val)]
[(#:dist-dir) (simple-string? val)]
[(#:dist-suffix) (simple-string? val)]
[(#:dist-catalogs) (and (list? val) (andmap string? val))]
[(#:max-vm) (real? val)]
[(#:server) (simple-string? val)]
[(#:host) (simple-string? val)]

View File

@ -0,0 +1,35 @@
#lang racket/base
(require racket/cmdline
racket/file
racket/path)
(define-values (config-file doc-search catalogs)
(command-line
#:args
(config-file doc-search . catalog)
(values config-file doc-search catalog)))
(define orig
(if (file-exists? config-file)
(call-with-input-file* config-file read)
(hash)))
(let* ([table orig]
[table
(if (equal? doc-search "")
table
(hash-set table 'doc-search-url doc-search))]
[table (if (equal? catalogs '(""))
table
(hash-set table 'catalogs
(for/list ([c (in-list catalogs)])
(if (equal? c "")
#f
c))))])
(unless (equal? table orig)
(make-directory* (path-only config-file))
(call-with-output-file config-file
#:exists 'truncate
(lambda (o)
(write table o)
(newline o)))))

View File

@ -75,7 +75,15 @@ directory:
@item{@racket['include-search-dirs] --- like
@racket[doc-search-dirs], but for directories containing C
header files}
header files.}
@item{@racket['doc-search-url] --- a URL string that is augmented
with version and search-tag queries to form a remote
documentation reference.}
@item{@racket['catalogs] --- a list of URL strings used as the search
path for resolving package names; an @racket[#f] in the list
is replaced with the default search path.}
@item{@racket['absolute-installation?] --- a boolean that is
@racket[#t] if the installation uses absolute path names,

View File

@ -1062,6 +1062,10 @@ v
Returns a path to the user's man-page directory; the directory
indicated by the returned path may or may not exist.}
@defproc[(get-doc-search-url) string?]{
Returns a string that is used by the documentation system, augmented
with a version and search-key query, for remote documentation links.}
@defthing[absolute-installation? boolean?]{
A binary boolean flag that is true if this installation is using
absolute path names.}

View File

@ -28,7 +28,7 @@
racket/place
pkg/lib
pkg/strip
(only-in net/url url->string path->url)
(prefix-in u: net/url)
(prefix-in html: scribble/html-render)
(prefix-in latex: scribble/latex-render)
(prefix-in contract: scribble/contract-render))
@ -614,7 +614,7 @@
(std-path "scribble-common.js")
(cons local-redirect-file "../local-redirect/local-redirect.js")))
(list (cons local-redirect-file
(url->string (path->url local-redirect-file)))))]
(u:url->string (u:path->url local-redirect-file)))))]
;; For main-directory, non-start files, up-path is #t, which makes the
;; "up" link go to the (user's) start page using cookies. For other files,
;;
@ -635,7 +635,14 @@
;; for all links external to the document, but also install the
;; "local-redirect.js" hook:
(send r set-external-tag-path
(format "http://pkg-docs.racket-lang.org?version=~a" (version)))
(u:url->string
(let ([u (u:string->url (get-doc-search-url))])
(struct-copy
u:url
u
[query
(cons (cons 'version (version))
(u:url-query u))]))))
(send r add-extra-script-file local-redirect-file))
;; Result is the renderer:
r)))
@ -1011,8 +1018,8 @@
;; and fix up the path if there is a reference:
(define js-path (if (doc-under-main? doc)
"../local-redirect"
(url->string (path->url (build-path (find-user-doc-dir)
"local-redirect")))))
(u:url->string (u:path->url (build-path (find-user-doc-dir)
"local-redirect")))))
(for ([p (in-directory dest-dir)])
(when (regexp-match? #rx#"[.]html$" (path->bytes p))
(fixup-local-redirect-reference p js-path)))

View File

@ -301,20 +301,29 @@
(define (read-pkg-cfg/def k)
(define c (read-pkg-cfg))
(hash-ref c k
(λ ()
(match k
["catalogs"
(list "https://pkg.racket-lang.org"
"https://planet-compat.racket-lang.org")]))))
(define (get-default)
(match k
['catalogs
(list "https://pkg.racket-lang.org"
"https://planet-compat.racket-lang.org")]
[_ #f]))
(define v (hash-ref c k get-default))
(match k
['catalogs
;; Replace "" with default URLs:
(apply append (for/list ([i (in-list v)])
(if (not i)
(get-default)
(list i))))]
[_ v]))
(define (pkg-config-catalogs)
(with-pkg-lock/read-only
(read-pkg-cfg/def "catalogs")))
(read-pkg-cfg/def 'catalogs)))
(define (pkg-catalogs)
(or (current-pkg-catalogs)
(map string->url (read-pkg-cfg/def "catalogs"))))
(map string->url (read-pkg-cfg/def 'catalogs))))
(define (db-path? p)
(regexp-match? #rx"[.]sqlite$" (path->bytes p)))
@ -540,7 +549,7 @@
(parameterize ([current-pkg-scope 'installation])
(with-pkg-lock/read-only
(define cfg (read-pkg-cfg))
(hash-ref cfg "default-scope" "user"))))
(hash-ref cfg 'default-scope "user"))))
(struct pkg-info (orig-pkg checksum auto?) #:prefab)
(struct sc-pkg-info pkg-info (collect) #:prefab) ; a pkg with a single collection
@ -1588,7 +1597,7 @@
[config:set
(match key+vals
[(list* (and key "catalogs") val)
(update-pkg-cfg! "catalogs" val)]
(update-pkg-cfg! 'catalogs val)]
[(list (and key "default-scope") val)
(unless (member val '("installation" "user" "shared"))
(pkg-error (~a "invliad value for config key\n"
@ -1598,7 +1607,7 @@
key
val))
(if (eq? 'installation (current-pkg-scope))
(update-pkg-cfg! "default-scope" val)
(update-pkg-cfg! 'default-scope val)
(pkg-error (~a "config key makes sense only with --installation/-i\n"
" config key: ~a\n"
" given value: ~a")
@ -1613,7 +1622,7 @@
[(list key)
(match key
["catalogs"
(for ([s (in-list (read-pkg-cfg/def "catalogs"))])
(for ([s (in-list (read-pkg-cfg/def 'catalogs))])
(printf "~a\n" s))]
["default-scope"
(if (eq? 'installation (current-pkg-scope))

View File

@ -73,14 +73,18 @@
(define-config config:cgc-suffix 'cgc-suffix values)
(define-config config:3m-suffix '3m-suffix values)
(define-config config:absolute-installation? 'absolute-installation? (lambda (x) (and x #t)))
(define-config config:doc-search-url 'doc-search-url values)
(provide get-absolute-installation?
get-cgc-suffix
get-3m-suffix)
get-3m-suffix
get-doc-search-url)
(define (get-absolute-installation?) (force config:absolute-installation?))
(define (get-cgc-suffix) (force config:cgc-suffix))
(define (get-3m-suffix) (force config:3m-suffix))
(define (get-doc-search-url) (or (force config:doc-search-url)
"http://docs.racket-lang.org"))
;; ----------------------------------------
;; "collects"