suzanne.soy/racket
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generalize use of farm config file

Browse Source 
Change `FARM_CONFIG' to just `CONFIG' and use it on the server, too.
This commit is contained in:
Matthew Flatt 2013-07-01 20:09:01 -06:00
parent 00a4cb611b
commit 3264f16b63
9 changed files with 193 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
INSTALL.txt
View File

@ -168,29 +168,27 @@ client hosts.
If "my-farm-config.rkt" is a configuration module, then
make farm FARM_CONFIG=my-farm-config.rkt PKGS="..."
make farm CONFIG=my-farm-config.rkt
drives the build farm, and the resulting installers are in
"build/installers", with a hash table mapping descriptions to
installer filenames in "build/installer/table.rktd".
The default FARM_CONFIG path is "build/farm-config.rkt", so you could
put your configuration file there and omit the `FARM_CONFIG' argument
to `make'. Similarly, you can omit `PKGS' to use the default specified
in the Makefile. Supply `FARM_MODE="..."' to pass a configuration mode
on to your farm-configuration module (accessible via the
`current-mode' parameter). Supply `CLEAN_MODE="--clean"' to make the
default `#:clean?' configration for a client #t instead of #f.
The default CONFIG path is "build/farm-config.rkt", so you could put
your configuration file there and omit the `CONFIG' argument to
`make'. Supply `CONFIG_MODE=...' to pass a configuration mode on to
your farm-configuration module (accessible via the `current-mode'
parameter). Supply `CLEAN_MODE=--clean' to make the default
`#:clean?' configration for a client #t instead of #f.
A configuration file can specify the packages to include, host address
of the server, distribution name, installer directory, and
documentation search URL, but defaults can be provided as `make'
arguments via `PKGS', `SERVER', `DIST_NAME', `DIST_BASE', and
`DIST_DIR', `DOC_SEARCH', respectively. Note that a sets of
packages specified in a configure file affects the clients, but it
does not affect the packages prepared by the server; only `PKGS' and
`DOC_SEARCH' affect the server (and the client's packages must be
a subset of the server's packages).
`DIST_DIR', `DOC_SEARCH', respectively. The farm configuration's
top-level options for packages and documentation search URL are used
to configure the set of packages that are available to client
machines to include in installers.
For each installer written to "build/installers", the installer's name
is

32
Makefile
View File

@ -55,7 +55,7 @@ core:
cd racket/src/build; $(MAKE) install SELF_RACKET_FLAGS="-G `cd ../../../build/config; pwd`"
win32-core:
IF NOT EXIST build\config cmd /c mkdir mkdir -p build\config
IF NOT EXIST build\config cmd /c mkdir -p build\config
cmd /c echo #hash((links-search-files . ())) > build\config\config.rktd
cmd /c racket\src\worksp\build-at racket\src\worksp ..\..\..\build\config
@ -114,14 +114,14 @@ DIST_CATALOGS_q = ""
# Configuration of clients to run for a build farm, normally
# implemented with `#lang distro-build/farm':
FARM_CONFIG = build/farm-config.rkt
CONFIG = build/farm-config.rkt
# A mode that is made available to the farm-configuration module
# through the `current-mode' parameter:
FARM_MODE = default
CONFIG_MODE = default
# Set to "--clean" to flush client directories in a build farm
# (except as overridden in the `FARM_CONFIG' module):
# (except as overridden in the `CONFIG' module):
CLEAN_MODE =
# A command to run after the server has started; normally set by
@ -152,6 +152,7 @@ LOCAL_USER_AUTO = --catalog build/local/catalog $(USER_AUTO_OPTIONS)
SOURCE_USER_AUTO_q = --catalog "$(SRC_CATALOG)" $(USER_AUTO_OPTIONS)
REMOTE_USER_AUTO = --catalog http://$(SERVER):9440/ $(USER_AUTO_OPTIONS)
REMOTE_INST_AUTO = --catalog http://$(SERVER):9440/ --scope installation --deps search-auto
CONFIG_MODE_q = "$(CONFIG)" "$(CONFIG_MODE)"
BUNDLE_CONFIG = bundle/racket/etc/config.rktd
# ------------------------------------------------------------
@ -209,7 +210,9 @@ local-build:
fresh-user:
rm -rf build/user
$(RACKET) $(DISTBLD)/set-config.rkt racket/etc/config.rktd "$(DOC_SEARCH)" ""
set-config:
$(RACKET) -l distro-build/set-config racket/etc/config.rktd $(CONFIG_MODE_q) "$(DOC_SEARCH)" ""
# Install packages from the source copies in this directory. The
# packages are installed in user scope, but we set the add-on
@ -217,7 +220,9 @@ fresh-user:
# current user's installation (and to a large degree we're insulated
# from it):
packages-from-local:
$(RACO) pkg install $(LOCAL_USER_AUTO) $(PKGS) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS)
$(RACO) pkg install $(LOCAL_USER_AUTO) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS)
$(MAKE) set-config
$(RACKET) -l distro-build/install-pkgs $(CONFIG_MODE_q) "$(PKGS)" $(LOCAL_USER_AUTO)
$(RACO) setup --avoid-main
# Install packages from a source catalog (as an alternative to
@ -225,7 +230,9 @@ packages-from-local:
# `SRC_CATALOG':
build-from-catalog:
$(MAKE) fresh-user
$(RACO) pkg install $(SOURCE_USER_AUTO_q) $(PKGS) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS)
$(RACO) pkg install $(SOURCE_USER_AUTO_q) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS)
$(MAKE) set-config
$(RACKET) -l distro-build/install-pkgs $(CONFIG_MODE_q) "$(CONFIG_MODE)" "$(PKGS)" $(SOURCE_USER_AUTO_q)
$(RACO) setup --avoid-main
# Although a client will build its own "collects", pack up the
@ -270,6 +277,7 @@ client:
$(MAKE) core
$(MAKE) distro-build-from-server
$(MAKE) bundle-from-server
$(MAKE) bundle-config
$(MAKE) installer-from-bundle
COPY_ARGS = SERVER=$(SERVER) PKGS="$(PKGS)" RELEASE_MODE=$(RELEASE_MODE) \
@ -282,6 +290,7 @@ win32-client:
$(MAKE) win32-core $(COPY_ARGS)
$(MAKE) win32-distro-build-from-server $(COPY_ARGS)
$(MAKE) win32-bundle-from-server $(COPY_ARGS)
$(WIN32_RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) $(CONFIG_MODE_q) "$(DOC_SEARCH)" $(DIST_CATALOGS_q)
$(MAKE) win32-installer-from-bundle $(COPY_ARGS)
# Install the "distro-build" package from the server into
@ -299,7 +308,9 @@ bundle-from-server:
$(RACKET) -l setup/unixstyle-install bundle racket bundle/racket
$(RACKET) -l distro-build/unpack-collects http://$(SERVER):9440/
bundle/racket/bin/raco pkg install $(REMOTE_INST_AUTO) $(PKGS) $(REQUIRED_PKGS)
$(RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) "$(DOC_SEARCH)" $(DIST_CATALOGS_q)
bundle-config:
$(RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) $(CONFIG_MODE_q) "$(DOC_SEARCH)" $(DIST_CATALOGS_q)
UPLOAD_q = --upload http://$(SERVER):9440/ --desc "$(DIST_DESC)"
DIST_ARGS_q = $(UPLOAD_q) $(RELEASE_MODE) "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR) "$(DIST_SUFFIX)"
@ -324,7 +335,6 @@ win32-bundle-from-server:
$(WIN32_RACKET) -l distro-build/unpack-collects http://$(SERVER):9440/
bundle\racket\raco pkg install $(REMOTE_INST_AUTO) $(REQUIRED_PKGS)
bundle\racket\raco pkg install $(REMOTE_INST_AUTO) $(PKGS)
$(WIN32_RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) "$(DOC_SEARCH)" $(DIST_CATALOGS_q)
win32-installer-from-bundle:
$(WIN32_RACKET) -l- distro-build/installer $(DIST_ARGS_q)
@ -332,11 +342,11 @@ win32-installer-from-bundle:
# ------------------------------------------------------------
# Drive installer build:
DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(FARM_CONFIG)" "$(FARM_MODE)" \
DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(CONFIG)" "$(CONFIG_MODE)" \
$(SERVER) "$(PKGS)" "$(DOC_SEARCH)" "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR)
DRIVE_CMD_q = $(RACKET) -l- distro-build/drive-clients $(DRIVE_ARGS_q)
# Full server build and clients drive, based on `FARM_CONFIG':
# Full server build and clients drive, based on `CONFIG':
farm:
$(MAKE) server SERVE_DURING_CMD_qq='$(DRIVE_CMD_q)'

12
pkgs/distro-build/drive-clients.rkt
View File

@ -8,7 +8,8 @@
(only-in "farm.rkt"
current-mode
farm-config?
farm-config-tag farm-config-options farm-config-content))
farm-config-tag farm-config-options farm-config-content)
"url-options.rkt")
;; See "farm.rkt" for an overview.
@ -194,8 +195,7 @@
(if l
(apply ~a #:separator " " l)
default-pkgs)))
(define doc-search (get-opt c '#:doc-search
default-doc-search))
(define doc-search (choose-doc-search c default-doc-search))
(define dist-name (or (get-opt c '#:dist-name)
default-dist-name))
(define dist-base (or (get-opt c '#:dist-base)
@ -203,7 +203,7 @@
(define dist-dir (or (get-opt c '#:dist-dir)
default-dist-dir))
(define dist-suffix (get-opt c '#:dist-suffix ""))
(define dist-catalogs (get-opt c '#:dist-catalogs '("")))
(define dist-catalogs (choose-catalogs c '("")))
(define pull? (get-opt c '#:pull? #t))
(~a " SERVER=" server
" PKGS=" (q pkgs)
@ -220,7 +220,9 @@
(define dir (or (get-opt c '#:dir)
"build/plt"))
(define (sh . args)
(list "/bin/sh" "-c" (~a "'" (apply ~a args) "'")))
(list "/bin/sh" "-c" (~a "'"
(regexp-replace* #rx"'" (apply ~a args) "'\"'\"'")
"'")))
(define j (or (get-opt c '#:j) 1))
(ssh-script
host port user

93
pkgs/distro-build/farm.rkt
View File

@ -4,11 +4,17 @@
;; repository's top-level makefile. That target, in turn, uses the
;; `distro-build/drive-clients' module.
;;
;; Each client is built by running commands via `ssh', where the
;; client's host (and optional port and/or user) indicate the ssh
;; target. Each client machine must be set up with a public-key
;; authenticaion, because a direct `ssh' is expected to work without a
;; password prompt.
;; The server machine first prepares packages for installation on
;; clients. The farm configuration's top-level entry is consulted for
;; a `#:pkgs' and/or `#:doc-search' option, which overrides any `PKGS'
;; and/or `DOC_SEARCH' configuration from the makefile.
;;
;; The farm configuration file otherwise describes and configures
;; client machines. Each client is built by running commands via
;; `ssh', where the client's host (and optional port and/or user)
;; indicate the ssh target. Each client machine must be set up with a
;; public-key authenticaion, because a direct `ssh' is expected to
;; work without a password prompt.
;;
;; On the client machine, all work is performed with a git clone at a
;; specified directory. The directory defaults to "build/plt" (Unix,
@ -54,13 +60,14 @@
;; individual machines, and groups them with `parallel' or
;; `sequential' to indicate whether the machine's builds should run
;; sequentially or in parallel. Options specified at `parallel' or
;; `sequential' are propagated to eachmachine in the group.
;; `sequential' are propagated to each machine in the group.
;;
;; For example, a configuration module might look like this:
;;
;; #lang distro-build/farm
;;
;; (sequential
;; #:pkgs '("drracket")
;; #:server "192.168.56.1"
;; (machine
;; #:desc "Linux (32-bit, Precise Pangolin)"
@ -90,33 +97,47 @@
;; #:repo <string> --- the git repository for Racket; defaults to
;; "http://<server>:9440/.git"
;; #:pkgs '(<string*> ...) --- packages to install; defaults to
;; `PKGS' in the makfile (or, more genereally,
;; `PKGS' in the makefile (or, particularly,
;; the `pkgs' command-line argument to
;; `distro-build/drive-clients')
;; #:dist-base-url <string> --- a URL that is used to construct
;; a default for #:doc-search and
;; #:dist-catalogs, where the
;; constructed values are consistent
;; with converting a build server's
;; content into a download site; since
;; URLs are constructed via relative
;; paths, this URL normally should end
;; with a slash
;; #:doc-search <string> --- URL to install as the configuration
;; for remote documentation searches in
;; generated installers; "" is replaced
;; with the PLT default; defaults to the
;; `DOC_SEARCH' makefile variable or the
;; `doc-search' argument
;; with the PLT default; defaults to
;; #:dist-base-url (if present) extended
;; with "doc/search.html", or the
;; `DOC_SEARCH' makefile variable (or the
;; `doc-search' argument)
;; #:dist-name <string> --- the distribution name; defaults to the
;; `DIST_NAME' makefile variable or `dist-name'
;; command-line argument
;; `DIST_NAME' makefile variable (or the
;; `dist-name' command-line argument)
;; #:dist-base <string*> --- the distribution's installater name prefix;
;; defaults to the `DIST_BASE' makefile variable
;; or the `dist-base' command-line argument
;; (or the `dist-base' command-line argument)
;; #:dist-dir <string*> --- the distribution's installation directory;
;; defaults to the `DIST_DIR' makefile variable
;; or the `dist-dir' command-line argument
;; (or the `dist-dir' command-line argument)
;; #:dist-suffix <string*> --- a suffix for the installer's name, usually
;; used for an OS variant; defaults to the
;; `DIST_SUFFIX' makefile variable or the
;; `dist-suffix' command-line argument
;; `DIST_SUFFIX' makefile variable (or the
;; `dist-suffix' command-line argument)
;; #:dist-catalogs '(<string> ...) --- catalog URLs to install as the
;; initial catalog configuration in
;; generated installed, where ""
;; is replaced with the PLT default
;; catalogs
;; initial catalog configuration
;; in generated installed, where
;; "" is replaced with the PLT
;; default catalogs; defaults to
;; #:dist-base-url (if present)
;; extended with "catalogs" in a
;; list followed by ""
;; #:max-vm <real> --- max number of VMs allowed to run with this
;; machine, counting the machine; defaults to 1
;; #:vbox <string> --- Virtual Box machine name; if provided the
@ -134,13 +155,14 @@
;; machine starts by removing <dir>; set this
;; to #f for a shared repo checkout; the default
;; is determined by the `CLEAN_MODE' makefile
;; variable or `--clean' command-line flag
;; variable (or `--clean' command-line flag)
;; #:pull? <boolean> --- if true, then the build process on the client
;; machine starts by a `git pull' in <dir>; set
;; to #f, for example, for a repo checkout that is
;; shared with server; the default is #t
;;
;; Machine-only keywords:
;;
;; #:name <string> --- defaults to host; this string is recorded as
;; a description of the installer (for use in a
;; generated table of installer links, for example)
@ -185,8 +207,10 @@
;; (current-mode s) -> void?
;; s : string?
;; A parameter whose value is the user's requested mode for this
;; configuration. The default mode is "default". The interpretation
;; of modes is completely up to the farm-configuration file.
;; configuration, normally as provided via the makefile's
;; `CONFIG_MODE' variable. The default mode is "default". The
;; interpretation of modes is completely up to the
;; farm configuration file.
;; ----------------------------------------
@ -204,7 +228,8 @@
farm-config-tag
farm-config-options
farm-config-content
current-mode)
current-mode
extract-options)
(module reader syntax/module-reader
distro-build/farm)
@ -279,10 +304,16 @@
tag
(for/hash ([kw (in-list kws)]
[val (in-list kw-vals)])
(define r (check kw val))
(when (eq? r 'bad-keyword)
(error tag
(~a "unrecognized keyword for option\n"
" keyword: ~s")
kw))
(unless (check kw val)
(error tag
(~a "bad value for keyword\n"
" keyword: ~s"
" keyword: ~s\n"
" value: ~e")
kw
val))
@ -301,6 +332,7 @@
[(#:dist-dir) (simple-string? val)]
[(#:dist-suffix) (simple-string? val)]
[(#:dist-catalogs) (and (list? val) (andmap string? val))]
[(#:dist-base-url) (string? val)]
[(#:max-vm) (real? val)]
[(#:server) (simple-string? val)]
[(#:host) (simple-string? val)]
@ -317,7 +349,7 @@
[(#:repo) (string? val)]
[(#:clean?) (boolean? val)]
[(#:pull?) (boolean? val)]
[else #f]))
[else 'bad-keyword]))
(define (check-machine-keyword kw val)
(case kw
@ -331,3 +363,12 @@
(regexp-match #rx"^[-a-zA-A0-9.]*$" s)))
(define current-mode (make-parameter "default"))
(define (extract-options config-file config-mode)
(or
(and (file-exists? config-file)
(parameterize ([current-mode config-mode])
(farm-config-options
(dynamic-require (path->complete-path config-file) 'farm-config))))
(hash)))

21
pkgs/distro-build/install-pkgs.rkt Normal file
View File

@ -0,0 +1,21 @@
#lang racket/base
(require racket/cmdline
racket/string
(only-in "farm.rkt" extract-options))
(define-values (config-file config-mode default-pkgs flags)
(command-line
#:args
(config-file config-mode pkgs . flag)
(values config-file config-mode pkgs flag)))
(define pkgs (or (hash-ref (extract-options config-file config-mode)
'#:pkgs
#f)
(string-split default-pkgs)))
(parameterize ([current-command-line-arguments
(list->vector (append (list "pkg" "install")
flags
pkgs))])
(dynamic-require 'raco #f))

24
pkgs/distro-build/set-config.rkt
View File

@ -1,17 +1,25 @@
#lang racket/base
(require racket/cmdline
racket/file
racket/path)
racket/path
(only-in "farm.rkt" extract-options)
"url-options.rkt")
(define-values (config-file doc-search catalogs)
(define-values (dest-config-file config-file config-mode default-doc-search default-catalogs)
(command-line
#:args
(config-file doc-search . catalog)
(values config-file doc-search catalog)))
(dest-config-file config-file config-mode doc-search . catalog)
(values dest-config-file config-file config-mode doc-search catalog)))
(define config (extract-options config-file config-mode))
(define doc-search (choose-doc-search config default-doc-search))
(define catalogs (choose-catalogs config default-catalogs))
(define orig
(if (file-exists? config-file)
(call-with-input-file* config-file read)
(if (file-exists? dest-config-file)
(call-with-input-file* dest-config-file read)
(hash)))
(let* ([table orig]
@ -27,8 +35,8 @@
#f
c))))])
(unless (equal? table orig)
(make-directory* (path-only config-file))
(call-with-output-file config-file
(make-directory* (path-only dest-config-file))
(call-with-output-file dest-config-file
#:exists 'truncate
(lambda (o)
(write table o)

27
pkgs/distro-build/stamp.rkt Normal file
View File

@ -0,0 +1,27 @@
#lang racket/base
(require racket/system
racket/format)
(provide get-date-stamp
get-commit-stamp
get-date+commit-stamp)
(define (get-commit-stamp)
(define git (or (find-executable-path "git")
(find-executable-path "git.exe")))
(define s (open-output-string))
(parameterize ([current-output-port s]
[current-input-port (open-input-string "")])
(system* git "log" "-1" "--pretty=format:%h"))
(define commit-id (get-output-string s))
commit-id)
(define (get-date-stamp)
(define d (seconds->date (current-seconds)))
(define (n n) (~a n #:align 'right #:pad-string "0" #:width 2))
(~a (date-year d) (n (date-month d)) (n (date-day d))))
(define (get-date+commit-stamp)
(~a (get-date-stamp)
"-"
(get-commit-stamp)))

22
pkgs/distro-build/url-options.rkt Normal file
View File

@ -0,0 +1,22 @@
#lang racket/base
(require net/url)
(provide choose-doc-search
choose-catalogs)
(define (choose-doc-search config default-doc-search)
(or (hash-ref config '#:doc-search #f)
(let ([v (hash-ref config '#:dist-base-url #f)])
(and v
(url->string
(combine-url/relative (string->url v) "docs/search.html"))))
default-doc-search))
(define (choose-catalogs config default-catalogs)
(or (hash-ref config '#:dist-catalogs #f)
(let ([v (hash-ref config '#:dist-base-url #f)])
(and v
(list (url->string
(combine-url/relative (string->url v) "catalog"))
"")))
default-catalogs))

2
racket/lib/collects/compiler/private/winsubsys.rkt
View File

@ -5,7 +5,7 @@
(define DF_SubsystemOffset #x5C)
(define (set-subsystem file subsys)
(let-values ([(in out) (open-input-output-file file 'update)])
(let-values ([(in out) (open-input-output-file file #:exists 'update)])
(file-position in DF_NewHeaderOffset)
(let ([offset (integer-bytes->integer (read-bytes 4 in) #f #f)])
(file-position out (+ offset DF_SubsystemOffset))