From 3264f16b63b483b9216b17b8999b50d6ba0b181b Mon Sep 17 00:00:00 2001 From: Matthew Flatt Date: Mon, 1 Jul 2013 20:09:01 -0600 Subject: [PATCH] Generalize use of farm config file Change `FARM_CONFIG' to just `CONFIG' and use it on the server, too. --- INSTALL.txt | 24 +++-- Makefile | 32 ++++--- pkgs/distro-build/drive-clients.rkt | 12 ++- pkgs/distro-build/farm.rkt | 93 +++++++++++++------ pkgs/distro-build/install-pkgs.rkt | 21 +++++ pkgs/distro-build/set-config.rkt | 24 +++-- pkgs/distro-build/stamp.rkt | 27 ++++++ pkgs/distro-build/url-options.rkt | 22 +++++ .../collects/compiler/private/winsubsys.rkt | 2 +- 9 files changed, 193 insertions(+), 64 deletions(-) create mode 100644 pkgs/distro-build/install-pkgs.rkt create mode 100644 pkgs/distro-build/stamp.rkt create mode 100644 pkgs/distro-build/url-options.rkt diff --git a/INSTALL.txt b/INSTALL.txt index 67018ed06a..7f0ee5f98e 100644 --- a/INSTALL.txt +++ b/INSTALL.txt @@ -168,29 +168,27 @@ client hosts. If "my-farm-config.rkt" is a configuration module, then - make farm FARM_CONFIG=my-farm-config.rkt PKGS="..." + make farm CONFIG=my-farm-config.rkt drives the build farm, and the resulting installers are in "build/installers", with a hash table mapping descriptions to installer filenames in "build/installer/table.rktd". -The default FARM_CONFIG path is "build/farm-config.rkt", so you could -put your configuration file there and omit the `FARM_CONFIG' argument -to `make'. Similarly, you can omit `PKGS' to use the default specified -in the Makefile. Supply `FARM_MODE="..."' to pass a configuration mode -on to your farm-configuration module (accessible via the -`current-mode' parameter). Supply `CLEAN_MODE="--clean"' to make the -default `#:clean?' configration for a client #t instead of #f. +The default CONFIG path is "build/farm-config.rkt", so you could put +your configuration file there and omit the `CONFIG' argument to +`make'. Supply `CONFIG_MODE=...' to pass a configuration mode on to +your farm-configuration module (accessible via the `current-mode' +parameter). Supply `CLEAN_MODE=--clean' to make the default +`#:clean?' configration for a client #t instead of #f. A configuration file can specify the packages to include, host address of the server, distribution name, installer directory, and documentation search URL, but defaults can be provided as `make' arguments via `PKGS', `SERVER', `DIST_NAME', `DIST_BASE', and -`DIST_DIR', `DOC_SEARCH', respectively. Note that a sets of -packages specified in a configure file affects the clients, but it -does not affect the packages prepared by the server; only `PKGS' and -`DOC_SEARCH' affect the server (and the client's packages must be -a subset of the server's packages). +`DIST_DIR', `DOC_SEARCH', respectively. The farm configuration's +top-level options for packages and documentation search URL are used +to configure the set of packages that are available to client +machines to include in installers. For each installer written to "build/installers", the installer's name is diff --git a/Makefile b/Makefile index 4ba21f0ff8..37464b094a 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ core: cd racket/src/build; $(MAKE) install SELF_RACKET_FLAGS="-G `cd ../../../build/config; pwd`" win32-core: - IF NOT EXIST build\config cmd /c mkdir mkdir -p build\config + IF NOT EXIST build\config cmd /c mkdir -p build\config cmd /c echo #hash((links-search-files . ())) > build\config\config.rktd cmd /c racket\src\worksp\build-at racket\src\worksp ..\..\..\build\config @@ -114,14 +114,14 @@ DIST_CATALOGS_q = "" # Configuration of clients to run for a build farm, normally # implemented with `#lang distro-build/farm': -FARM_CONFIG = build/farm-config.rkt +CONFIG = build/farm-config.rkt # A mode that is made available to the farm-configuration module # through the `current-mode' parameter: -FARM_MODE = default +CONFIG_MODE = default # Set to "--clean" to flush client directories in a build farm -# (except as overridden in the `FARM_CONFIG' module): +# (except as overridden in the `CONFIG' module): CLEAN_MODE = # A command to run after the server has started; normally set by @@ -152,6 +152,7 @@ LOCAL_USER_AUTO = --catalog build/local/catalog $(USER_AUTO_OPTIONS) SOURCE_USER_AUTO_q = --catalog "$(SRC_CATALOG)" $(USER_AUTO_OPTIONS) REMOTE_USER_AUTO = --catalog http://$(SERVER):9440/ $(USER_AUTO_OPTIONS) REMOTE_INST_AUTO = --catalog http://$(SERVER):9440/ --scope installation --deps search-auto +CONFIG_MODE_q = "$(CONFIG)" "$(CONFIG_MODE)" BUNDLE_CONFIG = bundle/racket/etc/config.rktd # ------------------------------------------------------------ @@ -209,7 +210,9 @@ local-build: fresh-user: rm -rf build/user - $(RACKET) $(DISTBLD)/set-config.rkt racket/etc/config.rktd "$(DOC_SEARCH)" "" + +set-config: + $(RACKET) -l distro-build/set-config racket/etc/config.rktd $(CONFIG_MODE_q) "$(DOC_SEARCH)" "" # Install packages from the source copies in this directory. The # packages are installed in user scope, but we set the add-on @@ -217,7 +220,9 @@ fresh-user: # current user's installation (and to a large degree we're insulated # from it): packages-from-local: - $(RACO) pkg install $(LOCAL_USER_AUTO) $(PKGS) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS) + $(RACO) pkg install $(LOCAL_USER_AUTO) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS) + $(MAKE) set-config + $(RACKET) -l distro-build/install-pkgs $(CONFIG_MODE_q) "$(PKGS)" $(LOCAL_USER_AUTO) $(RACO) setup --avoid-main # Install packages from a source catalog (as an alternative to @@ -225,7 +230,9 @@ packages-from-local: # `SRC_CATALOG': build-from-catalog: $(MAKE) fresh-user - $(RACO) pkg install $(SOURCE_USER_AUTO_q) $(PKGS) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS) + $(RACO) pkg install $(SOURCE_USER_AUTO_q) $(REQUIRED_PKGS) $(DISTRO_BUILD_PKGS) + $(MAKE) set-config + $(RACKET) -l distro-build/install-pkgs $(CONFIG_MODE_q) "$(CONFIG_MODE)" "$(PKGS)" $(SOURCE_USER_AUTO_q) $(RACO) setup --avoid-main # Although a client will build its own "collects", pack up the @@ -270,6 +277,7 @@ client: $(MAKE) core $(MAKE) distro-build-from-server $(MAKE) bundle-from-server + $(MAKE) bundle-config $(MAKE) installer-from-bundle COPY_ARGS = SERVER=$(SERVER) PKGS="$(PKGS)" RELEASE_MODE=$(RELEASE_MODE) \ @@ -282,6 +290,7 @@ win32-client: $(MAKE) win32-core $(COPY_ARGS) $(MAKE) win32-distro-build-from-server $(COPY_ARGS) $(MAKE) win32-bundle-from-server $(COPY_ARGS) + $(WIN32_RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) $(CONFIG_MODE_q) "$(DOC_SEARCH)" $(DIST_CATALOGS_q) $(MAKE) win32-installer-from-bundle $(COPY_ARGS) # Install the "distro-build" package from the server into @@ -299,7 +308,9 @@ bundle-from-server: $(RACKET) -l setup/unixstyle-install bundle racket bundle/racket $(RACKET) -l distro-build/unpack-collects http://$(SERVER):9440/ bundle/racket/bin/raco pkg install $(REMOTE_INST_AUTO) $(PKGS) $(REQUIRED_PKGS) - $(RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) "$(DOC_SEARCH)" $(DIST_CATALOGS_q) + +bundle-config: + $(RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) $(CONFIG_MODE_q) "$(DOC_SEARCH)" $(DIST_CATALOGS_q) UPLOAD_q = --upload http://$(SERVER):9440/ --desc "$(DIST_DESC)" DIST_ARGS_q = $(UPLOAD_q) $(RELEASE_MODE) "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR) "$(DIST_SUFFIX)" @@ -324,7 +335,6 @@ win32-bundle-from-server: $(WIN32_RACKET) -l distro-build/unpack-collects http://$(SERVER):9440/ bundle\racket\raco pkg install $(REMOTE_INST_AUTO) $(REQUIRED_PKGS) bundle\racket\raco pkg install $(REMOTE_INST_AUTO) $(PKGS) - $(WIN32_RACKET) -l distro-build/set-config $(BUNDLE_CONFIG) "$(DOC_SEARCH)" $(DIST_CATALOGS_q) win32-installer-from-bundle: $(WIN32_RACKET) -l- distro-build/installer $(DIST_ARGS_q) @@ -332,11 +342,11 @@ win32-installer-from-bundle: # ------------------------------------------------------------ # Drive installer build: -DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(FARM_CONFIG)" "$(FARM_MODE)" \ +DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(CONFIG)" "$(CONFIG_MODE)" \ $(SERVER) "$(PKGS)" "$(DOC_SEARCH)" "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR) DRIVE_CMD_q = $(RACKET) -l- distro-build/drive-clients $(DRIVE_ARGS_q) -# Full server build and clients drive, based on `FARM_CONFIG': +# Full server build and clients drive, based on `CONFIG': farm: $(MAKE) server SERVE_DURING_CMD_qq='$(DRIVE_CMD_q)' diff --git a/pkgs/distro-build/drive-clients.rkt b/pkgs/distro-build/drive-clients.rkt index 6bbbc984ec..ae6b1fcc19 100644 --- a/pkgs/distro-build/drive-clients.rkt +++ b/pkgs/distro-build/drive-clients.rkt @@ -8,7 +8,8 @@ (only-in "farm.rkt" current-mode farm-config? - farm-config-tag farm-config-options farm-config-content)) + farm-config-tag farm-config-options farm-config-content) + "url-options.rkt") ;; See "farm.rkt" for an overview. @@ -194,8 +195,7 @@ (if l (apply ~a #:separator " " l) default-pkgs))) - (define doc-search (get-opt c '#:doc-search - default-doc-search)) + (define doc-search (choose-doc-search c default-doc-search)) (define dist-name (or (get-opt c '#:dist-name) default-dist-name)) (define dist-base (or (get-opt c '#:dist-base) @@ -203,7 +203,7 @@ (define dist-dir (or (get-opt c '#:dist-dir) default-dist-dir)) (define dist-suffix (get-opt c '#:dist-suffix "")) - (define dist-catalogs (get-opt c '#:dist-catalogs '(""))) + (define dist-catalogs (choose-catalogs c '(""))) (define pull? (get-opt c '#:pull? #t)) (~a " SERVER=" server " PKGS=" (q pkgs) @@ -220,7 +220,9 @@ (define dir (or (get-opt c '#:dir) "build/plt")) (define (sh . args) - (list "/bin/sh" "-c" (~a "'" (apply ~a args) "'"))) + (list "/bin/sh" "-c" (~a "'" + (regexp-replace* #rx"'" (apply ~a args) "'\"'\"'") + "'"))) (define j (or (get-opt c '#:j) 1)) (ssh-script host port user diff --git a/pkgs/distro-build/farm.rkt b/pkgs/distro-build/farm.rkt index 058509e2ac..dbd40cac79 100644 --- a/pkgs/distro-build/farm.rkt +++ b/pkgs/distro-build/farm.rkt @@ -4,11 +4,17 @@ ;; repository's top-level makefile. That target, in turn, uses the ;; `distro-build/drive-clients' module. ;; -;; Each client is built by running commands via `ssh', where the -;; client's host (and optional port and/or user) indicate the ssh -;; target. Each client machine must be set up with a public-key -;; authenticaion, because a direct `ssh' is expected to work without a -;; password prompt. +;; The server machine first prepares packages for installation on +;; clients. The farm configuration's top-level entry is consulted for +;; a `#:pkgs' and/or `#:doc-search' option, which overrides any `PKGS' +;; and/or `DOC_SEARCH' configuration from the makefile. +;; +;; The farm configuration file otherwise describes and configures +;; client machines. Each client is built by running commands via +;; `ssh', where the client's host (and optional port and/or user) +;; indicate the ssh target. Each client machine must be set up with a +;; public-key authenticaion, because a direct `ssh' is expected to +;; work without a password prompt. ;; ;; On the client machine, all work is performed with a git clone at a ;; specified directory. The directory defaults to "build/plt" (Unix, @@ -54,13 +60,14 @@ ;; individual machines, and groups them with `parallel' or ;; `sequential' to indicate whether the machine's builds should run ;; sequentially or in parallel. Options specified at `parallel' or -;; `sequential' are propagated to eachmachine in the group. +;; `sequential' are propagated to each machine in the group. ;; ;; For example, a configuration module might look like this: ;; ;; #lang distro-build/farm ;; ;; (sequential +;; #:pkgs '("drracket") ;; #:server "192.168.56.1" ;; (machine ;; #:desc "Linux (32-bit, Precise Pangolin)" @@ -90,33 +97,47 @@ ;; #:repo --- the git repository for Racket; defaults to ;; "http://:9440/.git" ;; #:pkgs '( ...) --- packages to install; defaults to -;; `PKGS' in the makfile (or, more genereally, +;; `PKGS' in the makefile (or, particularly, ;; the `pkgs' command-line argument to ;; `distro-build/drive-clients') +;; #:dist-base-url --- a URL that is used to construct +;; a default for #:doc-search and +;; #:dist-catalogs, where the +;; constructed values are consistent +;; with converting a build server's +;; content into a download site; since +;; URLs are constructed via relative +;; paths, this URL normally should end +;; with a slash ;; #:doc-search --- URL to install as the configuration ;; for remote documentation searches in ;; generated installers; "" is replaced -;; with the PLT default; defaults to the -;; `DOC_SEARCH' makefile variable or the -;; `doc-search' argument +;; with the PLT default; defaults to +;; #:dist-base-url (if present) extended +;; with "doc/search.html", or the +;; `DOC_SEARCH' makefile variable (or the +;; `doc-search' argument) ;; #:dist-name --- the distribution name; defaults to the -;; `DIST_NAME' makefile variable or `dist-name' -;; command-line argument +;; `DIST_NAME' makefile variable (or the +;; `dist-name' command-line argument) ;; #:dist-base --- the distribution's installater name prefix; ;; defaults to the `DIST_BASE' makefile variable -;; or the `dist-base' command-line argument +;; (or the `dist-base' command-line argument) ;; #:dist-dir --- the distribution's installation directory; ;; defaults to the `DIST_DIR' makefile variable -;; or the `dist-dir' command-line argument +;; (or the `dist-dir' command-line argument) ;; #:dist-suffix --- a suffix for the installer's name, usually ;; used for an OS variant; defaults to the -;; `DIST_SUFFIX' makefile variable or the -;; `dist-suffix' command-line argument +;; `DIST_SUFFIX' makefile variable (or the +;; `dist-suffix' command-line argument) ;; #:dist-catalogs '( ...) --- catalog URLs to install as the -;; initial catalog configuration in -;; generated installed, where "" -;; is replaced with the PLT default -;; catalogs +;; initial catalog configuration +;; in generated installed, where +;; "" is replaced with the PLT +;; default catalogs; defaults to +;; #:dist-base-url (if present) +;; extended with "catalogs" in a +;; list followed by "" ;; #:max-vm --- max number of VMs allowed to run with this ;; machine, counting the machine; defaults to 1 ;; #:vbox --- Virtual Box machine name; if provided the @@ -134,13 +155,14 @@ ;; machine starts by removing ; set this ;; to #f for a shared repo checkout; the default ;; is determined by the `CLEAN_MODE' makefile -;; variable or `--clean' command-line flag +;; variable (or `--clean' command-line flag) ;; #:pull? --- if true, then the build process on the client ;; machine starts by a `git pull' in ; set ;; to #f, for example, for a repo checkout that is ;; shared with server; the default is #t ;; ;; Machine-only keywords: +;; ;; #:name --- defaults to host; this string is recorded as ;; a description of the installer (for use in a ;; generated table of installer links, for example) @@ -185,8 +207,10 @@ ;; (current-mode s) -> void? ;; s : string? ;; A parameter whose value is the user's requested mode for this -;; configuration. The default mode is "default". The interpretation -;; of modes is completely up to the farm-configuration file. +;; configuration, normally as provided via the makefile's +;; `CONFIG_MODE' variable. The default mode is "default". The +;; interpretation of modes is completely up to the +;; farm configuration file. ;; ---------------------------------------- @@ -204,7 +228,8 @@ farm-config-tag farm-config-options farm-config-content - current-mode) + current-mode + extract-options) (module reader syntax/module-reader distro-build/farm) @@ -279,10 +304,16 @@ tag (for/hash ([kw (in-list kws)] [val (in-list kw-vals)]) + (define r (check kw val)) + (when (eq? r 'bad-keyword) + (error tag + (~a "unrecognized keyword for option\n" + " keyword: ~s") + kw)) (unless (check kw val) (error tag (~a "bad value for keyword\n" - " keyword: ~s" + " keyword: ~s\n" " value: ~e") kw val)) @@ -301,6 +332,7 @@ [(#:dist-dir) (simple-string? val)] [(#:dist-suffix) (simple-string? val)] [(#:dist-catalogs) (and (list? val) (andmap string? val))] + [(#:dist-base-url) (string? val)] [(#:max-vm) (real? val)] [(#:server) (simple-string? val)] [(#:host) (simple-string? val)] @@ -317,7 +349,7 @@ [(#:repo) (string? val)] [(#:clean?) (boolean? val)] [(#:pull?) (boolean? val)] - [else #f])) + [else 'bad-keyword])) (define (check-machine-keyword kw val) (case kw @@ -331,3 +363,12 @@ (regexp-match #rx"^[-a-zA-A0-9.]*$" s))) (define current-mode (make-parameter "default")) + +(define (extract-options config-file config-mode) + (or + (and (file-exists? config-file) + (parameterize ([current-mode config-mode]) + (farm-config-options + (dynamic-require (path->complete-path config-file) 'farm-config)))) + (hash))) + diff --git a/pkgs/distro-build/install-pkgs.rkt b/pkgs/distro-build/install-pkgs.rkt new file mode 100644 index 0000000000..f6446e9ba3 --- /dev/null +++ b/pkgs/distro-build/install-pkgs.rkt @@ -0,0 +1,21 @@ +#lang racket/base +(require racket/cmdline + racket/string + (only-in "farm.rkt" extract-options)) + +(define-values (config-file config-mode default-pkgs flags) + (command-line + #:args + (config-file config-mode pkgs . flag) + (values config-file config-mode pkgs flag))) + +(define pkgs (or (hash-ref (extract-options config-file config-mode) + '#:pkgs + #f) + (string-split default-pkgs))) + +(parameterize ([current-command-line-arguments + (list->vector (append (list "pkg" "install") + flags + pkgs))]) + (dynamic-require 'raco #f)) diff --git a/pkgs/distro-build/set-config.rkt b/pkgs/distro-build/set-config.rkt index 1e93277193..dacf67b114 100644 --- a/pkgs/distro-build/set-config.rkt +++ b/pkgs/distro-build/set-config.rkt @@ -1,17 +1,25 @@ #lang racket/base (require racket/cmdline racket/file - racket/path) + racket/path + (only-in "farm.rkt" extract-options) + "url-options.rkt") -(define-values (config-file doc-search catalogs) +(define-values (dest-config-file config-file config-mode default-doc-search default-catalogs) (command-line #:args - (config-file doc-search . catalog) - (values config-file doc-search catalog))) + (dest-config-file config-file config-mode doc-search . catalog) + (values dest-config-file config-file config-mode doc-search catalog))) + +(define config (extract-options config-file config-mode)) + +(define doc-search (choose-doc-search config default-doc-search)) + +(define catalogs (choose-catalogs config default-catalogs)) (define orig - (if (file-exists? config-file) - (call-with-input-file* config-file read) + (if (file-exists? dest-config-file) + (call-with-input-file* dest-config-file read) (hash))) (let* ([table orig] @@ -27,8 +35,8 @@ #f c))))]) (unless (equal? table orig) - (make-directory* (path-only config-file)) - (call-with-output-file config-file + (make-directory* (path-only dest-config-file)) + (call-with-output-file dest-config-file #:exists 'truncate (lambda (o) (write table o) diff --git a/pkgs/distro-build/stamp.rkt b/pkgs/distro-build/stamp.rkt new file mode 100644 index 0000000000..3f08641e62 --- /dev/null +++ b/pkgs/distro-build/stamp.rkt @@ -0,0 +1,27 @@ +#lang racket/base +(require racket/system + racket/format) + +(provide get-date-stamp + get-commit-stamp + get-date+commit-stamp) + +(define (get-commit-stamp) + (define git (or (find-executable-path "git") + (find-executable-path "git.exe"))) + (define s (open-output-string)) + (parameterize ([current-output-port s] + [current-input-port (open-input-string "")]) + (system* git "log" "-1" "--pretty=format:%h")) + (define commit-id (get-output-string s)) + commit-id) + +(define (get-date-stamp) + (define d (seconds->date (current-seconds))) + (define (n n) (~a n #:align 'right #:pad-string "0" #:width 2)) + (~a (date-year d) (n (date-month d)) (n (date-day d)))) + +(define (get-date+commit-stamp) + (~a (get-date-stamp) + "-" + (get-commit-stamp))) diff --git a/pkgs/distro-build/url-options.rkt b/pkgs/distro-build/url-options.rkt new file mode 100644 index 0000000000..a751541c2e --- /dev/null +++ b/pkgs/distro-build/url-options.rkt @@ -0,0 +1,22 @@ +#lang racket/base +(require net/url) + +(provide choose-doc-search + choose-catalogs) + +(define (choose-doc-search config default-doc-search) + (or (hash-ref config '#:doc-search #f) + (let ([v (hash-ref config '#:dist-base-url #f)]) + (and v + (url->string + (combine-url/relative (string->url v) "docs/search.html")))) + default-doc-search)) + +(define (choose-catalogs config default-catalogs) + (or (hash-ref config '#:dist-catalogs #f) + (let ([v (hash-ref config '#:dist-base-url #f)]) + (and v + (list (url->string + (combine-url/relative (string->url v) "catalog")) + ""))) + default-catalogs)) diff --git a/racket/lib/collects/compiler/private/winsubsys.rkt b/racket/lib/collects/compiler/private/winsubsys.rkt index 0cc699833e..762adb115c 100644 --- a/racket/lib/collects/compiler/private/winsubsys.rkt +++ b/racket/lib/collects/compiler/private/winsubsys.rkt @@ -5,7 +5,7 @@ (define DF_SubsystemOffset #x5C) (define (set-subsystem file subsys) - (let-values ([(in out) (open-input-output-file file 'update)]) + (let-values ([(in out) (open-input-output-file file #:exists 'update)]) (file-position in DF_NewHeaderOffset) (let ([offset (integer-bytes->integer (read-bytes 4 in) #f #f)]) (file-position out (+ offset DF_SubsystemOffset))