diff --git a/INSTALL.txt b/INSTALL.txt index 7f0ee5f98e..396dece36a 100644 --- a/INSTALL.txt +++ b/INSTALL.txt @@ -154,30 +154,30 @@ create an installer for the server's platform. Running Build Farms ------------------- -The `farm' target of the makefile will do everything to generate +The `installers' target of the makefile will do everything to generate installers: build a server on the current machine, run clients on -hosts specified via FARM_CONFIG, and start/stop VirtualBox virtual -machines that implement clients. +hosts specified via CONFIG, and start/stop VirtualBox virtual machines +that act as client machines. See - pkgs/distro-build/farm.rkt + pkgs/distro-build/config.rkt -for a description of the farm-configuration module and requirements on +for a description of the site-configuration module and requirements on client hosts. -If "my-farm-config.rkt" is a configuration module, then +If "my-site-config.rkt" is a configuration module, then - make farm CONFIG=my-farm-config.rkt + make installers CONFIG=my-site-config.rkt drives the build farm, and the resulting installers are in "build/installers", with a hash table mapping descriptions to installer filenames in "build/installer/table.rktd". -The default CONFIG path is "build/farm-config.rkt", so you could put +The default CONFIG path is "build/site-config.rkt", so you could put your configuration file there and omit the `CONFIG' argument to `make'. Supply `CONFIG_MODE=...' to pass a configuration mode on to -your farm-configuration module (accessible via the `current-mode' +your site-configuration module (accessible via the `current-mode' parameter). Supply `CLEAN_MODE=--clean' to make the default `#:clean?' configration for a client #t instead of #f. @@ -185,7 +185,7 @@ A configuration file can specify the packages to include, host address of the server, distribution name, installer directory, and documentation search URL, but defaults can be provided as `make' arguments via `PKGS', `SERVER', `DIST_NAME', `DIST_BASE', and -`DIST_DIR', `DOC_SEARCH', respectively. The farm configuration's +`DIST_DIR', `DOC_SEARCH', respectively. The site configuration's top-level options for packages and documentation search URL are used to configure the set of packages that are available to client machines to include in installers. @@ -199,15 +199,15 @@ where defaults to "racket" (but can be set via `DIST_BASE'), is from `(system-library-subpath #f)' but normalizing the Windows results to "i386-win32" and "x86_63-win32", - is omitted unless a `#:dist-suffix' string is specified -for the client in the farm configuration, and is +for the client in the site configuration, and is platform-specific: ".sh" for Unix, ".dmg" for Mac Os X, and ".exe" for Windows. Separate Server and Clients --------------------------- -Instead of using the `farm' makefile target and a farm configuration -file, you can run server and client processes manually. +Instead of using the `installers' makefile target and a site +configuration file, you can run server and client processes manually. Roughly, the steps are diff --git a/Makefile b/Makefile index 37464b094a..5f4956727f 100644 --- a/Makefile +++ b/Makefile @@ -113,10 +113,10 @@ DIST_DESC = DIST_CATALOGS_q = "" # Configuration of clients to run for a build farm, normally -# implemented with `#lang distro-build/farm': -CONFIG = build/farm-config.rkt +# implemented with `#lang distro-build/config': +CONFIG = build/site.rkt -# A mode that is made available to the farm-configuration module +# A mode that is made available to the site-configuration module # through the `current-mode' parameter: CONFIG_MODE = default @@ -125,7 +125,7 @@ CONFIG_MODE = default CLEAN_MODE = # A command to run after the server has started; normally set by -# the `farm' target: +# the `installers' target: SERVE_DURING_CMD_qq = # ------------------------------------------------------------ @@ -266,7 +266,7 @@ binary-catalog-server: # # The `client' and `win32-client' targets are also used by # `distro-buid/drive-clients', which is in turn run by the -# `farm' target. +# `installers' target. # # For a non-Windows machine, if "build/drive" exists, then # keep the "build/user" directory on the grounds that the @@ -340,18 +340,18 @@ win32-installer-from-bundle: $(WIN32_RACKET) -l- distro-build/installer $(DIST_ARGS_q) # ------------------------------------------------------------ -# Drive installer build: +# Drive installer build across server and clients: DRIVE_ARGS_q = $(RELEASE_MODE) $(CLEAN_MODE) "$(CONFIG)" "$(CONFIG_MODE)" \ $(SERVER) "$(PKGS)" "$(DOC_SEARCH)" "$(DIST_NAME)" $(DIST_BASE) $(DIST_DIR) DRIVE_CMD_q = $(RACKET) -l- distro-build/drive-clients $(DRIVE_ARGS_q) # Full server build and clients drive, based on `CONFIG': -farm: +installers: $(MAKE) server SERVE_DURING_CMD_qq='$(DRIVE_CMD_q)' # Server is already built; start it and drive clients: -built-farm: +installers-from-built: $(MAKE) built-catalog-server SERVE_DURING_CMD_qq='$(DRIVE_CMD_q)' # Just the clients, assuming server is already running: diff --git a/pkgs/distro-build/assemble-site.rkt b/pkgs/distro-build/assemble-site.rkt new file mode 100644 index 0000000000..334b2e7dc9 --- /dev/null +++ b/pkgs/distro-build/assemble-site.rkt @@ -0,0 +1,64 @@ +#lang racket/base +(require racket/cmdline + racket/file + net/url + "download-page.rkt" + (only-in "site.rkt" extract-options)) + +(define build-dir (build-path "build")) +(define dest-dir (build-path build-dir "site")) + +(define built-dir (build-path build-dir "built")) + +(define installers-dir (build-path "installers")) +(define pkgs-dir (build-path "pkgs")) +(define catalog-dir (build-path "catalog")) + +(define-values (config-file config-mode) + (command-line + #:args + (config-file config-mode) + (values config-file config-mode))) + +(define config (extract-options config-file config-mode)) + +(define (copy dir [build-dir build-dir]) + (make-directory* dest-dir) + (printf "Copying ~s\n" (build-path build-dir dir)) + (copy-directory/files (build-path build-dir dir) + (build-path dest-dir dir) + #:keep-modify-seconds? #t)) + +(delete-directory/files dest-dir #:must-exist? #f) +(copy pkgs-dir built-dir) + +(printf "Building catalog\n") +(let ([c-dir (build-path built-dir catalog-dir "pkg")] + [d-dir (build-path dest-dir catalog-dir "pkg")]) + (make-directory* d-dir) + (define base-url (string->url (hash-ref config '#:dist-base-url))) + (for ([f (in-list (directory-list c-dir))]) + (define ht (call-with-input-file* (build-path c-dir f) read)) + (define new-ht + (hash-set ht 'source (url->string + (combine-url/relative + base-url + (path->string + (build-path + "pkgs" + (path-add-suffix f #".zip"))))))) + (call-with-output-file* + (build-path d-dir f) + (lambda (o) + (write new-ht o) + (newline o))))) + +(copy installers-dir) + +(make-download-page (build-path build-dir + installers-dir + "table.rktd") + #:installers-url "installers/" + #:dest (build-path dest-dir + "index.html") + #:git-clone (current-directory)) diff --git a/pkgs/distro-build/farm.rkt b/pkgs/distro-build/config.rkt similarity index 83% rename from pkgs/distro-build/farm.rkt rename to pkgs/distro-build/config.rkt index dbd40cac79..9f4a9bb87e 100644 --- a/pkgs/distro-build/farm.rkt +++ b/pkgs/distro-build/config.rkt @@ -1,15 +1,15 @@ #lang racket/base -;; A build farm is normally run via the `farm' target of the Racket -;; repository's top-level makefile. That target, in turn, uses the -;; `distro-build/drive-clients' module. +;; A build farm is normally run via the `installers' target of the +;; Racket repository's top-level makefile. That target, in turn, uses +;; the `distro-build/drive-clients' module. ;; ;; The server machine first prepares packages for installation on -;; clients. The farm configuration's top-level entry is consulted for +;; clients. The site configuration's top-level entry is consulted for ;; a `#:pkgs' and/or `#:doc-search' option, which overrides any `PKGS' ;; and/or `DOC_SEARCH' configuration from the makefile. ;; -;; The farm configuration file otherwise describes and configures +;; The site configuration file otherwise describes and configures ;; client machines. Each client is built by running commands via ;; `ssh', where the client's host (and optional port and/or user) ;; indicate the ssh target. Each client machine must be set up with a @@ -52,11 +52,11 @@ ;; or C:\Program Files (x86)\NSIS\makensis.exe ;; or instaled so that `makensis' in in yur PATH. ;; -;; Farm Configuration +;; Site Configuration ;; ------------------- ;; -;; A farm configuration module is normally wriiten in the -;; `distro-build/farm' language. The configuration describes +;; A site configuration module is normally wriiten in the +;; `distro-build/config' language. The configuration describes ;; individual machines, and groups them with `parallel' or ;; `sequential' to indicate whether the machine's builds should run ;; sequentially or in parallel. Options specified at `parallel' or @@ -64,7 +64,7 @@ ;; ;; For example, a configuration module might look like this: ;; -;; #lang distro-build/farm +;; #lang distro-build/config ;; ;; (sequential ;; #:pkgs '("drracket") @@ -85,7 +85,7 @@ ;; #:bits 64)) ;; ;; -;; Farm-configuration keywords (where means no spaces, etc.): +;; Site-configuration keywords (where means no spaces, etc.): ;; ;; #:host --- defaults to "localhost" ;; #:port --- ssh port for the client; defaults to 22 @@ -168,40 +168,40 @@ ;; generated table of installer links, for example) ;; ;; -;; More precisely, the `distro-build/farm' language is like +;; More precisely, the `distro-build/config' language is like ;; `racket/base' except that the module body must have exactly one ;; expression (plus any number of definitions, etc.) that produces a -;; farm-configuration value. The value is exported as `farm-config' -;; from the module. Any module can act as a farm-configuration module -;; a long as it exports `farm-config' as a farm-configuration value. +;; site-configuration value. The value is exported as `site-config' +;; from the module. Any module can act as a site-configuration module +;; a long as it exports `site-config' as a site-configuration value. ;; -;; The `distro-build/farm' language also adds the following functions +;; The `distro-build/config' language also adds the following functions ;; to `racket/base': ;; -;; (machine ... ...) -> farm-config? -;; Produces a farm configuration based on the given keyword-based +;; (machine ... ...) -> site-config? +;; Produces a site configuration based on the given keyword-based ;; options. The support keyword arguments are described above. ;; ;; (sequential ... ... config ...) -;; -> farm-config? -;; config : farm-config? -;; Produces a farm configuration that runs each `config' +;; -> site-config? +;; config : site-config? +;; Produces a site configuration that runs each `config' ;; sequentially. The support keyword arguments are described above. ;; ;; (parallel ... ... config ...) -;; -> farm-config? -;; config : farm-config? -;; Produces a farm configuration that runs each `config' in +;; -> site-config? +;; config : site-config? +;; Produces a site configuration that runs each `config' in ;; parallel. The support keyword arguments are described above. ;; -;; (farm-config? v) -> boolean? -;; (farm-config-tag config) -> (or/c 'machine 'sequential 'parallel) -;; config : farm-config? -;; (farm-config-options config) -> (hash/c keyword? any/c) -;; config : farm-config? -;; (farm-config-content config) -> (listof farm-config?) -;; config : farm-config? -;; Farm configuation inspection +;; (site-config? v) -> boolean? +;; (site-config-tag config) -> (or/c 'machine 'sequential 'parallel) +;; config : site-config? +;; (site-config-options config) -> (hash/c keyword? any/c) +;; config : site-config? +;; (site-config-content config) -> (listof site-config?) +;; config : site-config? +;; Site configuation inspection ;; ;; (current-mode) -> string? ;; (current-mode s) -> void? @@ -210,7 +210,7 @@ ;; configuration, normally as provided via the makefile's ;; `CONFIG_MODE' variable. The default mode is "default". The ;; interpretation of modes is completely up to the -;; farm configuration file. +;; site configuration file. ;; ---------------------------------------- @@ -224,32 +224,32 @@ sequential parallel machine - farm-config? - farm-config-tag - farm-config-options - farm-config-content + site-config? + site-config-tag + site-config-options + site-config-content current-mode extract-options) (module reader syntax/module-reader - distro-build/farm) + distro-build/site) -(struct farm-config (tag options content)) +(struct site-config (tag options content)) (define-syntax-rule (module-begin form ...) - (#%plain-module-begin (farm-begin #f form ...))) + (#%plain-module-begin (site-begin #f form ...))) -(define-syntax (farm-begin stx) +(define-syntax (site-begin stx) (syntax-case stx () [(_ #t) #'(begin)] [(_ #f) - (raise-syntax-error 'farm - "did not find an expression for the farm configuration")] + (raise-syntax-error 'site + "did not find an expression for the site configuration")] [(_ found? next . rest) (let ([expanded (local-expand #'next 'module (kernel-form-identifier-list))]) (syntax-case expanded (begin) [(begin next1 ...) - #`(farm-begin found? next1 ... . rest)] + #`(site-begin found? next1 ... . rest)] [(id . _) (and (identifier? #'id) (ormap (lambda (kw) (free-identifier=? #'id kw)) @@ -262,24 +262,24 @@ module* #%require #%provide)))) - #`(begin #,expanded (farm-begin found? . rest))] + #`(begin #,expanded (site-begin found? . rest))] [_else (if (syntax-e #'found?) - (raise-syntax-error 'farm + (raise-syntax-error 'site "found second top-level expression" #'next) #`(begin - (provide farm-config) - (define farm-config (let ([v #,expanded]) - (unless (farm-config? v) - (error 'farm - (~a "expression did not produce a farm configuration\n" + (provide site-config) + (define site-config (let ([v #,expanded]) + (unless (site-config? v) + (error 'site + (~a "expression did not produce a site configuration\n" " result: ~e\n" " expression: ~.s") v 'next)) v)) - (farm-begin + (site-begin #t . rest)))]))])) @@ -300,7 +300,7 @@ check-machine-keyword 'machine)))) (define (constructor kws kw-vals subs check tag) - (farm-config + (site-config tag (for/hash ([kw (in-list kws)] [val (in-list kw-vals)]) @@ -319,8 +319,8 @@ val)) (values kw val)) (for/list ([sub subs]) - (unless (farm-config? sub) - (raise-argument-error tag "farm-config?" sub)) + (unless (site-config? sub) + (raise-argument-error tag "site-config?" sub)) sub))) (define (check-group-keyword kw val) @@ -368,7 +368,7 @@ (or (and (file-exists? config-file) (parameterize ([current-mode config-mode]) - (farm-config-options - (dynamic-require (path->complete-path config-file) 'farm-config)))) + (site-config-options + (dynamic-require (path->complete-path config-file) 'site-config)))) (hash))) diff --git a/pkgs/distro-build/download-page.rkt b/pkgs/distro-build/download-page.rkt new file mode 100644 index 0000000000..7078750509 --- /dev/null +++ b/pkgs/distro-build/download-page.rkt @@ -0,0 +1,99 @@ +#lang racket/base +(require racket/format + racket/path + racket/system + net/url + openssl/sha1 + xml) + +(provide make-download-page) + +(module+ main + (require racket/cmdline) + + (define args null) + (define (arg! kw val) + (set! args (cons (cons kw val) args))) + + (define table-file + (command-line + #:once-each + [("--at") url "URL for installaters reletaive to download page" + (arg! '#:installers-url url)] + [("--dest") file "Write to " + (arg! '#:dest file)] + [("--git") dir "Report information from git clone " + (arg! '#:git-clone dir)] + #:args + (table-file) + table-file)) + + (let ([args (sort args keywordstring + (combine-url/relative + (string->url installers-url) + inst)))) + ,key)) + (td nbsp) + (td (span ([class "detail"]) + "SHA1: " + (span ([class "checksum"]) + ,(call-with-input-file* + (build-path (path-only table-file) + inst) + sha1))))))) + ,@(if git-clone + (let ([git (find-executable-path "git")]) + (define origin (let ([s (system*/string git "remote" "show" "origin")]) + (define m (regexp-match #rx"(?m:Fetch URL: (.*)$)" s)) + (if m + (cadr m) + "???"))) + (define stamp (system*/string git "log" "-1" "--format=%H")) + `((p + (div (span ([class "detail"]) "Repository: " (span ([class "path"]) ,origin))) + (div (span ([class "detail"]) "Commit: " (span ([class "checksum"]) ,stamp)))))) + null))) + o) + (void))))) diff --git a/pkgs/distro-build/drive-clients.rkt b/pkgs/distro-build/drive-clients.rkt index ae6b1fcc19..c651e9ec80 100644 --- a/pkgs/distro-build/drive-clients.rkt +++ b/pkgs/distro-build/drive-clients.rkt @@ -5,13 +5,13 @@ racket/format racket/file racket/string - (only-in "farm.rkt" + (only-in "config.rkt" current-mode - farm-config? - farm-config-tag farm-config-options farm-config-content) + site-config? + site-config-tag site-config-options site-config-content) "url-options.rkt") -;; See "farm.rkt" for an overview. +;; See "config.rkt" for an overview. ;; ---------------------------------------- @@ -35,24 +35,24 @@ dist-name dist-base dist-dir))) (define config (parameterize ([current-mode config-mode]) - (dynamic-require (path->complete-path config-file) 'farm-config))) + (dynamic-require (path->complete-path config-file) 'site-config))) -(unless (farm-config? config) +(unless (site-config? config) (error 'drive-clients - "configuration module did not provide a farm-configuration value: ~e" + "configuration module did not provide a site-configuration value: ~e" config)) ;; ---------------------------------------- (define (merge-options opts c) - (for/fold ([opts opts]) ([(k v) (in-hash (farm-config-options c))]) + (for/fold ([opts opts]) ([(k v) (in-hash (site-config-options c))]) (hash-set opts k v))) (define (get-opt opts kw [default #f]) (hash-ref opts kw default)) (define (get-content c) - (farm-config-content c)) + (site-config-content c)) (define (client-name opts) (or (get-opt opts '#:name) @@ -333,12 +333,12 @@ [mode 'sequential] [opts (hasheq)]) (unless stop? - (case (farm-config-tag config) + (case (site-config-tag config) [(parallel sequential) (define new-opts (merge-options opts config)) (define ts (map (lambda (c) (loop c - (farm-config-tag config) + (site-config-tag config) new-opts)) (get-content config))) (define (wait) diff --git a/pkgs/distro-build/install-pkgs.rkt b/pkgs/distro-build/install-pkgs.rkt index f6446e9ba3..ded15820ee 100644 --- a/pkgs/distro-build/install-pkgs.rkt +++ b/pkgs/distro-build/install-pkgs.rkt @@ -1,7 +1,7 @@ #lang racket/base (require racket/cmdline racket/string - (only-in "farm.rkt" extract-options)) + (only-in "site.rkt" extract-options)) (define-values (config-file config-mode default-pkgs flags) (command-line diff --git a/pkgs/distro-build/set-config.rkt b/pkgs/distro-build/set-config.rkt index dacf67b114..4ae3c075fa 100644 --- a/pkgs/distro-build/set-config.rkt +++ b/pkgs/distro-build/set-config.rkt @@ -2,7 +2,7 @@ (require racket/cmdline racket/file racket/path - (only-in "farm.rkt" extract-options) + (only-in "config.rkt" extract-options) "url-options.rkt") (define-values (dest-config-file config-file config-mode default-doc-search default-catalogs) diff --git a/pkgs/distro-build/url-options.rkt b/pkgs/distro-build/url-options.rkt index a751541c2e..3ccece267f 100644 --- a/pkgs/distro-build/url-options.rkt +++ b/pkgs/distro-build/url-options.rkt @@ -17,6 +17,6 @@ (let ([v (hash-ref config '#:dist-base-url #f)]) (and v (list (url->string - (combine-url/relative (string->url v) "catalog")) + (combine-url/relative (string->url v) "catalog/")) ""))) default-catalogs))