raco setup: multi-machine cross-compile support

Enable `raco {setup|make}` to build two sets of compiled files: one
set that is suitable for the current machine, and another set that is
suitable for a different machine or for all machines (i.e.,
machine-independent bytecode).

In the long run, this new `raco setup` mode support cross compilation
where the build machine and target machine have different bytecode
formats --- unlike the current cross-compliation mode, which relies on
there being a single bytecode format in traditional Racket for all
platforms.

In the short run, the new mode enables the faster creation of
Racket-on-Chez distribution builds. The build server can send out
machine-independent bytecode to client machines while using
machine-specific bytecode for itself to drive the build process.

The new compilation mode relies on a somewhat delicate balance of the
`current-compile-target-machine` and `current-compiled-file-roots`
parameters (as reflected by the `-M` and `-R` command-line flags for
Racket) as well as cross-compilation mode (as enabled by the `-C`
command-line flag).

INSTALL.txt

@@ -247,7 +247,9 @@ Specify `SETUP_MACHINE_FLAGS=...` to set Racket flags that control the
 target machine of compiled bytecode for `raco setup` and `raco pkg
 install`. For example `SETUP_MACHINE_FLAGS=-M` causes the generated
 bytecode to be machine-independent, which is mainly useful when the
-generated installation will be used as a template for other platforms.
+generated installation will be used as a template for other platforms
+or for cross-compilation.
+
 
 Installing Packages
 -------------------
@@ -355,12 +357,6 @@ respectively. The site configuration's top-level options for packages
 and documentation search URL are used to configure the set of packages
 that are available to client machines to include in installers.
 
-The `SETUP_MACHINE_FLAGS` argument to the makefile affects how
-bytecode is compiled on the server for distribrution to clients. Use
-`SETUP_MACHINE_FLAGS=-M` when building clients with the variant 'cs,
-since that will significantly speed up client builds with only a
-modest slowdown on the server.
-
 For each installer written to "build/installers", the installer's name
 is
 
@@ -374,6 +370,16 @@ for the client in the site configuration, and <ext> is
 platform-specific: ".sh" for Unix (including Linux), ".dmg" or ".pkg"
 for Mac OS, and ".exe" for Windows.
 
+When building clients with variant 'cs, you can speed up the build
+process by using `SERVER_COMPILE_MACHINE=any` with `make installers`.
+That mode causes the server to send built packages to clients in
+machine-independent form, which will need to be recompiled to the
+client's native format for bytecode, but recompiling from
+machine-independent form can be much faster than compiling from
+source. Beware that `SERVER_COMPILE_MACHINE=any`cannot be used in
+a respository checkout formerly used without it, unless you first
+run `raco setup --clean`.
+
 Generating Installer Web Sites
 ------------------------------
 

Makefile

@@ -582,7 +582,17 @@ win32-pkgs-catalog:
 # These targets require GNU `make', so that we don't have to propagate
 # variables through all of the target layers.
 
+SERVER_COMPILE_MACHINE = machine-specific
+ANY_COMPILE_MACHINE_ARGS_qq = SETUP_MACHINE_FLAGS="-MCR `pwd`/build/zo:" \
+                              MORE_CONFIGURE_ARGS="$(MORE_CONFIGURE_ARGS) --enable-crossany"
+
 server:
+	if [ "$(SERVER_COMPILE_MACHINE)" = "any" ] ; \
+         then $(MAKE) plain-server $(ANY_COMPILE_MACHINE_ARGS_qq) ; \
+         else $(MAKE) plain-server ; fi
+
+plain-server:
+	rm -rf build/zo
 	$(MAKE) base
 	$(MAKE) server-from-base
 

pkgs/racket-doc/scribblings/raco/make.scrbl

@@ -9,6 +9,7 @@
                      compiler/cm
                      compiler/cm-accomplice
                      setup/parallel-build
+                     setup/cross-system
                      compiler/compilation-path
                      compiler/compile-file
                      syntax/modread
@@ -229,6 +230,9 @@ file if
   @item{the version recorded in the @filepath{.dep} file does not
         match the result of @racket[(version)];}
 
+  @item{the target machine recorded in the @filepath{.dep} file does
+        not match the result of @racket[(current-compile-target-machine)];}
+
   @item{the source hash recorded in the @filepath{.dep} file does not
         match the current source hash;}
 
@@ -258,7 +262,27 @@ dependencies can be installed during compilation via
 @racketmodname[compiler/cm-accomplice]. The @filepath{.dep} file also
 records the SHA-1 hash of the module's source, and it records a
 combined SHA-1 hash of all of the dependencies that includes their
-recursive dependencies.
+recursive dependencies. If a bytecode file is generated by recompiling
+a bytecode file that was formerly compiled as machine-independent, then
+the @filepath{.dep} file also records the SHA-1 hash of the
+machine-independent form, since the recompiled module's behavior should
+be exactly the same.
+
+The special combination of @racket[(cross-installation?)] as
+@racket[#t], @racket[(current-compile-target-machine)] as @racket[#f],
+and @racket[(current-compiled-file-roots)] having two or more elements
+triggers a special compilation mode. Bytecode specific to the running
+Racket is written to the directory determined by the first element of
+@racket[(current-compiled-file-roots)]. Bytecode specific to the
+cross-compilation target is written to the directory determined by the
+first element of @racket[(current-compiled-file-roots)]. By
+configuring @racket[(current-compiled-file-roots)] so that the first
+element is outside a build tree and the second element is inside the
+build tree, cross-compilation can create a build tree suitable for the
+target machine while building and loading bytecode (for macro
+expansion, etc.) that is usable on the current machine. This mode
+works correctly for a build directory that starts with only source
+code and machine-independent bytecode.
 
 The handler caches timestamps when it checks @filepath{.dep} files,
 and the cache is maintained across calls to the same handler. The

racket/collects/compiler/private/cm-minimal.rkt

@@ -9,6 +9,7 @@
          racket/promise
          file/sha1
          setup/collects
+         setup/cross-system
          compiler/compilation-path
          compiler/private/dep)
 
@@ -272,12 +273,13 @@
                  op)
           (newline op))))))
 
-(define (write-updated-deps deps assume-compiled-sha1 zo-name)
+(define (write-updated-deps deps assume-compiled-sha1 zo-name
+                            #:target-machine [target-machine (current-compile-target-machine)])
   (let ([dep-path (path-replace-extension zo-name #".dep")])
     (with-compile-output dep-path
       (lambda (op tmp-path)
         (write (list* (version)
-                      (current-compile-target-machine)
+                      target-machine
                       (cons (deps-src-sha1 deps)
                             (cons (deps-imports-sha1 deps)
                                   assume-compiled-sha1))
@@ -310,6 +312,58 @@
                                         ", which appears to be in the future"
                                         ""))])))
 
+(define (cross-multi-compile? roots)
+  ;; Combination of cross-installation mode, compiling to machine-independent form,
+  ;; and multiple compiled-file roots triggers a special mutli-target compilation mode.
+  ;; Write code compiled for the running Racket to the first root, and write code for
+  ;; the cross-compile target to the second root --- but count the cross-compile target
+  ;; as machine-independent if it would be the same as the current target.
+  (and ((length roots) . > . 1)
+       (cross-installation?)
+       (not (current-compile-target-machine))))
+
+;; Handle cross-multi-compile mode, or just continue on to `compile-zo*`
+(define (compile-zo*/cross-compile path->mode roots path src-sha1 read-src-syntax orig-zo-name
+                                   up-to-date collection-cache
+                                   #:recompile-from recompile-from
+                                   #:assume-compiled-sha1 assume-compiled-sha1
+                                   #:use-existing-deps use-existing-deps)
+  (cond
+    [(cross-multi-compile? roots)
+     (define running-root (car roots))
+     (define target-root (cadr roots))
+     ;; First, generate machine-independent form at the second root:
+     (define mi-zo-name
+       (compile-zo* path->mode (list target-root) path src-sha1 read-src-syntax #f up-to-date collection-cache
+                    #:recompile-from recompile-from
+                    #:assume-compiled-sha1 assume-compiled-sha1
+                    #:use-existing-deps use-existing-deps))
+     (define mi-dep-path (path-replace-extension mi-zo-name #".dep"))
+     (define mi-deps (call-with-input-file* mi-dep-path read))
+     (define mi-sha1 (or (deps-assume-compiled-sha1 mi-deps)
+                         (call-with-input-file* mi-zo-name sha1)))
+     ;; Recompile to running-Racket form:
+     (define running-zo
+       (parameterize ([current-compile-target-machine (system-type 'target-machine)])
+         (compile-zo* path->mode (list running-root) path src-sha1 read-src-syntax #f up-to-date collection-cache
+                      #:recompile-from mi-zo-name
+                      #:assume-compiled-sha1 mi-sha1
+                      #:use-existing-deps mi-deps)))
+     (when (cross-system-type 'target-machine)
+       ;; Recompile to cross-compile target form:
+       (parameterize ([current-compile-target-machine (cross-system-type 'target-machine)])
+         (compile-zo* path->mode (list target-root) path src-sha1 read-src-syntax #f up-to-date collection-cache
+                      #:recompile-from mi-zo-name
+                      #:assume-compiled-sha1 mi-sha1
+                      #:use-existing-deps mi-deps)))
+     running-zo]
+    [else
+     ;; Regular mode, just [re]compile:
+     (compile-zo* path->mode roots path src-sha1 read-src-syntax orig-zo-name up-to-date collection-cache
+                  #:recompile-from recompile-from
+                  #:assume-compiled-sha1 assume-compiled-sha1
+                  #:use-existing-deps use-existing-deps)]))
+
 (define-struct ext-reader-guard (proc top)
   #:property prop:procedure (struct-field-index proc))
 (define-struct file-dependency (path module?) #:prefab)
@@ -402,7 +456,7 @@
     (get-compilation-dir+name path #:modes (list (path->mode path)) #:roots dest-roots))
   (define zo-name
     ;; If we have multiple roots, make sure that compilation uses the first one
-    (if (pair? (cdr roots))
+    (if (or (pair? (cdr roots)) (not orig-zo-name))
         (build-path code-dir (path-add-suffix code-name #".zo"))
         orig-zo-name))
 
@@ -463,6 +517,12 @@
         ;; Note that we check time and write ".dep" before returning from
         ;; with-compile-output...
         (verify-times path tmp-name)
+        (when (equal? recompile-from zo-name)
+          ;; In the case of recompiling, make sure that any concurrent
+          ;; process always sees recompile possibilities by writing
+          ;; the expected sha1 into ".dep" before deleting the ".zo"
+          (write-updated-deps use-existing-deps assume-compiled-sha1 zo-name
+                              #:target-machine #f))
         ;; Explicitly delete target file before writing ".dep", just so
         ;; ".dep" is doesn't claim a description of the wrong file
         (when (file-exists? zo-name)
@@ -474,7 +534,10 @@
            (write-deps code zo-name path->mode dest-roots path src-sha1
                        external-deps external-module-deps reader-deps 
                        up-to-date collection-cache read-src-syntax)])))
-    (trace-printf "wrote zo file: ~a" zo-name)))
+    (trace-printf "wrote zo file: ~a" zo-name))
+
+  ;; Return generated ".zo" path:
+  zo-name)
 
 (define (recompile-module-code recompile-from src-path deps collection-cache)
   ;; Force potential recompilation of dependencies. Otherwise, we
@@ -529,10 +592,38 @@
             alt-path
             path))))
 
-;; If `trying-sha1?`, then don't actually compile, but return a
+;; The `maybe-compile-zo` check is the ultimate word on whether a file
-;; boolean indicating whether a build is needed. Otherwise, actually
+;; needs to be recompiled. It is called through the `compile-root`
-;; build if the compiled form is out of date, and return #f to report
+;; layer, which tries to take shortcuts based on file timestamps and a
-;; that no further build is needed.
+;; cached decisions.
+;;
+;; There's a catch here: If `trying-sha1?` is #t, then the question is
+;; "must a recorded SHA-1 be disbelieved?", and it must be answered
+;; without committing to compiling the file right now. Crucially,
+;; calling the lock manager would mean committing to compiling, so
+;; the lock manager can't be used in that case. Also, the existence
+;; of the ".zo" file cannot be part of the answer if the ".dep" file
+;; provides a SHA-1 to assume, since that's related to recompilation,
+;; except in the special case when `(trust-existing-zos)` is #t.
+;;
+;; If `trying-sha1?` is #f, then actually build if the compiled form
+;; is out of date, and return #f to report that no further build is
+;; needed. Since there may be concurrent building processes, even if
+;; this process isn't the one to build a file, don't return until any
+;; concurrent builder is defintely done building; in other words,
+;; never return a #f unless the lock manager is consulted (or unless
+;; `trying-sha1?` is #t).
+;;
+;; Beware that if a ".dep" file provides a SHA-1 for the generated
+;; bytecode (because the bytecode was once recompiled from
+;; machine-independent bytecode) but the bytecode file isn't present,
+;; then dependent files will assume that compiling will produce te
+;; same SHA-1. That limitation is necessary to avoid recompilation
+;; when one concurrent processes is recompiling and other processes
+;; are checking whether they can use or merely recompile existing
+;; dependent files, where that checking is not allowed to test for the
+;; bytecode file's existence.
+;;
 (define (maybe-compile-zo deps path->mode roots path orig-path read-src-syntax up-to-date collection-cache seen
                           #:trying-sha1? [trying-sha1? #f])
   (let ([actual-path (actual-source-path orig-path)])
@@ -541,15 +632,18 @@
       (trace-printf "maybe-compile-zo starting ~a" actual-path))
     (begin0
      (parameterize ([indent (+ 2 (indent))])
-       (let* ([zo-name (path-add-extension (get-compilation-path path->mode roots path) #".zo")]
+       (let* ([zo-name (path-add-extension (get-compilation-path path->mode roots path) #".zo")])
-              [zo-exists? (file-exists? zo-name)])
          (cond
-           [(and zo-exists? (trust-existing-zos))
+           [(and (trust-existing-zos)
+                 (file-exists? zo-name))
             (trace-printf "trusting: ~a" zo-name)
             (touch zo-name)
             #f]
            [else
-            (define (build #:recompile-from [recompile-from #f]
+            ;; Called when `tryng-sha1?` is #f and this process (or some process)
+            ;; needs to compile, recompile, or touch:
+            (define (build #:just-touch? [just-touch? #f]
+                           #:recompile-from [recompile-from #f]
                            #:assume-compiled-sha1 [assume-compiled-sha1 #f]
                            #:use-existing-deps [use-existing-deps #f])
               (define lc (parallel-lock-client))
@@ -560,6 +654,12 @@
                (lambda () (void))
                (lambda ()
                  (when ok-to-compile?
+                   (cond
+                     [(and just-touch? (file-exists? zo-name))
+                      (log-compile-event path 'start-touch)
+                      (touch zo-name)]
+                     [else
+                      (when just-touch? (set! just-touch? #f))
                       (log-compile-event path (if recompile-from 'start-recompile 'start-compile))
                       (trace-printf "~acompiling ~a" (if recompile-from "re" "") actual-path)
                       (parameterize ([depth (+ (depth) 1)])
@@ -569,58 +669,90 @@
                                                                 (exn:get-module-code-path ex)
                                                                 (exn-message ex))
                                            (raise ex))])
-                       (compile-zo* path->mode roots path src-sha1 read-src-syntax zo-name up-to-date collection-cache
+                          (define recompile-from-exists? (and recompile-from
-                                    #:recompile-from recompile-from
+                                                              ;; Checking existence now after taking lock:
-                                    #:assume-compiled-sha1 (force assume-compiled-sha1)
+                                                              (file-exists? recompile-from)))
-                                    #:use-existing-deps use-existing-deps)))
+                          (compile-zo*/cross-compile path->mode roots path src-sha1 read-src-syntax zo-name up-to-date collection-cache
-                   (trace-printf "~acompiled ~a" (if recompile-from "re" "") actual-path)))
+                                                     #:recompile-from (and recompile-from-exists?
+                                                                           recompile-from)
+                                                     #:assume-compiled-sha1 (and recompile-from-exists?
+                                                                                 (force assume-compiled-sha1))
+                                                     #:use-existing-deps (and recompile-from-exists?
+                                                                              use-existing-deps))))
+                      (trace-printf "~acompiled ~a" (if recompile-from "re" "") actual-path)])))
                (lambda ()
                  (log-compile-event path (if (or (not lc) locked?)
-                                             (if recompile-from 'finish-recompile 'finish-compile)
+                                             (cond
+                                               [just-touch? 'finish-touch]
+                                               [recompile-from 'finish-recompile]
+                                               [else 'finish-compile])
                                              'already-done))
                  (when locked?
                    (lc 'unlock zo-name))))
               #f)
+            ;; Called to recompile bytecode that is currently in
+            ;; machine-independent form:
             (define (build/recompile)
               (build #:recompile-from zo-name
                      #:assume-compiled-sha1 (or (deps-assume-compiled-sha1 deps)
                                                 ;; delay until lock is held:
                                                 (delay (call-with-input-file* zo-name sha1)))
                      #:use-existing-deps deps))
-            (define src-sha1 (and zo-exists?
+            ;; Called to "build" the file by just updating its timestamp
-                                  deps
+            ;; -- unless it doesn't exist, in which case really build:
+            (define (build/touch)
+              (build #:just-touch? #t))
+            ;; Called when there's no need for this process to build, but make sure the
+            ;; bytecode is there, in case a concurrent process is building it:
+            (define (build/sync)
+              (define lc (parallel-lock-client))
+              (when lc
+                (when (lc 'lock zo-name)
+                  (lc 'unlock zo-name)))
+              #f)
+            ;; ----------------------------------------
+            ;; Determine whether and how to rebuild the file:
+            (define src-sha1 (and deps
                                   (deps-src-sha1 deps)
                                   (get-source-sha1 path)))
             (cond
-              [(and zo-exists?
+              [(and (not src-sha1)
-                    (not src-sha1)
                     (not (file-exists? actual-path)))
-               ;; If we have bytecode but not source, then maybe we need to recompile.
+               ;; If we have bytecode but not source, then we can't compile,
+               ;; but maybe we need to recompile
                (cond
-                 [(not (equal? (deps-machine deps) (current-compile-target-machine)))
+                 [(or (not (eq? (deps-machine deps) (current-compile-target-machine)))
+                      (and (not (deps-machine deps))
+                           (cross-multi-compile? roots)))
                   ;; We'd like to recompile, but that should end up with the same reported hash,
-                  ;; so we don't need to rebuild if just looking kfor the hash.
+                  ;; so we don't need to rebuild if just looking for the hash.
                   (cond
                     [trying-sha1? #f]
                     [else (build/recompile)])]
                  [else
                   ;; No need to build
-                  #f])]
+                  (cond
-              [(and zo-exists?
+                    [trying-sha1? #f]
-                    src-sha1
+                    [else (build/sync)])])]
+              [(and src-sha1
                     (equal? (version) (deps-version deps))
                     (equal? src-sha1 (and (pair? (deps-sha1s deps))
                                           (deps-src-sha1 deps)))
                     (equal? (get-dep-sha1s (deps-imports deps) up-to-date collection-cache read-src-syntax path->mode roots seen
                                            #:must-exist? #f)
                             (deps-imports-sha1 deps))
-                    (or (equal? (deps-machine deps) (current-compile-target-machine))
+                    (or (eq? (deps-machine deps) (current-compile-target-machine))
                         (not (deps-machine deps))))
+               ;; We need to recompile the file from machine-independent bytecode,
+               ;; or maybe just update the file's modification date
                (trace-printf "hash-equivalent: ~a" zo-name)
                (cond
-                 [(equal? (deps-machine deps) (current-compile-target-machine))
+                 [(and (eq? (deps-machine deps) (current-compile-target-machine))
-                  (touch zo-name)
+                       (or (deps-machine deps)
-                  #f]
+                           (not (cross-multi-compile? roots))))
+                  (cond
+                    [trying-sha1? #f]
+                    [else (build/touch)])]
                  [else
                   ;; (deps-machine deps) is #f, so we can recompile machine-independent
                   ;; bytecode to this machine's format
@@ -651,11 +783,17 @@
 
 ;; Gets a multi-sha1 string that represents the compiled code
 ;; as well as its dependencies:
-(define (try-file-sha1 path dep-path)
+(define (try-file-sha1 path dep-path roots)
   (with-module-reading-parameterization
    (lambda ()
+     ;; First, try SHA-1 of file; we need to try this first to be
+     ;; consistent with the order that `compile-zo*` writes and
+     ;; deletes files:
+     (define path-sha1
+       (with-handlers ([exn:fail:filesystem? (lambda (exn) #f)])
+         (call-with-input-file* path sha1)))
      ;; Extract sha1s from ".dep", if possible, including a sha1
-     ;; that we should assume for the cmopiled form:
+     ;; that we should assume for the compiled form:
      (define-values (imports-sha1 assume-compiled-sha1)
        (with-handlers ([exn:fail:filesystem? (lambda (exn)
                                                (values "" #f))])
@@ -664,20 +802,21 @@
           (lambda (p)
             (define deps (read p))
             (define ok-machine? (and (equal? (version) (deps-version deps))
-                                     (or (equal? (current-compile-target-machine) (deps-machine deps))
+                                     (or (eq? (current-compile-target-machine) (deps-machine deps))
-                                         (not (deps-machine deps)))))
+                                         (not (deps-machine deps))
+                                         (and (cross-multi-compile? roots)
+                                              (eq? (system-type 'target-machine) (deps-machine deps))))))
             (values (or (and ok-machine?
                              (deps-imports-sha1 deps))
                         "")
                     (and ok-machine?
                          (deps-assume-compiled-sha1 deps)))))))
      ;; Combine the sha1 for the compiled form with the sha1 of imports;
-     ;; if we have to read the compiled form and that fails (e.g., because
+     ;; if we have to read the compiled form and that failed (e.g., because
      ;; the file's not there), then return #f overall:
-     (with-handlers ([exn:fail:filesystem? (lambda (exn) #f)])
+     (let ([sha-1 (or assume-compiled-sha1 path-sha1)])
-       (string-append
+       (and sha-1
-        (or assume-compiled-sha1 (call-with-input-file* path sha1))
+            (string-append sha-1 imports-sha1))))))
-        imports-sha1)))))
 
 ;; Gets a multi-sha1 string that represents the compiled code
 ;; (plus dependencies), checking for a native library before
@@ -689,9 +828,11 @@
     (or (try-file-sha1 (build-path dir "native" (system-library-subpath)
                                    (path-add-extension name (system-type
                                                              'so-suffix)))
-                       dep-path)
+                       dep-path
+                       roots)
         (try-file-sha1 (build-path dir (path-add-extension name #".zo"))
-                       dep-path)
+                       dep-path
+                       roots)
         "")))
 
 (define (different-source-sha1-and-dep-recorded path deps)
@@ -707,11 +848,16 @@
       (path-replace-extension p #".ss")
       p))
 
+;; The `compile-root` function is a wrapper on `maybe-compile-zo` that
+;; tries to take shortcuts based on file timestamps and the supplied
+;; `update-to-date` cache. If the answer is not in timestamps or the
+;; cache, it has to defer to `maybe-compile-zo` to decide whether a
+;; file has to be built.
 (define (compile-root path->mode roots path0 up-to-date collection-cache read-src-syntax seen
                       #:sha1-only? [sha1-only? #f])
   (define orig-path (simple-form-path path0))
   (define (read-deps path)
-    (with-handlers ([exn:fail:filesystem? (lambda (ex) (list (version) (current-compile-target-machine) '(#f . #f)))])
+    (with-handlers ([exn:fail:filesystem? (lambda (ex) (list #f "none" '(#f . #f)))])
       (with-module-reading-parameterization
        (lambda ()
          (call-with-input-file*
@@ -757,8 +903,13 @@
                (trace-printf "newer version...")
                #t]
               [(not (and (deps-has-machine? deps)
-                         (or (equal? (current-compile-target-machine) (deps-machine deps))
+                         (or (eq? (current-compile-target-machine) (deps-machine deps))
-                             (and sha1-only? (not (deps-machine deps))))))
+                             (and sha1-only? (not (deps-machine deps)))
+                             (and (eq? (system-type 'target-machine) (deps-machine deps))
+                                  (cross-multi-compile? roots)))
+                         (or sha1-only?
+                             (deps-machine deps)
+                             (not (cross-multi-compile? roots)))))
                (trace-printf "different machine...")
                #t]
               [(> path-time (or path-zo-time -inf.0))
@@ -795,13 +946,15 @@
               (maybe-compile-zo deps path->mode roots path orig-path read-src-syntax up-to-date collection-cache new-seen))
             (let ([stamp (cons (or (get-compiled-time path->mode roots path) +inf.0)
                                (delay (get-compiled-sha1 path->mode roots path)))])
-              (when (or needs-build?
-                        ;; If `(deps-machine deps)` is #f and doesn't match the current machine,
-                        ;; then we still need to build.
-                        (equal? (current-compile-target-machine) (deps-machine deps)))
               (define (make-key p)
                 (if (or needs-build?
-                          (equal? (current-compile-target-machine) (deps-machine deps)))
+                        ;; If `(deps-machine deps)` is #f and doesn't match the current machine,
+                        ;; then we still need to build.
+                        (and (or (eq? (current-compile-target-machine) (deps-machine deps))
+                                 (and (eq? (system-type 'target-machine) (deps-machine deps))
+                                      (cross-multi-compile? roots)))
+                             (or (deps-machine deps)
+                                 (not (cross-multi-compile? roots)))))
                     p
                     ;; We didn't actually recompile, yet, so don't record the path
                     ;; as done. But record an "assume" sha1-stamp, so we don't keep
@@ -809,7 +962,7 @@
                     (cons 'assume p)))
               (hash-set! up-to-date (make-key main-path) stamp)
               (unless (eq? main-path alt-path)
-                  (hash-set! up-to-date (make-key alt-path) stamp)))
+                (hash-set! up-to-date (make-key alt-path) stamp))
               stamp)]))])))
   (or (hash-ref up-to-date orig-path #f)
       (and sha1-only?
@@ -941,7 +1094,9 @@
 
 ;; Exported:
 (define (get-compiled-file-sha1 path)
-  (try-file-sha1 path (path-replace-extension path #".dep")))
+  (try-file-sha1 path
+                 (path-replace-extension path #".dep")
+                 (current-compiled-file-roots)))
 
 (define (get-file-sha1 path)
   (get-source-sha1 path))

racket/collects/setup/cross-system.rkt

@@ -30,12 +30,18 @@
                                                            'library-subpath
                                                            'library-subpath-convention
                                                            system-type-symbols))])
-                                    (hash-ref ht sym #f))
+                                    (not (void? (hash-ref ht sym (void)))))
                                   (not
                                    (and (for/and ([sym (in-list system-type-symbols)]
                                                   #:unless (or (eq? sym 'machine)
                                                                (eq? sym 'gc)))
-                                          (equal? (hash-ref ht sym #f) (system-type sym)))
+                                          (define v (hash-ref ht sym))
+                                          (or (equal? v (system-type sym))
+                                              ;; If 'target-machine is set to #f, that's
+                                              ;; for SERVER_COMPILE_MACHINE=any mode
+                                              (and (not v)
+                                                   (eq? sym 'target-machine)
+                                                   (eq? (system-type 'cross) 'infer))))
                                         (equal? (bytes->path (hash-ref ht 'library-subpath)
                                                              (hash-ref ht 'library-subpath-convention))
                                                 (system-library-subpath #f))))
@@ -57,8 +63,10 @@
         "(or/c 'os 'word 'gc 'vm 'link 'machine 'target-machine 'so-suffix 'so-mode 'fs-change)"
         mode))
      (compute-cross!)
-     (or (hash-ref cross-system-table mode #f)
+     (define v (hash-ref cross-system-table mode (void)))
-         (system-type mode))]))
+     (if (eq? v (void))
+         (system-type mode)
+         v)]))
 
 (define (cross-system-library-subpath [mode (begin
                                               (compute-cross!)

racket/collects/setup/dirs.rkt

@@ -188,8 +188,10 @@
 
 (define host-lib-search-dirs
   (delay/sync
-   (or (to-path (hash-ref (force host-config) 'lib-search-dirs #f))
+   (combine-search
-       (list (build-path
+    (to-path (hash-ref (force host-config) 'lib-search-dirs #f))
+    (list (find-user-lib-dir)
+          (build-path
            (exe-relative-path->complete-path (find-system-path 'host-collects-dir))
            'up
            "lib")))))

racket/collects/setup/private/dirs.rkt

@@ -15,7 +15,8 @@
 ;; config: definitions
 
 (provide get-config-table
-         to-path)
+         to-path
+         combine-search)
 
 (define (get-config-table find-config-dir)
   (delay/sync

racket/src/ac/crossany.m4

@@ -0,0 +1,6 @@
+show_explicitly_enabled "${enable_crossany}" "Own cross-compile target as machine-independent"
+
+CROSS_COMPILE_TARGET_KIND=machine
+if test "${enable_crossany}" = "yes" ; then
+  CROSS_COMPILE_TARGET_KIND=any
+fi

racket/src/ac/crossany_arg.m4

@@ -0,0 +1 @@
+AC_ARG_ENABLE(crossany,   [  --enable-crossany       Record own cross target as machine-independent])

racket/src/cfg-racket

@@ -624,6 +624,7 @@ enable_option_checking=no
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
 subdirs
+CROSS_COMPILE_TARGET_KIND
 STARTUP_AS_AUTO
 STARTUP_AS_C
 STARTUP_AS_BYTECODE
@@ -837,6 +838,7 @@ enable_gcov
 enable_noopt
 enable_ubsan
 enable_jitframe
+enable_crossany
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1496,6 +1498,7 @@ Optional Features:
   --enable-noopt          drop -O C flags (useful for low-level debugging)
   --enable-ubsan          compile with -fsanitize=undefined)
   --enable-jitframe       x86_64: use frame pointer for internal calls
+  --enable-crossany       Record own cross target as machine-independent
 
 Some influential environment variables:
   CC          C compiler command
@@ -2872,6 +2875,13 @@ if test "${enable_jitframe+set}" = set; then :
 fi
 
 
+# Check whether --enable-crossany was given.
+if test "${enable_crossany+set}" = set; then :
+  enableval=$enable_crossany;
+fi
+
+
+
 ###### Some flags imply other flags #######
 
 if test "${enable_smalloskit}" = "yes" ; then
@@ -3294,6 +3304,14 @@ fi
 
 
 
+show_explicitly_enabled "${enable_crossany}" "Own cross-compile target as machine-independent"
+
+CROSS_COMPILE_TARGET_KIND=machine
+if test "${enable_crossany}" = "yes" ; then
+  CROSS_COMPILE_TARGET_KIND=any
+fi
+
+
 ###### Some defaults #######
 
 OPTIONS=
@@ -6940,6 +6958,8 @@ LIBS="$LIBS $EXTRALIBS"
 
 
 
+
+
 
 
 

racket/src/configure

@@ -30,7 +30,7 @@ for arg in $*; do
     --enable-scheme=*)
       supplied_scheme=yes
       ;;
-    --help)
+    --help | -h)
       echo $0:
       echo see --help-racket or --help-cs, since the traditional Racket "(3m/CGC)" build
       echo and the Chez Scheme "(CS)" build support different options. If you use

racket/src/cs/c/Makefile.in

@@ -237,7 +237,7 @@ common-install:
 	$(STRIP_DEBUG) "$(DESTDIR)$(libpltdir)/starter"
 	$(ICP) $(srcdir)/../../start/starter-sh "$(DESTDIR)$(libpltdir)/starter-sh"
 	$(RACKET) -cu "$(srcdir)/../../racket/collects-path.rkt" "$(DESTDIR)$(libpltdir)/starter" $(DESTDIR)@COLLECTS_PATH@ $(DESTDIR)@CONFIG_PATH@
-	$(RACKET) -cu "$(srcdir)/gen-system.rkt" $(DESTDIR)$(libpltdir)/system$(CS_INSTALLED).rktd $(MACH)
+	$(RACKET) -cu "$(srcdir)/gen-system.rkt" $(DESTDIR)$(libpltdir)/system$(CS_INSTALLED).rktd $(MACH) @CROSS_COMPILE_TARGET_KIND@
 
 unix-install:
 	$(MAKE) common-install

racket/src/cs/c/configure

@@ -622,6 +622,7 @@ ac_includes_default="\
 
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
+CROSS_COMPILE_TARGET_KIND
 CS_COMPILED_SUBDIR
 CS_INSTALLED
 FRAMEWORK_PREFIX
@@ -761,6 +762,7 @@ enable_noopt
 enable_csdefault
 enable_csonly
 enable_parent
+enable_crossany
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1400,6 +1402,7 @@ Optional Features:
   --enable-csdefault      use CS as default build
   --enable-csonly         build CS only
   --enable-parent         Create "../Makefile" (internal use)
+  --enable-crossany       Record own cross target as machine-independent
 
 Some influential environment variables:
   CC          C compiler command
@@ -2415,6 +2418,12 @@ if test "${enable_parent+set}" = set; then :
   enableval=$enable_parent;
 fi
 
+# Check whether --enable-crossany was given.
+if test "${enable_crossany+set}" = set; then :
+  enableval=$enable_crossany;
+fi
+
+
 
 
 enable_quartz=no
@@ -2558,6 +2567,14 @@ fi
 
 
 
+show_explicitly_enabled "${enable_crossany}" "Own cross-compile target as machine-independent"
+
+CROSS_COMPILE_TARGET_KIND=machine
+if test "${enable_crossany}" = "yes" ; then
+  CROSS_COMPILE_TARGET_KIND=any
+fi
+
+
 SUB_CONFIGURE_EXTRAS=
 
 
@@ -4632,6 +4649,7 @@ CPPFLAGS="$CPPFLAGS $PREFLAGS"
 
 
 
+
 
 
 makefiles="Makefile"

racket/src/cs/c/configure.ac

@@ -26,6 +26,7 @@ m4_include(../ac/strip_arg.m4)
 AC_ARG_ENABLE(csdefault,  [  --enable-csdefault      use CS as default build])
 AC_ARG_ENABLE(csonly,     [  --enable-csonly         build CS only])
 AC_ARG_ENABLE(parent,     [  --enable-parent         Create "../Makefile" (internal use)])
+m4_include(../ac/crossany_arg.m4)
 
 m4_include(../ac/sdk.m4)
 
@@ -79,6 +80,8 @@ fi
 
 m4_include(../ac/path_pkgscope.m4)
 
+m4_include(../ac/crossany.m4)
+
 SUB_CONFIGURE_EXTRAS=
 
 
@@ -449,6 +452,7 @@ AC_SUBST(FRAMEWORK_INSTALL_DIR)
 AC_SUBST(FRAMEWORK_PREFIX)
 AC_SUBST(CS_INSTALLED)
 AC_SUBST(CS_COMPILED_SUBDIR)
+AC_SUBST(CROSS_COMPILE_TARGET_KIND)
 
 makefiles="Makefile"
 

racket/src/cs/c/gen-system.rkt

@@ -1,6 +1,6 @@
 (module gen-system '#%kernel
   
-  ;; Command-line argument: <dest-file> <target-machine>
+  ;; Command-line argument: <dest-file> <target-machine> <cross-target-machine>
   
   (define-values (ht)
     (hash 'os (system-type 'os)
@@ -12,7 +12,9 @@
           'so-suffix (system-type 'so-suffix)
           'so-mode 'local
           'fs-change '#(#f #f #f #f)
-          'target-machine (string->symbol (vector-ref (current-command-line-arguments) 1))))
+          'target-machine (if (equal? "any" (vector-ref (current-command-line-arguments) 2))
+                              #f
+                              (string->symbol (vector-ref (current-command-line-arguments) 1)))))
 
   (call-with-output-file
    (vector-ref (current-command-line-arguments) 0)

racket/src/racket/Makefile.in

@@ -171,7 +171,7 @@ no-cgc-needed:
 	cd dynsrc; $(MAKE) ../starter@EXE_SUFFIX@
 
 ALL_CPPFLAGS = -I$(builddir) -I$(srcdir)/include -I$(srcdir)/src $(CPPFLAGS) @OPTIONS@ @GC2OPTIONS@ @MZOPTIONS@ 
-MKSYSTEM_ARGS = -cqu $(srcdir)/mksystem.rkt system.rktd "$(CPP) $(ALL_CPPFLAGS) $(srcdir)/src/systype.c" "@MMM_INSTALLED@"
+MKSYSTEM_ARGS = -cqu $(srcdir)/mksystem.rkt system.rktd "$(CPP) $(ALL_CPPFLAGS) $(srcdir)/src/systype.c" "@MMM_INSTALLED@" @CROSS_COMPILE_TARGET_KIND@
 
 sysinfer@CGC@:
 	@RUN_RACKET_CGC@ $(MKSYSTEM_ARGS) "@RUN_RACKET_CGC@" "$(RUN_THIS_RACKET_CGC)"

racket/src/racket/configure.ac

@@ -80,6 +80,8 @@ AC_ARG_ENABLE(noopt,   [  --enable-noopt          drop -O C flags (useful for lo
 AC_ARG_ENABLE(ubsan,   [  --enable-ubsan          compile with -fsanitize=undefined)])
 AC_ARG_ENABLE(jitframe,[  --enable-jitframe       x86_64: use frame pointer for internal calls])
 
+m4_include(../ac/crossany_arg.m4)
+
 ###### Some flags imply other flags #######
 
 if test "${enable_smalloskit}" = "yes" ; then
@@ -240,6 +242,8 @@ fi
 
 m4_include(../ac/path_pkgscope.m4)
 
+m4_include(../ac/crossany.m4)
+
 ###### Some defaults #######
 
 OPTIONS=
@@ -1506,6 +1510,8 @@ AC_SUBST(STARTUP_AS_BYTECODE)
 AC_SUBST(STARTUP_AS_C)
 AC_SUBST(STARTUP_AS_AUTO)
 
+AC_SUBST(CROSS_COMPILE_TARGET_KIND)
+
 mk_needed_dir()
 {
   if test ! -d "$1" ; then

racket/src/racket/mksystem.rkt

@@ -1,8 +1,13 @@
 (module mksystem '#%kernel
   ;; Arguments are
-  ;;   <output-file> [<cpp-command> <3m-exe-suffix> <run-racket-command> <this-racket-command>]
+  ;;   <output-file> [<cpp-command> <3m-exe-suffix> <cross-target-kind> <run-racket-command> <this-racket-command>]
   (define-values (args) (current-command-line-arguments))
 
+  (define-values (mi-target?)
+    (if (> (vector-length args) 3)
+        (equal? "any" (vector-ref args 3))
+        #f))
+
   (define-values (ht)
     (if (if (= (vector-length args) 1)
             #t
@@ -22,7 +27,7 @@
               'so-suffix (system-type 'so-suffix)
               'so-mode (system-type 'so-mode)
               'fs-change (system-type 'fs-change)
-              'target-machine 'racket
+              'target-machine (if mi-target? #f 'racket)
               'library-subpath (path->bytes (system-library-subpath #f))
               'library-subpath-convention (system-path-convention-type))
         ;; Cross-compiling; use `cpp` to get details
@@ -65,7 +70,7 @@
                             'so-suffix (string->bytes/utf-8 (get-string "system_type_so_suffix"))
                             'so-mode (get-symbol "system_type_so_mode")
                             'fs-change '#(#f #f #f #f)
-                            'target-machine 'racket
+                            'target-machine (if mi-target? #f 'racket)
                             'library-subpath (string->bytes/utf-8 library-subpath)
                             'library-subpath-convention (if (eq? os 'windows)
                                                             'windows

racket/src/worksp/csbuild.rkt

@@ -260,4 +260,5 @@
 (system*! (find-exe)
           "../cs/c/gen-system.rkt"
           (format "../../lib/system~a.rktd" cs-suffix)
-          machine)
+          machine
+          "machine")