raco pkg install: make --force override scope-conflict errors

It's possible that an installation will have a package X already and
a user wants to install a different X. To make it all work out,
the user may have to also install a new Y for every installation-scoped
Y that refers to X --- maybe not easy, but at least possible as
a last resort.

pkgs/racket-pkgs/racket-doc/pkg/scribblings/pkg.scrbl

@@ -253,8 +253,16 @@ specific to both the current user and the installation's name/version
 all users of the Racket installation.
 Finally, a directory path can be used as a @tech{package scope}, in which case
 package operations affect the set of packages installations in the
-directory (and an installation can be configured to include the
+directory; an installation can be configured to include the
-directory in its search path for installed packages).
+directory in its search path for installed packages (see
+@secref["config-file" #:doc '(lib "scribblings/raco/raco.scrbl")]).
+Conflict checking disallows installation of the same or conflicting
+package in different scopes, but if such a configuration is forced,
+collections are found first in packages with @exec{user} @tech{package
+scope}; search then proceeds in a configured order, where
+@exec{installation} @tech{package scope} typically precedes other
+directory @tech{package scopes}.
+
 
 @; ----------------------------------------
 
@@ -285,7 +293,8 @@ sub-commands.
   @itemlist[
    @item{@exec{fail} --- Cancels the installation if dependencies are uninstalled or version requirements are unmet. 
         This behavior is the default for a @nonterm{pkg-source} that is not a @tech{package name}.}
-   @item{@exec{force} --- Installs the package(s) despite missing dependencies or version requirements (unsafe).}
+   @item{@exec{force} --- Installs the package(s) despite missing dependencies or version requirements.
+         Forcing an installation may leave package content in an inconsistent state.}
    @item{@exec{search-ask} --- Looks for dependencies (when uninstalled) or updates (when version requirements are unmet)
          via the configured @tech{package catalogs},
          but asks if you would like the packages installed or updated. This behavior is the default for a
@@ -342,7 +351,9 @@ sub-commands.
         whose name corresponds to an already-installed package, except for promoting auto-installed
         packages to explicitly installed.}
 
-  @item{@DFlag{force} --- Ignores conflicts (unsafe).}
+  @item{@DFlag{force} --- Ignores module conflicts, including conflicts due to installing a single
+        package in multiple scopes. Forcing an installation may leave package content in an
+        inconsistent state.}
 
   @item{@DFlag{ignore-checksums} --- Ignores errors verifying package @tech{checksums} (unsafe).}
 
@@ -799,13 +810,12 @@ multi-collection packages) with @exec{raco link}.
 related for conflict checking?}
 
 User-specific packages are checked against installation-wide packages
-for conflicts. Installation-wide packages are checked only against
+for package-name conflicts and provided-module
-other installation-wide packages.
+conflicts. Installation-wide packages are checked against
+user-specific packages only for provided-module conflicts.
 
-Beware that a new installation-wide package can invalidate previous
+Beware that a conflict-free, installation-wide change by one user can
-conflict checks for user-specific packages. Similarly, new
+create conflicts for a different user.
-user-specific but all-version packages can invalidate previous
-user-specific conflict checks for a different Racket version.
 
 @subsection{Do I need to change a package's version when I update a package with error fixes, @|etc|?}
 

pkgs/racket-pkgs/racket-doc/scribblings/raco/config.scrbl

@@ -52,8 +52,7 @@ directory:
 
  @item{@indexed-racket['links-file] --- a path, string, or byte string for the
        @tech[#:doc reference-doc]{collection links file}; it defaults
-       to a @filepath{links.rktd} file in the @filepath{share} sibling
+       to a @filepath{links.rktd} file in the main shared-file directory.}
-       of the main collection directory.}
 
  @item{@indexed-racket['links-search-files] --- like @racket['lib-search-dirs],
        but for @tech[#:doc reference-doc]{collection links file}.}

pkgs/racket-pkgs/racket-test/tests/pkg/test.rkt

@@ -52,6 +52,7 @@
  ;; "main-server"
  "update-deps"
  "update-auto"
+ "scope"
  "migrate" 
  "versions"
  "platform"

pkgs/racket-pkgs/racket-test/tests/pkg/tests-raco.rkt

@@ -18,12 +18,12 @@
    $ "raco pkg create --format plt test-pkgs/raco-pkg"
    $ "raco raco-pkg" =exit> 1
    $ "raco pkg install --no-setup test-pkgs/raco-pkg.plt"
-   $ "raco raco-pkg" =exit> 1
+   $ "raco raco-pkg" =exit> 1))
-   (putenv "PLT_PKG_NOSETUP" "")))
 
  (with-fake-root
   (shelly-case
    "raco install/update uses raco setup"
+   (putenv "PLT_PKG_NOSETUP" "")
    $ "raco pkg create --format plt test-pkgs/raco-pkg"
    $ "raco raco-pkg" =exit> 1
    $ "raco pkg install test-pkgs/raco-pkg.plt"
@@ -32,6 +32,7 @@
  (with-fake-root
   (shelly-case
    "raco install uses raco setup with single collect"
+   (putenv "PLT_PKG_NOSETUP" "")
    $ "raco pkg install --copy test-pkgs/pkg-test3-v3" =exit> 0))
 
  (shelly-begin
@@ -39,6 +40,7 @@
   
   (shelly-case
    "update of package runs setup on package with dependency"
+   (putenv "PLT_PKG_NOSETUP" "")
    (shelly-wind
     $ "mkdir -p test-pkgs/update-test"
     $ "cp -f test-pkgs/pkg-test1.zip test-pkgs/update-test/pkg-test1.zip"

pkgs/racket-pkgs/racket-test/tests/pkg/tests-scope.rkt

@@ -0,0 +1,67 @@
+#lang racket/base
+(require "shelly.rkt"
+         "util.rkt")
+
+(pkg-tests
+ (with-fake-installation
+  (with-fake-root
+
+   (shelly-case
+    "default scope in different scopes"
+    $ "raco pkg show"
+    $ "raco pkg config --set -i default-scope installation"
+    $ "raco pkg config -u default-scope" =stdout> "installation\n" ; inherited
+    $ "raco pkg config --set default-scope user" ; set user at installation scope
+    $ "raco pkg config -u default-scope" =stdout> "user\n" ; inherited
+    $ "raco pkg config -i default-scope" =stdout> "user\n"
+    
+    $ "raco pkg config --set default-scope installation" ; set installation at user scope
+    $ "raco pkg config -u default-scope" =stdout> "installation\n"
+    $ "raco pkg config -i default-scope" =stdout> "user\n"
+    $ "raco pkg config default-scope" =stdout> "user\n" ; misleading!
+
+    $ "raco pkg config --set default-scope user" ; still sets user in installation
+    $ "raco pkg config -i default-scope" =stdout> "user\n"
+    $ "raco pkg config -u default-scope" =stdout> "installation\n"
+
+    $ "raco pkg config -u --set default-scope user"
+    $ "raco pkg config default-scope" =stdout> "user\n")
+
+   (shelly-case
+    "conflict due to installations in different scopes: installation-wide first"
+    $ "raco pkg install -i test-pkgs/pkg-test1"
+    $ "racket -l pkg-test1"  =stdout> #rx"main loaded"
+    $ "raco pkg install -u test-pkgs/pkg-test1" =exit> 1 =stderr> #rx"installed in a wider scope"
+    $ "raco pkg install -u --name base test-pkgs/pkg-test1" =exit> 1 =stderr> #rx"installed in a wider scope"
+    $ "raco pkg remove pkg-test1" =stdout> #rx"Inferred package scope: installation")
+
+   (shelly-case
+    "conflict due to installations in different scopes: user-specific first"
+    $ "raco pkg install -u test-pkgs/pkg-test1"
+    $ "racket -l pkg-test1"  =stdout> #rx"main loaded"
+    $ "raco pkg install -i test-pkgs/pkg-test1" =exit> 1 =stderr> #rx"packages in different scopes conflict"
+    $ "raco pkg remove pkg-test1" =stdout> "Removing pkg-test1\n")
+
+   (shelly-case
+    "override conflist, installation first"
+    $ "raco pkg install -i test-pkgs/pkg-test1"
+    $ "racket -l racket/base -l pkg-test1/number -e '(number)'"  =stdout> "1\n"
+    $ "raco pkg install -u --name pkg-test1 test-pkgs/pkg-test1-v2" =exit> 1
+    $ "raco pkg install --force -u --name pkg-test1 test-pkgs/pkg-test1-v2"
+    $ "racket -l racket/base -l pkg-test1/number -e '(number)'"  =stdout> "2\n"
+    $ "raco pkg remove pkg-test1" =stdout> "Removing pkg-test1\n"
+    $ "racket -l racket/base -l pkg-test1/number -e '(number)'"  =stdout> "1\n"
+    $ "raco pkg remove pkg-test1" =stdout> #rx"Inferred package scope: installation")
+
+   (shelly-case
+    "override conflist, user first"
+    $ "raco pkg install -u test-pkgs/pkg-test1"
+    $ "racket -l racket/base -l pkg-test1/number -e '(number)'"  =stdout> "1\n"
+    $ "raco pkg install -i --name pkg-test1 test-pkgs/pkg-test1-v2" =exit> 1
+    $ "raco pkg install --force -i --name pkg-test1 test-pkgs/pkg-test1-v2"
+    $ "racket -l racket/base -l pkg-test1/number -e '(number)'"  =stdout> "1\n"
+    $ "raco pkg remove pkg-test1" =stdout> "Removing pkg-test1\n"
+    $ "racket -l racket/base -l pkg-test1/number -e '(number)'"  =stdout> "2\n"
+    $ "raco pkg remove pkg-test1" =stdout> #rx"Inferred package scope: installation")
+
+    )))

pkgs/racket-pkgs/racket-test/tests/pkg/tests-update.rkt

@@ -138,7 +138,7 @@
                      $ "cp -f test-pkgs/pkg-test1-v2.zip.CHECKSUM test-pkgs/update-test/pkg-test1.zip.CHECKSUM"
                      $ "cp -f test-pkgs/pkg-test3-v2.zip test-pkgs/update-test/pkg-test3.zip"
                      $ "cp -f test-pkgs/pkg-test3-v2.zip.CHECKSUM test-pkgs/update-test/pkg-test3.zip.CHECKSUM"
-                     $ "raco pkg update --update-deps pkg-test3" =exit> 0
+                     $ "raco pkg update --update-deps --deps search-auto pkg-test3" =exit> 0
                      $ "racket -e '(require pkg-test1/update)'" =exit> 43
                      $ "racket -e '(require pkg-test3)'" =stdout> #rx"version 2 loaded"
                      $ "raco pkg remove pkg-test3")
@@ -189,7 +189,7 @@
                      $ "cp -f test-pkgs/pkg-test1-v2.zip.CHECKSUM test-pkgs/update-test/pkg-test1.zip.CHECKSUM"
                      $ "cp -f test-pkgs/pkg-test3.zip test-pkgs/update-test/pkg-test3.zip"
                      $ "cp -f test-pkgs/pkg-test3.zip.CHECKSUM test-pkgs/update-test/pkg-test3.zip.CHECKSUM"
-                     $ "raco pkg update --update-deps pkg-test3" =exit> 0
+                     $ "raco pkg update --update-deps --deps search-auto pkg-test3" =exit> 0
                      $ "racket -e '(require pkg-test1/update)'" =exit> 43
                      $ "racket -e '(require pkg-test3)'" =stdout> #rx"main loaded"
                      $ "raco pkg remove pkg-test3")

pkgs/racket-pkgs/racket-test/tests/pkg/util.rkt

@@ -9,6 +9,7 @@
          racket/runtime-path
          racket/path
          racket/list
+         setup/dirs
          pkg/util
          "shelly.rkt")
 
@@ -20,26 +21,64 @@
   (and (file-exists? p)
        p))
 
+(define (with-fake-installation* t)
+  (define tmp-dir
+    (make-temporary-file ".racket.fake-installation~a" 'directory
+                         (find-system-path 'temp-dir)))
+  (make-directory* tmp-dir)
+  (dynamic-wind
+      void
+      (λ ()
+         (define ->s path->string)
+         (define config
+           (hash
+            ;; redirect main installation via "share" to
+            ;; our temporary directory:
+            'share-dir
+            (->s (build-path tmp-dir))
+
+            ;; Find existing links and packages from the
+            ;; old configuration:
+            'links-search-files 
+            (cons #f
+                  (map ->s (get-links-search-files)))
+            'pkgs-search-dirs
+            (cons #f
+                  (map ->s (get-pkgs-search-dirs)))))
+         (call-with-output-file*
+          (build-path tmp-dir "config.rktd")
+          (lambda (o)
+            (write config o)
+            (newline o)))
+         (define tmp-dir-s
+           (path->string tmp-dir))
+         (parameterize ([current-environment-variables
+                         (environment-variables-copy
+                          (current-environment-variables))])
+           (putenv "PLTCONFIGDIR" tmp-dir-s)
+           (t)))
+      (λ ()
+        (delete-directory/files tmp-dir))))
+(define-syntax-rule (with-fake-installation e ...)
+  (with-fake-installation* (λ ()  e ...)))
+
 (define (with-fake-root* t)
   (define tmp-dir
     (make-temporary-file ".racket.fake-root~a" 'directory
                          (find-system-path 'home-dir)))
   (make-directory* tmp-dir)
-  (define tmp-dir-s
-    (path->string tmp-dir))
-  (define before 
-    (or (getenv "PLTADDONDIR")
-        (path->string (find-system-path 'addon-dir))))
   (dynamic-wind
       void
       (λ ()
-        (putenv "PLTADDONDIR"
+         (define tmp-dir-s
-                tmp-dir-s)
+           (path->string tmp-dir))
-        (t))
+         (parameterize ([current-environment-variables
+                         (environment-variables-copy
+                          (current-environment-variables))])
+           (putenv "PLTADDONDIR" tmp-dir-s)
+           (t)))
       (λ ()
-        (delete-directory/files tmp-dir)
+        (delete-directory/files tmp-dir))))
-        (putenv "PLTADDONDIR"
-                before))))
 (define-syntax-rule (with-fake-root e ...)
   (with-fake-root* (λ ()  e ...)))
 

racket/collects/pkg/lib.rkt

@@ -1227,43 +1227,48 @@
       (for/hash ([i (in-list infos)])
         (values (install-info-name i) (install-info-directory i))))
     (cond
-      [(and (not updating?) (hash-ref all-db pkg-name #f))
+      [(and (not updating?)
-       (define this-pkg-info (hash-ref all-db pkg-name #f))
+            (hash-ref all-db pkg-name #f)
+            ;; Already installed, but can force if the install is for
+            ;; a wider scope:
+            (not (and (not (hash-ref current-scope-db pkg-name #f))
+                      force?)))
+       (define existing-pkg-info (hash-ref all-db pkg-name #f))
        (cond
-        [(and (pkg-info-auto? this-pkg-info)
+        [(and (pkg-info-auto? existing-pkg-info)
               (not (pkg-desc-auto? desc))
               ;; Don't confuse a promotion request with a different-source install:
-              (equal? (pkg-info-orig-pkg this-pkg-info) orig-pkg)
+              (equal? (pkg-info-orig-pkg existing-pkg-info) orig-pkg)
               ;; Also, make sure it's installed in the scope that we're changing:
               (hash-ref current-scope-db pkg-name #f))
          ;; promote an auto-installed package to a normally installed one
          (lambda ()
            (unless quiet?
              (download-printf "Promoting ~a from auto-installed to explicitly installed\n" pkg-name))
-           (update-pkg-db! pkg-name (update-auto this-pkg-info #f)))]
+           (update-pkg-db! pkg-name (update-auto existing-pkg-info #f)))]
         [else
          ;; Fail --- already installed
          (clean!)
-         (if (and (pkg-info-auto? this-pkg-info)
+         (cond
-                  (not (pkg-desc-auto? desc)))
+          [(not (hash-ref current-scope-db pkg-name #f))
-             ;; It failed either due to scope or source:
-             (if (equal? (pkg-info-orig-pkg this-pkg-info) orig-pkg)
            (pkg-error (~a "package is currently installed in a wider scope\n"
                           "  package: ~a\n"
                           "  installed scope: ~a\n"
                           "  given scope: ~a")
                       pkg-name
                       (find-pkg-installation-scope pkg-name #:next? #t)
-                            (current-pkg-scope))
+                      (current-pkg-scope))]
+          [(not (equal? (pkg-info-orig-pkg existing-pkg-info) orig-pkg))
            (pkg-error (~a "package is already installed from a different source\n"
                           "  package: ~a\n"
                           "  installed source: ~a\n"
                           "  given source: ~a")
                       pkg-name
-                            (pkg-info-orig-pkg this-pkg-info)
+                      (pkg-info-orig-pkg existing-pkg-info)
-                            orig-pkg))
+                      orig-pkg)]
+          [else
            (pkg-error "package is already installed\n  package: ~a"
-                        pkg-name))])]
+                      pkg-name)])])]
       [(and
         (not force?)
         (for/or ([mp (in-set module-paths)])
@@ -1300,10 +1305,13 @@
          (clean!)
          (match-define (cons conflicting-pkg mp) conflicting-pkg*mp)
          (if conflicting-pkg
-             (pkg-error (~a "packages conflict\n"
+             (pkg-error (~a "packages ~aconflict\n"
                             "  package: ~a\n"
                             "  package: ~a\n"
                             "  module path: ~s")
+                        (if (equal? conflicting-pkg pkg-name)
+                            "in different scopes "
+                            "")
                         pkg conflicting-pkg (pretty-module-path mp))
              (pkg-error (~a "package conflicts with existing installed\n"
                             "  package: ~a\n"