raco pkg {install,update}: add --checksum argument

The `--checksum` argument's main use is that it lets you pick a specific
commit from a GitHub repository. More generally, it lets you simulate
a package-catalog result, which includes a checksum.

Also, adjust checking of downloaded checksums to ensure that they
match the expected checksum, as predicted by a package catalog or
by the `--checksum` argument.
This commit is contained in:
Matthew Flatt 2013-09-01 07:57:57 -06:00
parent e0385f36c3
commit 6530062120
10 changed files with 150 additions and 75 deletions

View File

@ -102,9 +102,10 @@ scope}.}
@deftogether[(
@defproc[(pkg-desc? [v any/c]) boolean?]
@defproc[(pkg-desc [name string?]
@defproc[(pkg-desc [source string?]
[type (or/c #f 'file 'dir 'link 'static-link
'file-url 'dir-url 'github 'name)]
[name (or/c string? #f)]
[checksum (or/c string? #f)]
[auto? boolean?])
pkg-desc?]

View File

@ -61,7 +61,7 @@ Each @tech{package} has associated @deftech{package metadata}:
@item{a @deftech{package name} --- a string made of the characters @|package-name-chars|.}
@item{a @deftech{checksum} --- a string that identifies different releases of a package. A
package can be updated when its @tech{checksum} changes,
whether or not its @tech{version} changes. The checksum
whether or not its @tech{version} changes. The checksum normally
can be computed as the SHA1 (see @racketmodname[openssl/sha1])
of the package's content.}
@item{a @deftech{version} --- a string of the form @nonterm{maj}@litchar{.}@nonterm{min},
@ -334,6 +334,13 @@ sub-commands.
which makes sense only when a single @nonterm{pkg-source} is provided. The name is normally
inferred for each @nonterm{pkg-source}.}
@item{@DFlag{checksum} @nonterm{checksum} --- specifies a checksum for the package,
which normally makes sense only when a single @nonterm{pkg-source} is provided. The use of
@nonterm{checksum} depends on @nonterm{pkg-source}: for a GitHub source, @nonterm{checksum} selects a checksum;
for a @tech{package name}, file path, or remote URL as a source, @nonterm{checksum} specifies an expected checksum;
for a directory path (including a remote directory URL without a @filepath{.CHECKSUM} file) as a source,
@nonterm{checksum} assigns a checksum.}
@item{@DFlag{deps} @nonterm{behavior} --- Selects the behavior for dependencies, where @nonterm{behavior} is one of
@itemlist[
@item{@exec{fail} --- Cancels the installation if dependencies are uninstalled or version requirements are unmet.
@ -433,6 +440,7 @@ the given @nonterm{pkg-source}s.
@item{@DFlag{type} @nonterm{type} or @Flag{t} @nonterm{type} --- Same as for @command-ref{install}.}
@item{@DFlag{name} @nonterm{pkg} or @Flag{n} @nonterm{pkg} --- Same as for @command-ref{install}.}
@item{@DFlag{checksum} @nonterm{checksum} --- Same as for @command-ref{install}.}
@item{@DFlag{deps} @nonterm{behavior} --- Same as for @command-ref{install}.}
@item{@DFlag{auto} --- Shorthand for @exec{@DFlag{deps} search-auto} plus @DFlag{update-deps}.}
@item{@DFlag{update-deps} --- Same as for @command-ref{install}, but

View File

@ -28,9 +28,9 @@
(hash-ref details "pkg-test1"))
(define-values (cksum mods deps)
(get-pkg-content (pkg-desc "pkg-test1" #f #f #f)))
(get-pkg-content (pkg-desc "pkg-test1" #f #f #f #f)))
(define-values (cksum1 mods1 deps1)
(get-pkg-content (pkg-desc "http://localhost:9999/pkg-test1.zip" #f #f #f)))
(get-pkg-content (pkg-desc "http://localhost:9999/pkg-test1.zip" #f #f #f #f)))
(check-equal? cksum cksum1)
(check-equal? (sort mods string<? #:key cadr)
@ -42,5 +42,5 @@
(check-equal? deps '())
(define-values (cksum2 mods2 deps2)
(get-pkg-content (pkg-desc "pkg-test2" 'name #f #f)))
(get-pkg-content (pkg-desc "pkg-test2" 'name #f #f #f)))
(check-equal? deps2 '("pkg-test1")))

View File

@ -61,6 +61,11 @@
(shelly-case
"local directory name fails because called a URL"
$ "raco pkg install --type file-url test-pkgs/pkg-a-first/" =exit> 1)
(shelly-case
"local file fails due to mismatch with specified checksum"
$ "raco pkg install --checksum zzz test-pkgs/pkg-a-first.plt"
=exit> 1
=stderr> #rx"unexpected checksum")
(shelly-case
"remote/URL/http directory, non-existant file"
@ -74,6 +79,11 @@
"remote/URL/http directory, bad manifest"
;; XXX why does this error now?
$ "raco pkg install http://localhost:9999/pkg-test1-manifest-error/" =exit> 1)
(shelly-case
"remote/URL/file, bad checksum"
$ "raco pkg install --checksum zzz http://localhost:9999/pkg-test1.tgz"
=exit> 1
=stderr> #rx"unexpected checksum")
(shelly-case
"local directory fails when not there"

View File

@ -13,27 +13,38 @@
"shelly.rkt"
"util.rkt")
;; todo: to move the test packages to the "plt" account on GitHub
(pkg-tests
(shelly-begin
(shelly-install "remote/github"
"git://github.com/jeapostrophe/galaxy?path=tests/planet2/test-pkgs/planet2-test1")
(shelly-install
"remote/github" "git://github.com/mflatt/pkg-test?path=pkg-test1"
$ "racket -l racket/base -l pkg-test1/number -e '(number)'" =stdout> "1\n")
(shelly-install "remote/github with slash"
"git://github.com/jeapostrophe/galaxy?path=tests/planet2/test-pkgs/planet2-test1/")
(shelly-install "remote/github with auto prefix"
"--type github jeapostrophe/galaxy?path=tests/planet2/test-pkgs/planet2-test1/")
"git://github.com/mflatt/pkg-test?path=pkg-test1/")
(shelly-install
"remote/github with auto prefix and with branch"
"--type github mflatt/pkg-test?path=pkg-test1/#alt"
$ "racket -l racket/base -l pkg-test1/number -e '(number)'" =stdout> "10\n")
(shelly-install
"remote/github with checksum"
"--checksum f9b4eef22cdd9ab88b254cb027fc1ebe7fb596fd git://github.com/mflatt/pkg-test?path=pkg-test1"
$ "racket -l racket/base -l pkg-test1/number -e '(number)'" =stdout> "100\n"
$ "raco pkg update pkg-test1"
$ "racket -l racket/base -l pkg-test1/number -e '(number)'" =stdout> "1\n")
(hash-set! *index-ht-1* "planet2-test1-github-different-checksum"
(hash-set! *index-ht-1* "pkg-test1-github-different-checksum"
(hasheq 'checksum
"23eeaee731e72a39bddbacdf1ed6cce3bcf423a5"
"f9b4eef22cdd9ab88b254cb027fc1ebe7fb596fd"
'source
"git://github.com/jeapostrophe/galaxy?path=tests/planet2/test-pkgs/planet2-test1/"))
"git://github.com/mflatt/pkg-test?path=pkg-test1"))
(with-fake-root
(shelly-case
"remote/name package"
$ "raco pkg config --set catalogs http://localhost:9990"
$ "racket -e '(require planet2-test1)'" =exit> 1
$ "raco pkg install planet2-test1-github-different-checksum"
$ "racket -e '(require planet2-test1)'"
$ "raco pkg remove planet2-test1-github-different-checksum"
$ "racket -e '(require planet2-test1)'" =exit> 1))))
$ "racket -l pkg-test1/number" =exit> 1
$ "raco pkg install pkg-test1-github-different-checksum"
$ "racket -l racket/base -l pkg-test1/number -e '(number)'" =stdout> "100\n"
$ "raco pkg remove pkg-test1-github-different-checksum"
$ "racket -l pkg-test1/number" =exit> 1))))

View File

@ -49,6 +49,13 @@
(shelly-install "replacement checksum can be checked"
"test-pkgs/pkg-test1.zip"
$ "raco pkg update test-pkgs/pkg-test1.zip" =stdout> "No updates available\n")
(shelly-install "checksum can be supplied for local directory"
"test-pkgs/pkg-test1.zip"
$ "racket -e '(require pkg-test1/update)'" =exit> 42
$ "raco pkg update --name pkg-test1 --checksum abcdef test-pkgs/pkg-test1-v2"
$ "racket -e '(require pkg-test1/update)'" =exit> 43
$ "raco pkg show" =stdout> #rx"abcdef"
$ "raco pkg update --name pkg-test1 --checksum abcdef test-pkgs/pkg-test1-v2" =stdout> "No updates available\n")
(shelly-wind
$ "mkdir -p test-pkgs/update-test"

View File

@ -68,12 +68,13 @@
(apply printf fmt args)
(flush-output))
(struct pkg-desc (source type name auto?))
(struct pkg-desc (source type name checksum auto?))
(define (pkg-desc=? a b)
(define (->list a)
(list (pkg-desc-source a)
(pkg-desc-type a)
(pkg-desc-name a)
(pkg-desc-checksum a)
(pkg-desc-auto? a)))
(equal? (->list a) (->list b)))
@ -423,17 +424,18 @@
#:download-printf download-printf
#:pkg-name pkg-name)]))
(define (checksum-for-pkg-source pkg-source type pkg-name download-printf)
(define (checksum-for-pkg-source pkg-source type pkg-name given-checksum download-printf)
(cond
[(or (eq? type 'file-url) (eq? type 'dir-url) (eq? type 'github))
(remote-package-checksum `(url ,pkg-source) download-printf pkg-name)]
(or (remote-package-checksum `(url ,pkg-source) download-printf pkg-name)
given-checksum)]
[(eq? type 'file)
(define checksum-pth (format "~a.CHECKSUM" pkg-source))
(or (and (file-exists? checksum-pth)
(file->string checksum-pth))
(and (file-exists? pkg-source)
(call-with-input-file* pkg-source sha1)))]
[else #f]))
[else given-checksum]))
(define (write-file-hash! file new-db)
(unless (eq? (pkg-lock-held) 'exclusive)
@ -804,6 +806,19 @@
reason
s))
(define (check-checksum given-checksum checksum what pkg-src)
(when (and given-checksum
checksum
(not (equal? given-checksum checksum)))
(pkg-error (~a "~a checksum on package\n"
" package source: ~a\n"
" expected: ~e\n"
" got: ~e")
what
pkg-src
given-checksum
checksum)))
;; Downloads a package (if needed) and unpacks it (if needed) into a
;; temporary directory.
(define (stage-package/info pkg
@ -852,8 +867,17 @@
(define scheme (url-scheme pkg-url))
(define orig-pkg `(url ,pkg))
(define checksum (or given-checksum
(remote-package-checksum orig-pkg download-printf pkg-name)))
(define found-checksum
(case type
[(github)
;; For a github source, we want to use any given checksum to get a tarball.
(or given-checksum
(remote-package-checksum orig-pkg download-printf pkg-name))]
[else
(remote-package-checksum orig-pkg download-printf pkg-name)]))
(when check-sums?
(check-checksum given-checksum found-checksum "unexpected" pkg))
(define checksum (or found-checksum given-checksum))
(define info
(update-install-info-orig-pkg
(match type
@ -998,16 +1022,10 @@
(pkg-error (~a "remote package had no checksum\n"
" package: ~a")
pkg))
(when (and checksum
(install-info-checksum info)
check-sums?
(not (equal? (install-info-checksum info) checksum)))
(pkg-error (~a "incorrect checksum on package\n"
" package: ~a\n"
" expected ~e\n"
" got ~e")
pkg
(install-info-checksum info) checksum))
(when check-sums?
(check-checksum checksum (install-info-checksum info)
"mismatched"
pkg))
(update-install-info-checksum
info
checksum)]
@ -1019,16 +1037,12 @@
(and (file-exists? checksum-pth)
check-sums?
(file->string checksum-pth)))
(check-checksum given-checksum expected-checksum "unexpected" pkg)
(define actual-checksum
(with-input-from-file pkg
(λ ()
(sha1 (current-input-port)))))
(unless (or (not expected-checksum)
(string=? expected-checksum actual-checksum))
(pkg-error (~a "incorrect checksum on package\n"
" expected: ~e\n"
" got: ~e")
expected-checksum actual-checksum))
(check-checksum expected-checksum actual-checksum "mismatched" pkg)
(define checksum
actual-checksum)
(define pkg-format (filename-extension pkg))
@ -1095,7 +1109,8 @@
(simple-form-path pkg)
#:more-than-root? #t)))
pkg
#f #f
#f
given-checksum ; if a checksum is provided, just use it
(directory->module-paths pkg pkg-name metadata-ns))]
[else
(define pkg-dir
@ -1117,7 +1132,7 @@
`(dir ,(simple-form-path* pkg))
pkg-dir
(or (not in-place?) in-place-clean?)
#f
given-checksum ; if a checksum is provided, just use it
(directory->module-paths pkg-dir pkg-name metadata-ns))]))]
[(eq? type 'name)
(define catalog-info (package-catalog-lookup pkg #f download-printf))
@ -1131,10 +1146,9 @@
download-printf
metadata-ns
#:strip strip-mode))
(when (and (install-info-checksum info)
check-sums?
(not (equal? (install-info-checksum info) checksum)))
(pkg-error "incorrect checksum on package\n package: ~a" pkg))
(when check-sums?
(check-checksum given-checksum checksum "unexpected" pkg)
(check-checksum checksum (install-info-checksum info) "incorrect" pkg))
(update-install-info-orig-pkg
(update-install-info-checksum
info
@ -1145,13 +1159,12 @@
(define (pkg-stage desc
#:namespace [metadata-ns (make-metadata-namespace)]
#:checksum [checksum #f]
#:in-place? [in-place? #f]
#:strip [strip-mode #f])
(define i (stage-package/info (pkg-desc-source desc)
(pkg-desc-type desc)
(pkg-desc-name desc)
#:given-checksum checksum
#:given-checksum (pkg-desc-checksum desc)
#t
void
metadata-ns
@ -1203,7 +1216,7 @@
(define all-db (merge-pkg-dbs))
(define path-pkg-cache (make-hash))
(define (install-package/outer infos desc info)
(match-define (pkg-desc pkg type orig-name auto?) desc)
(match-define (pkg-desc pkg type orig-name given-checksum auto?) desc)
(match-define
(install-info pkg-name orig-pkg pkg-dir clean? checksum module-paths)
info)
@ -1384,7 +1397,8 @@
((packages-to-update download-printf current-scope-db
#:must-update? #f #:deps? #t
#:update-cache update-cache
#:namespace metadata-ns)
#:namespace metadata-ns
#:ignore-checksums? ignore-checksums?)
name))
null))
deps))
@ -1478,7 +1492,8 @@
(let ([to-update (append-map (packages-to-update download-printf db
#:deps? update-deps?
#:update-cache update-cache
#:namespace metadata-ns)
#:namespace metadata-ns
#:ignore-checksums? ignore-checksums?)
update-pkgs)])
(λ () (for-each (compose (remove-package quiet?) pkg-desc-name) to-update))))
(match this-dep-behavior
@ -1538,7 +1553,8 @@
(define metadata-ns (make-metadata-namespace))
(define infos
(for/list ([v (in-list descs)])
(stage-package/info (pkg-desc-source v) (pkg-desc-type v) (pkg-desc-name v)
(stage-package/info (pkg-desc-source v) (pkg-desc-type v) (pkg-desc-name v)
#:given-checksum (pkg-desc-checksum v)
check-sums? download-printf
metadata-ns
#:strip strip-mode
@ -1714,7 +1730,7 @@
(for/list ([dep (in-list deps)])
(if (pkg-desc? dep)
dep
(pkg-desc dep #f #f #t))))])])
(pkg-desc dep #f #f #f #t))))])])
(install-packages
#:old-infos old-infos
#:old-descs old-descs
@ -1748,7 +1764,8 @@
#:must-update? [must-update? #t]
#:deps? [deps? #f]
#:namespace metadata-ns
#:update-cache update-cache)
#:update-cache update-cache
#:ignore-checksums? ignore-checksums?)
pkg-name)
(cond
[(pkg-desc? pkg-name)
@ -1762,17 +1779,31 @@
inferred-name))
;; Check that the package is installed, and get current checksum:
(define info (package-info name #:db db))
(if (or (not (pkg-info-checksum info))
(not (equal? (pkg-info-checksum info)
(checksum-for-pkg-source (pkg-desc-source pkg-name)
type
name download-printf))))
(define new-checksum (checksum-for-pkg-source (pkg-desc-source pkg-name)
type
name
(pkg-desc-checksum pkg-name)
download-printf))
(unless (or ignore-checksums? (not (pkg-desc-checksum pkg-name)))
(unless (equal? (pkg-desc-checksum pkg-name) new-checksum)
(pkg-error (~a "incorrect checksum on package\n"
" package source: ~a\n"
" expected: ~e\n"
" got: ~e")
(pkg-desc-source pkg-name)
(pkg-desc-checksum pkg-name)
new-checksum)))
(if (or (not (equal? (pkg-info-checksum info)
new-checksum))
;; No checksum available => always update
(not new-checksum))
;; Update:
(begin
(hash-set! update-cache (pkg-desc-source pkg-name) #t)
(list (pkg-desc (pkg-desc-source pkg-name)
(pkg-desc-type pkg-name)
name
(pkg-desc-checksum pkg-name)
(pkg-desc-auto? pkg-name))))
;; No update needed, but maybe check dependencies:
(if deps?
@ -1780,7 +1811,8 @@
#:must-update? #f
#:deps? #t
#:update-cache update-cache
#:namespace metadata-ns)
#:namespace metadata-ns
#:ignore-checksums? ignore-checksums?)
pkg-name)
null))]
[(eq? #t (hash-ref update-cache pkg-name #f))
@ -1833,7 +1865,7 @@
(clear-checksums-in-cache! update-cache)
;; FIXME: the type shouldn't be #f here; it should be
;; preseved from install time:
(list (pkg-desc orig-pkg-source #f pkg-name auto?))))
(list (pkg-desc orig-pkg-source #f pkg-name #f auto?))))
(if deps?
;; Check dependencies
(append-map
@ -1841,7 +1873,8 @@
#:must-update? #f
#:deps? #t
#:update-cache update-cache
#:namespace metadata-ns)
#:namespace metadata-ns
#:ignore-checksums? ignore-checksums?)
((package-dependencies metadata-ns db) pkg-name))
null))]))]
[else null]))
@ -1880,7 +1913,8 @@
#:deps? (or update-deps?
all-mode?) ; avoid races
#:update-cache update-cache
#:namespace metadata-ns)
#:namespace metadata-ns
#:ignore-checksums? ignore-checksums?)
pkgs))
(cond
[(empty? to-update)
@ -1979,7 +2013,7 @@
[(list 'url url) (values url #f)]
[(list 'link path) (values path 'link)]
[(list 'static-link path) (values path 'static-link)]))
(pkg-desc source type name #f))
(pkg-desc source type name #f #f))
string<?
#:key pkg-desc-name))
(unless quiet?
@ -2608,8 +2642,9 @@
(printf/flush "Downloading ~s\n" source))
(define-values (checksum modules deps)
(get-pkg-content (pkg-desc source
#f
(hash-ref ht 'checksum #f)
#f
name
(hash-ref ht 'checksum #f)
#f)))
(db:set-pkg-modules! name catalog checksum modules)
(db:set-pkg-dependencies! name catalog checksum deps)))))))
@ -2669,7 +2704,8 @@
[pkg-desc
(-> string?
(or/c #f 'file 'dir 'link 'static-link 'file-url 'dir-url 'github 'name)
(or/c string? #f)
(or/c string? #f)
(or/c string? #f)
boolean?
pkg-desc?)]
[pkg-config
@ -2750,7 +2786,6 @@
(hash/c string? pkg-info?))]
[pkg-stage (->* (pkg-desc?)
(#:namespace namespace?
#:checksum (or/c #f string?)
#:in-place? boolean?
#:strip (or/c #f 'source 'binary))
(values string?

View File

@ -127,7 +127,7 @@
;; ----------------------------------------
[install
"Install packages"
#:once-any
#:once-each
install-type-flags ...
#:once-any
[install-dep-flags ...
@ -165,7 +165,7 @@
#:strip (or (and source 'source) (and binary 'binary))
#:link-dirs? link-dirs?
(for/list ([p (in-list pkg-source)])
(pkg-desc p a-type name #f))))))
(pkg-desc p a-type name checksum #f))))))
(setup no-setup setup-collects jobs)))]
;; ----------------------------------------
[update
@ -173,7 +173,7 @@
#:once-each
[#:bool all ("-a") ("Update all packages if no <pkg-source> is given")]
[#:bool lookup () "For each name <pkg-source>, look up in catalog"]
#:once-any
#:once-each
install-type-flags ...
#:once-any
[install-dep-flags ...
@ -203,13 +203,13 @@
(pkg-update (for/list ([pkg-source (in-list pkg-source)])
(cond
[lookup
(pkg-desc pkg-source a-type name #f)]
(pkg-desc pkg-source a-type name checksum #f)]
[else
(define-values (pkg-name pkg-type)
(package-source->name+type pkg-source a-type))
(if (eq? pkg-type 'name)
pkg-name
(pkg-desc pkg-source a-type name #f))]))
(pkg-desc pkg-source a-type name checksum #f))]))
#:all? all
#:dep-behavior (if auto 'search-auto deps)
#:force? force
@ -427,7 +427,9 @@
"valid <types>s are: file, dir, file-url, dir-url, github, or name;"
"if not specified, the type is inferred syntactically")]
[(#:str name #f) name ("-n") ("Name of package, instead of inferred"
"(makes sense only when a single <pkg-source> is given)")])
"(makes sense only when a single <pkg-source> is given)")]
[(#:str checksum #f) checksum () ("Checksum of package, either expected or selected"
"(makes sense only when a single <pkg-source> is given)")])
#:install-dep-flags
([(#:sym mode [fail force search-ask search-auto] #f) deps ()
("Specify the behavior for uninstalled dependencies, with"

View File

@ -77,7 +77,7 @@
(error 'check-dependencies "package not installed: ~s" pkg))
;; Get package information:
(define-values (checksum mods deps+build-deps)
(get-pkg-content (pkg-desc (if (path? dir) (path->string dir) dir) 'dir pkg #f)
(get-pkg-content (pkg-desc (if (path? dir) (path->string dir) dir) 'dir pkg #f #f)
#:namespace metadata-ns
#:extract-info (lambda (i)
(if (and i

View File

@ -183,6 +183,7 @@
(pkg-desc (path->string dir)
'static-link
#f
#f
auto?))))))
;; link configuration