Merge pull request #53 from florence/expose-compiler

Expose compiler
This commit is contained in:
Ryan Plessner 2015-03-10 21:59:24 -04:00
commit 91a185e3b3
6 changed files with 226 additions and 130 deletions

240
cover.rkt
View File

@ -1,5 +1,9 @@
#lang racket/base
(provide test-files! clear-coverage! get-test-coverage)
(provide test-files!
make-cover-environment clear-coverage!
get-test-coverage
current-cover-environment environment?
environment-compile environment-namespace)
#|
@ -15,9 +19,12 @@ in "coverage.rkt". This raw coverage information is converted to a usable form b
syntax/modcode
racket/function
syntax/modread
syntax/modresolve
syntax/parse
unstable/syntax
racket/bool
racket/runtime-path
racket/match
rackunit
unstable/error
racket/list
@ -25,30 +32,31 @@ in "coverage.rkt". This raw coverage information is converted to a usable form b
"private/shared.rkt"
"private/file-utils.rkt")
;; namespace used for coverage
(define ns #f)
(struct environment (namespace compile ann-top raw-cover cch))
;;; ---------------------- Running Files ---------------------------------
;; PathString * -> Boolean
;; Test files and build coverage map
;; returns true if no tests reported as failed, and no files errored.
(define (test-files! #:submod [submod-name 'test] . paths)
(unless ns (unloaded-error))
(define abs
(for/list ([p (in-list paths)])
(if (list? p)
(cons (->absolute (car p)) (cdr p))
(->absolute p))))
(define abs-paths (map (lambda (p) (if (list? p) (first p) p)) abs))
(parameterize ([current-load/use-compiled (make-cover-load/use-compiled abs-paths)]
[current-output-port
(if (verbose) (current-output-port) (open-output-nowhere))])
(define (test-files! #:submod [submod-name 'test] #:env [env (current-cover-environment)] . files)
(parameterize ([current-cover-environment env])
(define abs
(for/list ([p (in-list files)])
(if (list? p)
(cons (->absolute (car p)) (cdr p))
(->absolute p))))
(define abs-names
(for/list ([p (in-list abs)])
(match p
[(cons p _) p]
[_ p])))
(define tests-failed #f)
(for ([p (in-list abs)])
(vprintf "attempting to run ~s\n" p)
(define old-check (current-check-handler))
(define path (if (list? p) (car p) p))
(define the-file (if (list? p) (car p) p))
(define argv (if (list? p) (cadr p) #()))
(vprintf "running file: ~s with args: ~s\n" path argv)
(vprintf "running file: ~s with args: ~s\n" the-file argv)
(struct an-exit (code))
(with-handlers ([(lambda (x) (or (not (exn? x)) (exn:fail? x)))
(lambda (x)
@ -57,35 +65,54 @@ in "coverage.rkt". This raw coverage information is converted to a usable form b
[else
(set! tests-failed #t)
(error-display x)]))])
(parameterize* ([current-command-line-arguments argv]
(parameterize* ([current-load/use-compiled (make-cover-load/use-compiled abs-names)]
[current-output-port
(if (verbose) (current-output-port) (open-output-nowhere))]
[current-command-line-arguments argv]
[exit-handler (lambda (x) (raise (an-exit x)))]
[current-namespace ns]
[current-namespace (get-namespace)]
[(get-check-handler-parameter)
(lambda x
(set! tests-failed #t)
(vprintf "file ~s had failed tests\n" p)
(apply old-check x))])
(define file `(file ,(if (path? path) (path->string path) path)))
(define submod `(submod ,file ,submod-name))
(run-mod (if (module-declared? submod #t) submod file)))))
(vprintf "ran ~s\n" paths)
(remove-unneeded-results! abs-paths)
(run-file the-file submod-name))))
(vprintf "ran ~s\n" files)
(remove-unneeded-results! abs-names)
(not tests-failed)))
;; ModulePath -> Void
;; evaluate the current module in the current namespace
;;; ---------------------- Running Aux ---------------------------------
(define (run-file the-file submod-name)
(define sfile `(file ,(if (path? the-file) (path->string the-file) the-file)))
(define submod `(submod ,sfile ,submod-name))
(run-mod (if (module-declared? submod #t) submod sfile)))
(define (run-mod to-run)
(vprintf "running ~s\n" to-run)
(eval `(dynamic-require ',to-run 0))
(eval (make-dyn-req-expr to-run))
(vprintf "finished running ~s\n" to-run))
;; [Listof Path] -> Loader Compiler
(define (make-dyn-req-expr to-run)
`(dynamic-require ',to-run 0))
;; [Listof Any] -> Void
;; remove any files not in paths from the raw coverage
(define (remove-unneeded-results! names)
(define c (get-raw-coverage))
(for ([s (in-list (hash-keys c))]
#:when (not (member (srcloc-source s) names)))
(hash-remove! c s)))
;;; ---------------------- Compiling ---------------------------------
;; (U [Listof Path] #f) -> Loader Compiler
;; returns a value that can be set of `current-load/use-compiled`
;; forces the given files to be recompiled whenever load/use-compiled is called
(define (make-cover-load/use-compiled paths)
(define load/use-compiled (current-load/use-compiled))
(define load (current-load))
(define cover-compile (make-cover-compile))
(define cover-compile (get-compile))
(define cover-use-compiled-file-paths
(cons (build-path "compiled" "cover")
(use-compiled-file-paths)))
@ -94,7 +121,7 @@ in "coverage.rkt". This raw coverage information is converted to a usable form b
(define lst (explode-path abs))
(define dir-list (take lst (sub1 (length lst))))
(parameterize ([current-load-relative-directory (apply build-path dir-list)])
(if (member abs paths)
(if (implies paths (member abs paths))
(parameterize ([current-compile cover-compile]
[use-compiled-file-paths
cover-use-compiled-file-paths])
@ -103,12 +130,12 @@ in "coverage.rkt". This raw coverage information is converted to a usable form b
;; -> Compiler
;; makes a value sutable for current-compile, such that compile
;; annotates the source code. should only be used by `make-cover-load/uze-compiled`
(define (make-cover-compile)
;; annotates the source code with annotate-top. meant to be called
;; only by initialize-cover-environment
(define (make-cover-compile ns annotate-top)
(define compile (current-compile))
(define reg (namespace-module-registry ns))
(define phase (namespace-base-phase ns))
(define annotate-top (get-annotate-top))
;; define so its named in stack traces
(define cover-compile
(lambda (e immediate-eval?)
@ -129,80 +156,69 @@ in "coverage.rkt". This raw coverage information is converted to a usable form b
(compile to-compile immediate-eval?)))
cover-compile)
;; [Listof PathString] -> Void
;; remove any files not in paths from the raw coverage
(define (remove-unneeded-results! paths)
(define c (get-raw-coverage))
(for ([s (in-list (hash-keys c))]
#:when (not (member (srcloc-source s) paths)))
(hash-remove! c s)))
;;; ---------------------- Environments ---------------------------------
;; -> Void
;; clear coverage map. Effectively recreates and rebuilds `ns`
(define (clear-coverage!)
(set! ns (make-base-namespace))
(current-cover-environment (make-cover-environment)))
(define (make-cover-environment [ns (make-base-namespace)])
(parameterize ([current-namespace ns])
(namespace-require 'cover/coverage)
(namespace-require 'cover/strace)
(namespace-require 'rackunit))
(load-names!))
(define ann (load-annotate-top))
(environment
ns
(make-cover-compile ns ann)
ann
(load-raw-coverage)
(load-current-check-handler))))
;; -> Void
;; loads any needed names from `ns` before it can get polluted.
(define (load-names!)
(load-annotate-top!)
(load-raw-coverage!)
(load-current-check-handler!))
(define ann-top #f)
(define (get-annotate-top)
(or ann-top (unloaded-error)))
(define (load-annotate-top!)
(set! ann-top (get-ns-var 'annotate-top)))
(get-val environment-ann-top))
(define (load-annotate-top)
(dynamic-require 'cover/strace 'annotate-top))
(define raw-cover #f)
(define (get-raw-coverage)
(or raw-cover (unloaded-error)))
(define (load-raw-coverage!)
(set! raw-cover (get-ns-var 'coverage)))
(get-val environment-raw-cover))
(define (load-raw-coverage)
(dynamic-require 'cover/coverage 'coverage))
(define cch #f)
(define (load-current-check-handler!)
(set! cch (get-ns-var 'current-check-handler)))
(define (get-check-handler-parameter)
(or cch (unloaded-error)))
(get-val environment-cch))
(define (load-current-check-handler)
(dynamic-require 'rackunit 'current-check-handler))
(define (unloaded-error)
(error 'cover "Test coverage not loaded."))
(define (get-namespace)
(get-val environment-namespace))
(define (get-ns-var sym)
(namespace-variable-value sym #t #f ns))
(define (get-compile)
(get-val environment-compile))
(define (get-val access)
(access (current-cover-environment)))
;; -> [Hashof PathString (Listof (List Boolean srcloc))]
;; returns a hash of file to a list, where the first of the list is if
;; that srcloc was covered or not
;; based on <pkgs>/drracket/drracket/private/debug.rkt
(define (get-test-coverage)
(vprintf "generating test coverage\n")
(define (get-test-coverage [env (current-cover-environment)])
(parameterize ([current-cover-environment env])
(vprintf "generating test coverage\n")
;; filtered : (listof (list boolean srcloc))
;; remove redundant expressions
(define filtered (hash-map (get-raw-coverage) (λ (k v) (list v k))))
;; filtered : (listof (list boolean srcloc))
;; remove redundant expressions
(define filtered (hash-map (get-raw-coverage) (λ (k v) (list v k))))
(define out (make-hash))
(define out (make-hash))
(for ([v (in-list filtered)])
(define file (srcloc-source (cadr v)))
(hash-update! out
file
(lambda (l) (cons v l))
null))
out)
;; A little hack to setup coverage namespace for the first time
(clear-coverage!)
(for ([v (in-list filtered)])
(define file (srcloc-source (cadr v)))
(hash-update! out
file
(lambda (l) (cons v l))
null))
out))
(define current-cover-environment
(make-parameter (make-cover-environment)))
;; here live tests for actually saving compiled files
(module+ test
@ -213,15 +229,43 @@ in "coverage.rkt". This raw coverage information is converted to a usable form b
"tests/compiled/prog_rkt.zo"
"tests/compiled/prog_rkt.dep"))
(test-begin
(after
(for-each (lambda (f) (when (file-exists? f) (delete-file f)))
compiled)
(check-false (ormap file-exists? compiled))
(check-not-exn
(lambda ()
(parameterize ([current-load/use-compiled
(make-cover-load/use-compiled (list (->absolute prog.rkt)))]
[current-namespace ns])
(managed-compile-zo prog.rkt))))
(check-true (andmap file-exists? compiled))
(clear-coverage!))))
(parameterize ([current-cover-environment (make-cover-environment)])
(for-each (lambda (f) (when (file-exists? f) (delete-file f)))
compiled)
(check-false (ormap file-exists? compiled))
(check-not-exn
(lambda ()
(parameterize ([current-load/use-compiled
(make-cover-load/use-compiled (list (->absolute prog.rkt)))]
[current-namespace (get-namespace)])
(managed-compile-zo prog.rkt))))
(check-true (andmap file-exists? compiled)))))
;; tests repl like interactions
(module+ test
(require rackunit racket/runtime-path racket/file
racket/format
racket/lazy-require)
;; break cyclic dependency in testing
(lazy-require ["private/format-utils.rkt" (make-covered?)])
(define-runtime-path simple-multi/2.rkt "tests/simple-multi/2.rkt")
(define env (make-cover-environment))
(define ns (environment-namespace env))
(parameterize ([current-cover-environment env]
[current-namespace ns])
(test-begin
(define file (path->string simple-multi/2.rkt))
(define modpath file)
(parameterize ([current-load/use-compiled
(make-cover-load/use-compiled (list file))])
(namespace-require `(file ,modpath)))
(check-equal? (eval `(two)) 10)
(define x (get-test-coverage env))
(define covered?
(make-covered? (hash-ref x file) file))
(for ([_ (in-string (file->string file))]
[i (in-naturals 1)])
(define c (covered? i))
(check-true (or (eq? c 'covered)
(eq? c 'irrelevant))
(~a i))))))

View File

@ -2,25 +2,42 @@
(require "cover.rkt" "format.rkt" "private/contracts.rkt" "private/format-utils.rkt"
"private/raw.rkt" racket/contract)
(define (not-impersonated/c c)
(and/c (lambda (v) (not (impersonator? v)))
c))
(provide
(contract-out
[coverage/c contract?]
[file-coverage/c contract?]
[test-files! (->* () (#:submod symbol?)
[test-files! (->* () (#:submod symbol?
#:env environment?)
#:rest
(listof (or/c path-string?
(list/c path-string?
(and/c (lambda (v) (not (impersonator? v)))
(vectorof string? #:immutable #t)))))
(not-impersonated/c
(vectorof (not-impersonated/c string?) #:immutable #t)))))
any)]
[environment? (-> any/c any/c)]
[environment-namespace (-> environment? namespace?)]
[environment-compile
(-> environment? (any/c boolean? . -> . compiled-expression?))]
[clear-coverage! (-> any)]
[get-test-coverage (-> coverage/c)]
[make-cover-environment (->* () ((-> namespace?)) environment?)]
[current-cover-environment (parameter/c environment?)]
[get-test-coverage (->* () (environment?) coverage/c)]
[irrelevant-submodules (parameter/c (or/c #f (listof symbol?)))]
[make-covered?
(-> file-coverage/c path-string?
(->* (exact-positive-integer?)
(#:byte? boolean?)
(or/c 'covered 'uncovered 'irrelevant)))]
[generate-coveralls-coverage coverage-gen/c]
[generate-html-coverage coverage-gen/c]
[generate-raw-coverage coverage-gen/c]))

View File

@ -3,6 +3,6 @@
(require racket/contract)
(define file-coverage/c (listof (list/c boolean? srcloc?)))
(define coverage/c (hash/c (and/c path-string? absolute-path?)
file-coverage/c))
;; if its a file path, will be an absolute path
(define coverage/c (hash/c any/c file-coverage/c))
(define coverage-gen/c (->* (coverage/c) (path-string?) any))

View File

@ -88,7 +88,7 @@
(define file (path->string (simplify-path tests/prog.rkt)))
(test-files! (path->string (simplify-path tests/prog.rkt)))
(define coverage (get-test-coverage))
(define report
(define report
(with-env ("COVERALLS_REPO_TOKEN" "abc") (generate-coveralls-report coverage)))
(check-equal?
(hash-ref report 'source_files)
@ -128,7 +128,8 @@
;; Generates a string that represents a valid coveralls json_file object
(define (generate-source-files coverage)
(define src-files
(for/list ([file (in-list (hash-keys coverage))])
(for/list ([file (in-list (hash-keys coverage))]
#:when (absolute-path? file))
(define local-file (path->string (find-relative-path (current-directory) file)))
(define src (file->string file))
(define c (line-coverage coverage file))

View File

@ -46,7 +46,8 @@
(define (get-files coverage dir)
(define file-list
(for/list ([(k v) (in-hash coverage)])
(for/list ([(k v) (in-hash coverage)]
#:when (absolute-path? k))
(vprintf "building html coverage for: ~a\n" k)
(define exploded (explode-path k))
(define-values (_ dir-list)

View File

@ -9,40 +9,43 @@ In addition to being a raco tool, Cover provides racket bindings for running
tests and collecting coverage information. The following are the basic
functions of test coverage.
@section[#:tag "higher"]{A High Level API}
@deftogether[(@defthing[coverage/c
contract?
#:value (hash/c (and/c path-string? absolute-path?)
file-coverage/c)]
#:value (hash/c any/c file-coverage/c)]
@defthing[file-coverage/c contract? #:value (listof (list/c boolean? srcloc?))])]{
Coverage information is a hash map mapping absolute
file paths to a list detailing the coverage of that file. The file coverage
information is a list of lists, mapping a boolean to a range of
characters within the file. True means the @racket[srcloc] structure
represents an expression that was run, and False means the structure
represents an expression that was not run. Some expressions may not be
represented directly in this coverage information.
For example, type annotations in @racketmodname[typed/racket]
removed during macro expansion and are thus neither run or not run.
Note that the @racket[srcloc]s are one indexed, meaning a @racket[1]
represents the first character in the file.}
Coverage information is a hash map mapping absolute file paths to a list detailing the coverage of
that file. The file is keyed on the @racket[syntax-source] of the syntax objects from that
file. Usually this will be the absolute path to the file. The file coverage information is a list of
lists, mapping a boolean to a range of characters within the file. True means the @racket[srcloc]
structure represents an expression that was run, and False means the structure represents an
expression that was not run. Some expressions may not be represented directly in this coverage
information. For example, type annotations in @racketmodname[typed/racket] removed during macro
expansion and are thus neither run or not run. Note that the @racket[srcloc]s are one indexed,
meaning a @racket[1] represents the first character in the file.}
@defproc[(test-files! (#:submod submod symbol? 'test)
(files (or/c path-string?
(list/c path-string
(and/c (negate impersonator?)
(vectorof string? #:immutable #t))))) ...)
(files
(or/c path-string?
(list/c path-string?
(vectorof string? #:immutable #t)))) ...)
any]{
Runs all given @racket[files] and their submodule @racket[submod] (if it exists), storing the
coverage information. If the path is paired with a vector then that vector is used as the
@racket[current-command-line-arguments] when executing that file. This vector must be immuatable and
not wrapped by a @racket[chaperone] or @racket[impersonator]. The function returns false if any
tests fail. Test coverage information is still collected when test fail. Test coverage info is
added to existing coverage info.}
@racket[current-command-line-arguments] when executing that file. This vector must be immutable and
not wrapped by a @racket[chaperone?] or @racket[impersonator?], nor may its elements be wrapped in a
@racket[chaperone?] or @racket[impersonator?]. The function returns false if any tests fail. Test
coverage information is still collected when test fail. Test coverage info is added to existing
coverage info.}
@defproc[(clear-coverage!) any]{Clears all coverage information.}
@defproc[(clear-coverage! [environment environment? (current-coverage-environment)]) any]{
Clears all coverage information.}
@defproc[(get-test-coverage) coverage/c]{Gets the current coverage information.}
@defproc[(get-test-coverage [environment environment? (current-coverage-environment)]) coverage/c]{
Gets the current coverage information.}
@defproc[(make-covered? (coverage file-coverage/c) (path path-string?))
(->* (exact-positive-integer?)
(#:byte? boolean?)
@ -73,3 +76,33 @@ considered irrelevant.}
Generates coverage information in the coveralls and html formats. Equivalent to the specifications
of the @Flag{c} argument to @exec{raco cover}. Both use @racket[make-covered?] to determine file
coverage.}
@section[#:tag "lower"]{A Lower Level API}
The high level API may not be enough for some applications. For example an IDE may need separate
instances of the coverage table, or may need direct access to the namespace code is run in. For this
purpose @racket[cover] directly expose coverage environments.
Coverage environments are values that package together a coverage namespace, a compiler for
annotating code, and a coverage table to write coverage results to. All other coverage functions use
the @racket[current-coverage-environment] for code coverage, unless explicitly given a different
environment.
@defproc[(environment? [v any/c]) any/c]{
Tests if the given value is a coverage environment.}
@defthing[current-coverage-environment (parameter/c environment?)]{
The current coverage environment. Defaults to an environment built from
@racket[make-base-namespace]}
@defproc[(environment-namespace [environment environment?]) namespace?]{
Get the namespace that coverage should be run in. This is the same namespace given to
@racket[make-cover-environment]}
@defproc[(environment-compile [environment environment?])
(any/c boolean? . -> . compiled-expression?)]{
Returns a value suitable for @racket[current-compile] that will compile code with coverage
annotations. That code must be run in @racket[environment]'s namespace.}
@defproc[(make-cover-environment [namespace namespace? (make-base-namespace)]) environment?]{
Makes a coverage environment such that @racket[environment-namespace] will return
@racket[namespace], and @racket[namespace] will be set up to handle coverage information.}