enable unreliable tests via PLT_RUN_UNRELIABLE_TESTS

The attempt in 7a12d55e7d to handle some flaky tests didn't work.
This attempt effectively removes the tests, although they can
be enabled by setting `PLT_RUN_UNRELIABLE_TESTS` to "timing".

Also, replace the old hack of enabling some locale-specific tests
to also use `PLT_RUN_UNRELIABLE_TESTS`.
This commit is contained in:
Matthew Flatt 2019-11-29 07:46:26 -07:00
parent b8d2513b6d
commit 2920fc3f64
4 changed files with 26 additions and 32 deletions

View File

@ -8,16 +8,6 @@
racket/list
(prefix-in k: '#%kernel))
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Make sure `flaky-test` works
(let ([x 0])
(define (inc! v)
(set! x (+ x v))
x)
;; Will pass on the thrid try:
(flaky-test 3 inc! 1))
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(test '() 'null null)

View File

@ -140,16 +140,20 @@
;; ----------------------------------------
;; Alarms
(unless (eq? (system-type 'gc) 'cgc)
(flaky-test #f sync/timeout 0.1 (alarm-evt (+ (current-inexact-milliseconds) 200))))
(flaky-test 'ok sync/timeout 0.1
(wrap-evt
(alarm-evt (+ (current-inexact-milliseconds) 50))
(lambda (x) 'ok)))
(flaky-test 'ok sync/timeout 100
(wrap-evt
(alarm-evt (+ (current-inexact-milliseconds) 50))
(lambda (x) 'ok)))
;; These tests are inherently flaky, because they rely on Racket
;; running fast enough relative to wall-clock time.
(when (run-unreliable-tests? 'timing)
(test #f sync/timeout 0.1 (alarm-evt (+ (current-inexact-milliseconds) 200)))
(test 'ok sync/timeout 0.1
(wrap-evt
(alarm-evt (+ (current-inexact-milliseconds) 50))
(lambda (x) 'ok)))
(test 'ok sync/timeout 100
(wrap-evt
(alarm-evt (+ (current-inexact-milliseconds) 50))
(lambda (x) 'ok))))
;; ----------------------------------------
;; Waitable sets

View File

@ -137,13 +137,6 @@ transcript.
(define (test expect fun . args) (test* expect fun args #f #f))
(make-keyword-procedure test/kw test)))
;; A flaky test is one that won't always pass, perhaps because it
;; is sensitive to timing or GC. But it should pass if we
;; try enough times. The test must never error.
(define-syntax-rule (flaky-test arg ...)
(parameterize ([wrong-result-retries 10])
(test arg ...)))
(define (nonneg-exact? x)
(and (exact? x)
(integer? x)
@ -419,3 +412,12 @@ transcript.
;; No way to detect stack overflow, and it's less interesting anyway,
;; but make up a number for testing purposes
1000]))
;; Set the `PLT_RUN_UNRELIABLE_TESTS` environment to a comma-separated set of
;; extra tests to enable.
(define (run-unreliable-tests? mode)
(define s (getenv "PLT_RUN_UNRELIABLE_TESTS"))
(and s
(let ([l (map string->symbol (string-split s ","))])
(memq mode l))))

View File

@ -1157,11 +1157,9 @@
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Let Matthew perform some basic sanity checks for locale-sensitive
;; comparisons:
(define known-locale? (and (regexp-match "mflatt|matthewf" (path->string (find-system-path 'home-dir)))
(or (regexp-match "linux" (path->string (system-library-subpath)))
(eq? 'macosx (system-type)))))
;; Enable unreliable to run some basic sanity checks for locale-sensitive
;; comparisons that need a locale wirth various properties:
(define known-locale? (run-unreliable-tests? 'locale))
(printf "Known locale?: ~a\n" known-locale?)