suzanne.soy/racket
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

syntax-color/racket-lexer: fix long-standing number-parsing bug

Browse Source 
The lexer mishandled cases like "#x1E+2", because "E" is
not an exponent marker for hexadecimal --- and it has been that
way forever, but the repair isn't all that difficult.
This commit is contained in:
Matthew Flatt 2013-03-28 20:29:02 -06:00
parent f1ebb6136b
commit 1dcc7ca4dc
2 changed files with 31 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
collects/syntax-color/racket-lexer.rkt
View File

@ -107,6 +107,7 @@
[special-numbers (:or (:: n a n ".0") (:: i n f ".0"))] [special-numbers (:or (:: n a n ".0") (:: i n f ".0"))]
[exponent-marker (:or e s f d l)] [exponent-marker (:or e s f d l)]
[exponent-marker16 (:or s l)]
[sign (char-set "+-")] [sign (char-set "+-")]
[exactness (:or "#i" "#e" "#I" "#E")] [exactness (:or "#i" "#e" "#I" "#E")]
[radix2 (:or "#b" "#B")] [radix2 (:or "#b" "#B")]
@ -162,7 +163,8 @@
(define-lex-trans make-num (define-lex-trans make-num
(syntax-rules () (syntax-rules ()
((_ digit radix) (:: (make-prefix radix) (make-complex digit))))) ((_ digit radix exponent-marker)
(:: (make-prefix radix) (make-complex digit exponent-marker)))))
(define-lex-trans make-prefix (define-lex-trans make-prefix
(syntax-rules () (syntax-rules ()
@ -171,27 +173,29 @@
(define-lex-trans make-complex (define-lex-trans make-complex
(syntax-rules () (syntax-rules ()
((_ digit) ((_ digit exponent-marker)
(:or (make-real digit) (:or (make-real digit exponent-marker)
(:: (make-real digit) "@" (make-real digit)) (:: (make-real digit exponent-marker) "@" (make-real digit exponent-marker))
(:: (make-real digit) "+" (:or special-numbers (make-ureal digit)) i) (:: (make-real digit exponent-marker) "+" (:or special-numbers (make-ureal digit exponent-marker)) i)
(:: (make-real digit) "-" (:or special-numbers (make-ureal digit)) i) (:: (make-real digit exponent-marker) "-" (:or special-numbers (make-ureal digit exponent-marker)) i)
(:: (make-real digit) "+" i) (:: (make-real digit exponent-marker) "+" i)
(:: (make-real digit) "-" i) (:: (make-real digit exponent-marker) "-" i)
(:: "+" (:or special-numbers (make-ureal digit)) i) (:: "+" (:or special-numbers (make-ureal digit exponent-marker)) i)
(:: "-" (:or special-numbers (make-ureal digit)) i) (:: "-" (:or special-numbers (make-ureal digit exponent-marker)) i)
(:: "+" i) (:: "+" i)
(:: "-" i))))) (:: "-" i)))))
(define-lex-trans make-ureal (define-lex-trans make-ureal
(syntax-rules () (syntax-rules ()
((_ digit) (:or (make-uinteger digit) ((_ digit exponent-marker)
(:: (make-uinteger digit) "/" (make-uinteger digit) (:? (make-suffix digit))) (:or (make-uinteger digit)
(make-decimal digit))))) (:: (make-uinteger digit) "/" (make-uinteger digit) (:? (make-suffix digit exponent-marker)))
(make-decimal digit exponent-marker)))))
(define-lex-trans make-real (define-lex-trans make-real
(syntax-rules () (syntax-rules ()
((_ digit) (:or (:: (:? sign) (make-ureal digit)) ((_ digit exponent-marker)
(:or (:: (:? sign) (make-ureal digit exponent-marker))
(:: (char-set "+-") special-numbers))))) (:: (char-set "+-") special-numbers)))))
(define-lex-trans make-uinteger (define-lex-trans make-uinteger
@ -200,15 +204,15 @@
(define-lex-trans make-decimal (define-lex-trans make-decimal
(syntax-rules () (syntax-rules ()
((_ digit) ((_ digit exponent-marker)
(:or (:: (make-uinteger digit) (make-suffix digit)) (:or (:: (make-uinteger digit) (make-suffix digit exponent-marker))
(:: "." (:+ digit) (:* "#") (make-suffix digit)) (:: "." (:+ digit) (:* "#") (make-suffix digit exponent-marker))
(:: (:+ digit) "." (:* digit) (:* "#") (make-suffix digit)) (:: (:+ digit) "." (:* digit) (:* "#") (make-suffix digit exponent-marker))
(:: (:+ digit) (:+ "#") "." (:* "#") (make-suffix digit)))))) (:: (:+ digit) (:+ "#") "." (:* "#") (make-suffix digit exponent-marker))))))
(define-lex-trans make-suffix (define-lex-trans make-suffix
(syntax-rules () (syntax-rules ()
((_ digit) (:or "" (:: exponent-marker (:? sign) (:+ digit)))))) ((_ digit exponent-marker) (:or "" (:: exponent-marker (:? sign) (:+ digit))))))
(define (ret lexeme type paren start-pos end-pos status) (define (ret lexeme type paren start-pos end-pos status)
@ -314,10 +318,10 @@
'error) 'error)
#f start-pos end-pos 'datum)] #f start-pos end-pos 'datum)]
[(:or character [(:or character
(make-num digit2 radix2) (make-num digit2 radix2 exponent-marker)
(make-num digit8 radix8) (make-num digit8 radix8 exponent-marker)
(make-num digit10 (:? radix10)) (make-num digit10 (:? radix10) exponent-marker)
(make-num digit16 radix16)) (make-num digit16 radix16 exponent-marker16))
(ret lexeme 'constant #f start-pos end-pos 'datum)] (ret lexeme 'constant #f start-pos end-pos 'datum)]
[keyword (ret lexeme 'hash-colon-keyword #f start-pos end-pos 'datum)] [keyword (ret lexeme 'hash-colon-keyword #f start-pos end-pos 'datum)]
[str (ret lexeme 'string #f start-pos end-pos 'datum)] [str (ret lexeme 'string #f start-pos end-pos 'datum)]

8
collects/tests/syntax-color/scheme-lexer.rkt
View File

@ -482,12 +482,8 @@ end-string
(test "1-inf.0I" "cccccccc") (test "1-inf.0I" "cccccccc")
;; Bad numbers ;; Bad numbers
;; these next two tests are WRONG; if you (test "#x1E+2" "xxxxxx")
;; fix something that makes these tests return (test "#x1d+2" "xxxxxx")
;; the proper results, great! They are this
;; way now so we notice later.
(test "#x1E+2" "cccccc") ;; should be "xxxxxx"
(test "#x1d+2" "cccccc") ;; should be "xxxxxx"
;; Keywords ;; Keywords
(test "#:" "hh") (test "#:" "hh")