diff --git a/collects/honu/core/read.rkt b/collects/honu/core/read.rkt index 8cbc71c5b1..c49b885b8d 100644 --- a/collects/honu/core/read.rkt +++ b/collects/honu/core/read.rkt @@ -1,18 +1,35 @@ #lang racket/base (require rackunit) -(require parser-tools/lex) +(require parser-tools/lex + (prefix-in : parser-tools/lex-sre)) (require racket/match) -(define-tokens honu-tokens (number)) +(define-tokens honu-tokens (number identifier)) -(define-empty-tokens honu-empty-tokens (eof fail whitespace)) +(define-empty-tokens honu-empty-tokens (eof fail whitespace + left-parens right-parens + left-bracket right-bracket + left-brace right-brace)) + +(define-lex-abbrev identifier-first-character (:or (:/ #\a #\z) + (:/ #\A #\Z))) +(define-lex-abbrev identifier-character identifier-first-character) +(define-lex-abbrev identifier (:: identifier-first-character + (:+ identifier-character))) (define honu-lexer (lexer-src-pos [(eof) (token-eof)] [(char-range #\0 #\9) (token-number (string->number lexeme))] + ["(" (token-left-parens)] + [")" (token-right-parens)] + ["[" (token-left-bracket)] + ["]" (token-right-bracket)] + ["{" (token-left-brace)] + ["}" (token-right-brace)] + [identifier (token-identifier (string->symbol lexeme))] [(union " ") (token-whitespace)] )) @@ -43,4 +60,16 @@ (list (token-number 5))) (check-equal? (lex-string "5 8") (list (token-number 5) (token-number 8))) + (check-equal? (lex-string "hello") + (list (token-identifier 'hello))) + (check-equal? (lex-string "()") + (list (token-left-parens) + (token-right-parens))) + (check-equal? (lex-string "()[]{}") + (list (token-left-parens) + (token-right-parens) + (token-left-bracket) + (token-right-bracket) + (token-left-brace) + (token-right-brace))) )