suzanne.soy/racket
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lex identifiers and delimiters

Browse Source
This commit is contained in:
Jon Rafkind 2011-07-24 17:06:16 -04:00
parent 4e8e18463b
commit f941e56f71
1 changed files with 32 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
collects/honu/core/read.rkt
View File

@ -1,18 +1,35 @@
#lang racket/base
(require rackunit)
(require parser-tools/lex)
(require parser-tools/lex
(prefix-in : parser-tools/lex-sre))
(require racket/match)
(define-tokens honu-tokens (number))
(define-tokens honu-tokens (number identifier))
(define-empty-tokens honu-empty-tokens (eof fail whitespace))
(define-empty-tokens honu-empty-tokens (eof fail whitespace
left-parens right-parens
left-bracket right-bracket
left-brace right-brace))
(define-lex-abbrev identifier-first-character (:or (:/ #\a #\z)
(:/ #\A #\Z)))
(define-lex-abbrev identifier-character identifier-first-character)
(define-lex-abbrev identifier (:: identifier-first-character
(:+ identifier-character)))
(define honu-lexer
(lexer-src-pos
[(eof) (token-eof)]
[(char-range #\0 #\9)
(token-number (string->number lexeme))]
["(" (token-left-parens)]
[")" (token-right-parens)]
["[" (token-left-bracket)]
["]" (token-right-bracket)]
["{" (token-left-brace)]
["}" (token-right-brace)]
[identifier (token-identifier (string->symbol lexeme))]
[(union " ") (token-whitespace)]
))
@ -43,4 +60,16 @@
(list (token-number 5)))
(check-equal? (lex-string "5 8")
(list (token-number 5) (token-number 8)))
(check-equal? (lex-string "hello")
(list (token-identifier 'hello)))
(check-equal? (lex-string "()")
(list (token-left-parens)
(token-right-parens)))
(check-equal? (lex-string "()[]{}")
(list (token-left-parens)
(token-right-parens)
(token-left-bracket)
(token-right-bracket)
(token-left-brace)
(token-right-brace)))
)