lesson-lisp/tests/arcana/tokenize.scm

(assert-equal '() (arcana/tokenize ""))
(assert-equal '() (arcana/tokenize "   "))
;(assert-equal '() (arcana/tokenize "\n"))
(assert-equal '() (arcana/tokenize "; foo"))
;(assert-equal '() (arcana/tokenize "   ; foo\n   ; foo"))

(assert-equal (arcana/tokenize "(") (list (cons 'TOKEN_ROUND_OPEN    "(")))
(assert-equal (arcana/tokenize ")") (list (cons 'TOKEN_ROUND_CLOSE   ")")))
(assert-equal (arcana/tokenize "[") (list (cons 'TOKEN_SQUARE_OPEN   "[")))
(assert-equal (arcana/tokenize "]") (list (cons 'TOKEN_SQUARE_CLOSE  "]")))
(assert-equal (arcana/tokenize "{") (list (cons 'TOKEN_CURLY_OPEN    "{")))
(assert-equal (arcana/tokenize "}") (list (cons 'TOKEN_CURLY_CLOSE   "}")))
(assert-equal (arcana/tokenize "'") (list (cons 'TOKEN_QUOTE         "'")))
(assert-equal (arcana/tokenize "`") (list (cons 'TOKEN_QUASI_QUOTE   "`")))
(assert-equal (arcana/tokenize ",") (list (cons 'TOKEN_QUASI_UNQUOTE ",")))

(assert-equal (arcana/tokenize "#f")       (list (cons 'TOKEN_TAG "f")))
(assert-equal (arcana/tokenize "#t")       (list (cons 'TOKEN_TAG "t")))
(assert-equal (arcana/tokenize "#false")   (list (cons 'TOKEN_TAG "false")))
(assert-equal (arcana/tokenize "#true")    (list (cons 'TOKEN_TAG "true")))
(assert-equal (arcana/tokenize "#qwe")     (list (cons 'TOKEN_TAG "qwe")))

(assert-equal (arcana/tokenize "qwe")      (list (cons 'TOKEN_IDENT "qwe")))

(assert-equal (arcana/tokenize "123")      (list (cons 'TOKEN_NUM "123")))
;(assert-equal (arcana/tokenize "-123")     (list (cons 'TOKEN_NUM "-123")))

(assert-equal (arcana/tokenize "\"\"")     (list (cons 'TOKEN_STRING "")))
(assert-equal (arcana/tokenize "\"qwe\"")  (list (cons 'TOKEN_STRING "qwe")))
(assert-equal (arcana/tokenize "\"\\\"\"") (list (cons 'TOKEN_STRING "\"")))

(assert-equal
  (arcana/tokenize "(displayln \"qwe\")")
  (list
    (cons 'TOKEN_ROUND_OPEN  "(")
    (cons 'TOKEN_IDENT       "displayln")
    (cons 'TOKEN_STRING      "qwe")
    (cons 'TOKEN_ROUND_CLOSE ")")))
(assert-equal
  (arcana/tokenize "(displayln (list 1))")
  (list
    (cons 'TOKEN_ROUND_OPEN  "(")
    (cons 'TOKEN_IDENT       "displayln")
    (cons 'TOKEN_ROUND_OPEN  "(")
    (cons 'TOKEN_IDENT       "list")
    (cons 'TOKEN_NUM         "1")
    (cons 'TOKEN_ROUND_CLOSE ")")
    (cons 'TOKEN_ROUND_CLOSE ")")))