1 USING: xmode.tokens xmode.catalog
2 xmode.marker tools.test kernel ;
6 T{ token f "int" KEYWORD3 }
10 } [ f "int x" "c" load-mode tokenize-line nip ] unit-test
14 T{ token f "\"" LITERAL1 }
15 T{ token f "hello\\\"" LITERAL1 }
16 T{ token f " " LITERAL1 }
17 T{ token f "world" LITERAL1 }
18 T{ token f "\"" LITERAL1 }
20 } [ f "\"hello\\\" world\"" "c" load-mode tokenize-line nip ] unit-test
24 T{ token f "\"" LITERAL1 }
25 T{ token f "hello\\\ world" LITERAL1 }
26 T{ token f "\"" LITERAL1 }
28 } [ f "\"hello\\\ world\"" "c" load-mode tokenize-line nip ] unit-test
32 T{ token f "int" KEYWORD3 }
36 } [ f "int x" "java" load-mode tokenize-line nip ] unit-test
40 T{ token f "#" COMMENT1 }
41 T{ token f " " COMMENT1 }
42 T{ token f "hello" COMMENT1 }
43 T{ token f " " COMMENT1 }
44 T{ token f "world" COMMENT1 }
46 } [ f "# hello world" "python" load-mode tokenize-line nip ] unit-test
51 T{ token f "hello" f }
53 T{ token f "world" f }
56 } [ f "hello world:" "java" load-mode tokenize-line nip ] unit-test
60 T{ token f "hello_world" LABEL }
61 T{ token f ":" OPERATOR }
63 } [ f "hello_world:" "java" load-mode tokenize-line nip ] unit-test
68 T{ token f "hello_world" LABEL }
69 T{ token f ":" OPERATOR }
71 } [ f "\thello_world:" "java" load-mode tokenize-line nip ] unit-test
75 T{ token f "<!" KEYWORD2 }
76 T{ token f "ELEMENT" KEYWORD2 }
77 T{ token f " " KEYWORD2 }
78 T{ token f "%" LITERAL2 }
79 T{ token f "hello" LITERAL2 }
80 T{ token f ";" LITERAL2 }
81 T{ token f " " KEYWORD2 }
82 T{ token f ">" KEYWORD2 }
85 f "<!ELEMENT %hello; >" "xml" load-mode tokenize-line nip
90 T{ token f "<!" KEYWORD2 }
91 T{ token f "ELEMENT" KEYWORD2 }
92 T{ token f " " KEYWORD2 }
93 T{ token f "%" LITERAL2 }
94 T{ token f "hello-world" LITERAL2 }
95 T{ token f ";" LITERAL2 }
96 T{ token f " " KEYWORD2 }
97 T{ token f ">" KEYWORD2 }
100 f "<!ELEMENT %hello-world; >" "xml" load-mode tokenize-line nip
105 T{ token f "$" KEYWORD2 }
106 T{ token f "FOO" KEYWORD2 }
109 f "$FOO" "shellscript" load-mode tokenize-line nip
114 T{ token f "AND" KEYWORD1 }
117 f "AND" "pascal" load-mode tokenize-line nip
122 T{ token f "Comment {" COMMENT1 }
123 T{ token f "XXX" COMMENT1 }
124 T{ token f "}" COMMENT1 }
127 f "Comment {XXX}" "rebol" load-mode tokenize-line nip
133 f "font:75%/1.6em \"Lucida Grande\", \"Lucida Sans Unicode\", verdana, geneva, sans-serif;" "css" load-mode tokenize-line 2drop
138 T{ token f "<" MARKUP }
139 T{ token f "aaa" MARKUP }
140 T{ token f ">" MARKUP }
142 } [ f "<aaa>" "html" load-mode tokenize-line nip ] unit-test