simple-tokenizer: consider \t \n \r spaces also.

diff --git a/basis/simple-tokenizer/simple-tokenizer-tests.factor b/basis/simple-tokenizer/simple-tokenizer-tests.factor
index 3b44f03650147ace53045b390c9330ce9060aa4e..5f7190331311025d12da2a3acc3c476bc97e8e97 100644 (file)
--- a/basis/simple-tokenizer/simple-tokenizer-tests.factor
+++ b/basis/simple-tokenizer/simple-tokenizer-tests.factor
@@ -31,3 +31,8 @@ USING: simple-tokenizer tools.test ;
 ] [
     "\"Hello world.app/Contents/MacOS/hello-ui\" -i=boot.macosx-ppc.image \"-include= math compiler ui\" -deploy-vocab=hello-ui \"-output-image=Hello world.app/Contents/Resources/hello-ui.image\" -no-stack-traces -no-user-init" tokenize
 ] unit-test
+
+{ V{ "ls" "-l" } } [ "ls -l" tokenize ] unit-test
+{ V{ "ls" "-l" } } [ "ls -l\n" tokenize ] unit-test
+{ V{ "ls" "-l" } } [ "\nls -l" tokenize ] unit-test
+{ V{ "ls" "-l" } } [ "\nls -l\n" tokenize ] unit-test

diff --git a/basis/simple-tokenizer/simple-tokenizer.factor b/basis/simple-tokenizer/simple-tokenizer.factor
index f7de0822c1ce0addb1b3fecde71b182102f9e87c..cf6b5a6091429db94034453a41e1f4e49079b988 100644 (file)
--- a/basis/simple-tokenizer/simple-tokenizer.factor
+++ b/basis/simple-tokenizer/simple-tokenizer.factor
@@ -4,10 +4,10 @@ USING: peg.ebnf strings ;
 IN: simple-tokenizer
 
 EBNF: tokenize
-space = " "
+space = [ \t\n\r]
 escaped-char = "\" .:ch => [[ ch ]]
 quoted = '"' (escaped-char | [^"])*:a '"' => [[ a ]]
-unquoted = (escaped-char | [^ "])+
+unquoted = (escaped-char | [^ \t\n\r"])+
 argument = (quoted | unquoted) => [[ >string ]]
 command = space* (argument:a space* => [[ a ]])+:c !(.) => [[ c ]]
 ;EBNF