tokencase: split on graphemes.

diff --git a/extra/tokencase/tokencase.factor b/extra/tokencase/tokencase.factor
index 226dd3d22a1b1e4c94a6b5220ace55577a279df3..b92eeef17fdd8a11265f72d66c043585bfa9ae17 100644 (file)
--- a/extra/tokencase/tokencase.factor
+++ b/extra/tokencase/tokencase.factor
@@ -8,16 +8,16 @@ IN: tokencase
 <PRIVATE
 
 : case-index ( str -- i/f )
-    dup [ 1string lower? ] find drop [
-        over [ 1string lower? not ] find-from drop
+    dup [ lower? ] find drop [
+        over [ lower? not ] find-from drop
         [ nip ] [ length ] if*
     ] [ length ] if* ;
 
 : split-case ( str -- words )
-    [ dup empty? not ] [
+    >graphemes [ dup empty? not ] [
         dup case-index
         [ cut-slice swap ]
-        [ f 0 rot [ length ] keep <slice> ] if*
+        [ f 0 rot [ length ] keep <slice> ] if* concat
     ] produce nip ;
 
 : split-tokens ( str -- words )