unicode: Update to 11.0

[factor.git] / basis / unicode / normalize / normalize-tests.factor

diff --git a/basis/unicode/normalize/normalize-tests.factor b/basis/unicode/normalize/normalize-tests.factor
index 51630d29326f8f54edc1dccac6bbf4eb97a22922..71f0a4f7ef31cd0aac184862a0015da69125dda1 100644 (file)
--- a/basis/unicode/normalize/normalize-tests.factor
+++ b/basis/unicode/normalize/normalize-tests.factor
@@ -1,12 +1,17 @@
-USING: assocs combinators.short-circuit kernel locals math
-math.parser sequences simple-flat-file splitting tools.test
-unicode unicode.normalize.private ;
+USING: arrays assocs combinators combinators.short-circuit
+grouping io.encodings.utf8 io.files io.streams.null kernel
+locals math math.parser quotations random sequences splitting
+splitting.extras strings tools.test unicode
+unicode.normalize.private ;
 IN: unicode.normalize.tests
 
 { "ab\u000323\u000302cd" } [ "ab\u000302" "\u000323cd" string-append ] unit-test
 
-{ "ab\u00064b\u000347\u00034e\u00034d\u000346" } [ "ab\u000346\u000347\u00064b\u00034e\u00034d" dup reorder ] unit-test
+{ "ab\u00064b\u000347\u00034e\u00034d\u000346" }
+[ "ab\u000346\u000347\u00064b\u00034e\u00034d" dup reorder ] unit-test
+
 { "hello" "hello" } [ "hello" [ nfd ] keep nfkd ] unit-test
+
 { "\u00FB012\u002075\u00017F\u000323\u000307" "fi25s\u000323\u000307" }
 [ "\u00FB012\u002075\u001E9B\u000323" [ nfd ] keep nfkd ] unit-test
 
@@ -20,22 +25,36 @@ IN: unicode.normalize.tests
 { "\u001112\u001161\u0011ab" } [ "\u00d55c" nfd ] unit-test
 { "\u00d55c" } [ "\u001112\u001161\u0011ab" nfc ] unit-test
 
-:: check? ( test spec quot -- ? )
+! Could use simple-flat-file after some cleanup
+: parse-normalization-tests ( -- tests )
+    "vocab:unicode/UCD/NormalizationTest.txt" utf8 file-lines
+    [ "#" head? ] reject
+    [ "@" head? ] split*-when
+    2 <groups> [ first2 [ first ] dip 2array ] map
+    values [
+        [
+            "#@" split first [ CHAR: \s = ] trim-tail ";" split harvest
+            [ " " split [ hex> ] "" map-as ] map
+        ] map
+    ] map concat ;
+
+:: check-normalization-test? ( test spec quot -- ? )
     spec [
         [
             [ 1 - test nth ] bi@ quot call( str -- str' ) =
         ] with all?
     ] assoc-all? ;
 
-{ 17768 { } } [
-    "vocab:unicode/normalize/NormalizationTest.txt" load-data-file
-    [ 5 head [ " " split [ hex> ] "" map-as ] map ] map
-    [ length ] keep [
+{ { } } [
+    parse-normalization-tests [
         {
-            [ { { 2 { 1 2 3 } } { 4 { 4 5 } } } [ nfc ] check? ]
-            [ { { 3 { 1 2 3 } } { 5 { 4 5 } } } [ nfd ] check? ]
-            [ { { 4 { 1 2 3 4 5 } } } [ nfkc ] check? ]
-            [ { { 5 { 1 2 3 4 5 } } } [ nfkd ] check? ]
+            [ { { 2 { 1 2 3 } } { 4 { 4 5 } } } [ nfc ] check-normalization-test? ]
+            [ { { 3 { 1 2 3 } } { 5 { 4 5 } } } [ nfd ] check-normalization-test? ]
+            [ { { 4 { 1 2 3 4 5 } } } [ nfkc ] check-normalization-test? ]
+            [ { { 5 { 1 2 3 4 5 } } } [ nfkd ] check-normalization-test? ]
         } 1&&
     ] reject
 ] unit-test
+
+{ { 4018 820 3953 3968 } }
+[ { 3958 3953 820 } >string nfd >array ] unit-test