unicode: Update to 11.0

[factor.git] / basis / unicode / collation / collation.factor

diff --git a/basis/unicode/collation/collation.factor b/basis/unicode/collation/collation.factor
index b6f55cfd6ba298dd5a13ec898ef7b34a666dbc40..13c2e68b0d2f4349518aecfe766752711d89ada2 100644 (file)
--- a/basis/unicode/collation/collation.factor
+++ b/basis/unicode/collation/collation.factor
@@ -1,20 +1,25 @@
 ! Copyright (C) 2008 Daniel Ehrenberg.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: accessors arrays assocs combinators
-combinators.short-circuit kernel locals make math math.order
-math.parser namespaces sequences simple-flat-file splitting
-strings unicode.data ;
+combinators.short-circuit combinators.smart kernel locals make
+math math.order math.parser namespaces sequences
+simple-flat-file splitting strings unicode.data ;
 IN: unicode.collation
 
 <PRIVATE
 
 SYMBOL: ducet
 
-TUPLE: weight primary secondary tertiary ignorable? ;
+TUPLE: weight-levels primary secondary tertiary ignorable? ;
+: <weight-levels> ( primary secondary tertiary -- weight-levels> )
+    weight-levels new
+        swap >>tertiary
+        swap >>secondary
+        swap >>primary ; inline
 
 : parse-weight ( string -- weight )
     "]" split but-last [
-        weight new swap rest unclip CHAR: * = swapd >>ignorable?
+        weight-levels new swap rest unclip CHAR: * = swapd >>ignorable?
         swap "." split first3 [ hex> ] tri@
         [ >>primary ] [ >>secondary ] [ >>tertiary ] tri*
     ] map ;
@@ -23,10 +28,9 @@ TUPLE: weight primary secondary tertiary ignorable? ;
     " " split [ hex> ] "" map-as ;
 
 : parse-ducet ( file -- ducet )
-    load-data-file
-    [ [ parse-keys ] [ parse-weight ] bi* ] H{ } assoc-map-as ;
+    load-data-file [ [ parse-keys ] [ parse-weight ] bi* ] H{ } assoc-map-as ;
 
-"vocab:unicode/collation/allkeys.txt" parse-ducet ducet set-global
+"vocab:unicode/UCA/allkeys.txt" parse-ducet ducet set-global
 
 ! Fix up table for long contractions
 : help-one ( assoc key -- )
@@ -37,24 +41,37 @@ TUPLE: weight primary secondary tertiary ignorable? ;
     ] if ;
 
 : insert-helpers ( assoc -- )
-    dup keys [ length 3 >= ] filter
-    [ help-one ] with each ;
+    dup keys [ length 3 >= ] filter [ help-one ] with each ;
 
 ducet get-global insert-helpers
 
-:: base ( char -- base )
+: tangut-block? ( char -- ? )
+    ! Tangut Block, Tangut Components Block
+    { [ 0x17000 0x187FF between? ] [ 0x18800 0x18AFF between? ] } 1|| ; inline
+
+! Unicode TR10 - Computing Implicit Weights
+: base ( char -- base )
     {
-        { [ char 0x03400 0x04DB5 between? ] [ 0xFB80 ] } ! Extension A
-        { [ char 0x20000 0x2A6D6 between? ] [ 0xFB80 ] } ! Extension B
-        { [ char 0x04E00 0x09FC3 between? ] [ 0xFB40 ] } ! CJK
-        [ 0xFBC0 ] ! Other
+        { [ dup 0x03400 0x04DB5 between? ] [ drop 0xFB80 ] } ! Extension A
+        { [ dup 0x20000 0x2A6D6 between? ] [ drop 0xFB80 ] } ! Extension B
+        { [ dup 0x2A700 0x2B734 between? ] [ drop 0xFB80 ] } ! Extension C
+        { [ dup 0x2B740 0x2B81D between? ] [ drop 0xFB80 ] } ! Extension D
+        { [ dup 0x2B820 0x2CEA1 between? ] [ drop 0xFB80 ] } ! Extension E
+        { [ dup 0x04E00 0x09FD5 between? ] [ drop 0xFB40 ] } ! CJK
+        [ drop 0xFBC0 ] ! Other
     } cond ;
 
-: AAAA ( char -- weight )
-    [ base ] [ -15 shift ] bi + 0x20 2 f weight boa ;
+: tangut-AAAA ( char -- weight-levels )
+    drop 0xfb00 0x0020 0x0002 <weight-levels> ; inline
+
+: tangut-BBBB ( char -- weight-levels )
+    0x17000 - 0x8000 bitor 0 0 <weight-levels> ; inline
 
-: BBBB ( char -- weight )
-    0x7FFF bitand 0x8000 bitor 0 0 f weight boa ;
+: AAAA ( char -- weight-levels )
+    [ base ] [ -15 shift ] bi + 0x0020 0x0002 <weight-levels> ; inline
+
+: BBBB ( char -- weight-levels )
+    0x7FFF bitand 0x8000 bitor 0 0 <weight-levels> ; inline
 
 : illegal? ( char -- ? )
     {
@@ -62,11 +79,16 @@ ducet get-global insert-helpers
         [ category "Cs" = ]
     } 1|| ;
 
-: derive-weight ( char -- weights )
-    first dup illegal? [
-        drop { }
+: derive-weight ( 1string -- weight-levels-pair )
+    first
+    dup tangut-block? [
+        [ tangut-AAAA ] [ tangut-BBBB ] bi 2array
     ] [
-        [ AAAA ] [ BBBB ] bi 2array
+        dup illegal? [
+            drop { }
+        ] [
+            [ AAAA ] [ BBBB ] bi 2array
+        ] if
     ] if ;
 
 : building-last ( -- char )
@@ -96,28 +118,32 @@ ducet get-global insert-helpers
 : string>graphemes ( string -- graphemes )
     [ [ add ] each ] { } make ;
 
+: char>weight-levels ( 1string -- weight-levels )
+    ducet get-global ?at [ derive-weight ] unless ; inline
+
 : graphemes>weights ( graphemes -- weights )
     [
-        dup weight? [ 1array ] ! From tailoring
-        [ dup ducet get-global at [ ] [ derive-weight ] ?if ] if
+        dup weight-levels?
+        [ 1array ] ! From tailoring
+        [ char>weight-levels ] if
     ] { } map-as concat ;
 
-: append-weights ( weights quot -- )
-    [ [ ignorable?>> ] reject ] dip map
-    [ zero? ] reject % 0 , ; inline
+: append-weights ( weight-levels quot -- seq )
+    [ [ ignorable?>> ] reject ] dip
+    map [ zero? ] reject ; inline
 
-: variable-weight ( weight -- )
-    dup ignorable?>> [ primary>> ] [ drop 0xFFFF ] if , ;
+: variable-weight ( weight-levels -- obj )
+    dup ignorable?>> [ primary>> ] [ drop 0xFFFF ] if ;
 
-: weights>bytes ( weights -- byte-array )
+: weights>bytes ( weights -- array )
     [
         {
-            [ [ primary>> ] append-weights ]
-            [ [ secondary>> ] append-weights ]
-            [ [ tertiary>> ] append-weights ]
-            [ [ variable-weight ] each ]
+            [ [ primary>> ] append-weights { 0 } ]
+            [ [ secondary>> ] append-weights { 0 } ]
+            [ [ tertiary>> ] append-weights { 0 } ]
+            [ [ [ secondary>> ] [ tertiary>> ] bi [ zero? ] bi@ and not ] filter [ variable-weight ] map ]
         } cleave
-    ] { } make ;
+    ] { } append-outputs-as ;
 
 PRIVATE>