unicode: Update to 11.0

[factor.git] / basis / unicode / data / data.factor

diff --git a/basis/unicode/data/data.factor b/basis/unicode/data/data.factor
index 397f4839747d1653cbe78677bcbfaee6c28b5ba1..bc744a747db21a116f950d5f697a457b8c0ef898 100644 (file)
--- a/basis/unicode/data/data.factor
+++ b/basis/unicode/data/data.factor
@@ -20,8 +20,8 @@ CONSTANT: category-map BV{ }
 CONSTANT: special-casing H{ }
 CONSTANT: properties H{ }
 
-: >2ch ( a b -- c ) [ 21 shift ] dip + ;
-: 2ch> ( c -- a b ) [ -21 shift ] [ 21 on-bits mask ] bi ;
+: >2ch ( a b -- c ) [ 21 shift ] dip + ; inline
+: 2ch> ( c -- a b ) [ -21 shift ] [ 21 on-bits mask ] bi ; inline
 
 PRIVATE>
 
@@ -57,7 +57,7 @@ CONSTANT: num-chars 0x2FA1E
 
 PRIVATE>
 
-: category# ( char -- n )
+: category-num ( char -- n )
     ! There are a few characters that should be Cn
     ! that this gives Cf or Mn
     ! Cf = 26; Mn = 5; Cn = 29
@@ -70,14 +70,14 @@ PRIVATE>
     ] ?if ; inline
 
 : category ( char -- category )
-    category# categories nth ;
+    category-num categories nth ;
 
 <PRIVATE
 
 ! Loading data from UnicodeData.txt
 
 : load-data ( -- data )
-    "vocab:unicode/data/UnicodeData.txt" load-data-file ;
+    "vocab:unicode/UCD/UnicodeData.txt" load-data-file ;
 
 : (process-data) ( index data -- newdata )
     [ [ nth ] keep first swap ] with { } map>assoc
@@ -103,15 +103,18 @@ PRIVATE>
     [ " " split [ hex> ] map ] assoc-map ;
 
 : exclusions-file ( -- filename )
-    "vocab:unicode/data/CompositionExclusions.txt" ;
+    "vocab:unicode/UCD/CompositionExclusions.txt" ;
 
 : exclusions ( -- set )
     exclusions-file utf8 file-lines
     [ "#" split1 drop [ blank? ] trim-tail hex> ] map
-    [ 0 = ] reject fast-set ;
+    [ 0 = ] reject ;
+
+: unique ( seq -- assoc )
+    [ dup ] H{ } map>assoc ;
 
 : remove-exclusions ( alist -- alist )
-    exclusions [ nip sets:in? ] curry assoc-reject ;
+    exclusions unique assoc-diff ;
 
 : process-canonical ( data -- hash hash )
     (process-decomposed) [ first* ] filter
@@ -170,7 +173,7 @@ C: <code-point> code-point
 
 ! Extra properties {{[a,b],prop}}
 : parse-properties ( -- assoc )
-    "vocab:unicode/data/PropList.txt" load-data-file [
+    "vocab:unicode/UCD/PropList.txt" load-data-file [
         [
             ".." split1 [ dup ] unless*
             [ hex> ] bi@ 2array
@@ -187,7 +190,7 @@ C: <code-point> code-point
 
 ! Special casing data
 : load-special-casing ( -- special-casing )
-    "vocab:unicode/data/SpecialCasing.txt" load-data-file
+    "vocab:unicode/UCD/SpecialCasing.txt" load-data-file
     [ length 5 = ] filter
     [ [ set-code-point ] each ] H{ } make ;