CONSTANT: special-casing H{ }
CONSTANT: properties H{ }
-: >2ch ( a b -- c ) [ 21 shift ] dip + ;
-: 2ch> ( c -- a b ) [ -21 shift ] [ 21 on-bits mask ] bi ;
+: >2ch ( a b -- c ) [ 21 shift ] dip + ; inline
+: 2ch> ( c -- a b ) [ -21 shift ] [ 21 on-bits mask ] bi ; inline
PRIVATE>
PRIVATE>
-: category# ( char -- n )
+: category-num ( char -- n )
! There are a few characters that should be Cn
! that this gives Cf or Mn
! Cf = 26; Mn = 5; Cn = 29
] ?if ; inline
: category ( char -- category )
- category# categories nth ;
+ category-num categories nth ;
<PRIVATE
! Loading data from UnicodeData.txt
: load-data ( -- data )
- "vocab:unicode/data/UnicodeData.txt" load-data-file ;
+ "vocab:unicode/UCD/UnicodeData.txt" load-data-file ;
: (process-data) ( index data -- newdata )
[ [ nth ] keep first swap ] with { } map>assoc
[ " " split [ hex> ] map ] assoc-map ;
: exclusions-file ( -- filename )
- "vocab:unicode/data/CompositionExclusions.txt" ;
+ "vocab:unicode/UCD/CompositionExclusions.txt" ;
: exclusions ( -- set )
exclusions-file utf8 file-lines
[ "#" split1 drop [ blank? ] trim-tail hex> ] map
- [ 0 = ] reject fast-set ;
+ [ 0 = ] reject ;
+
+: unique ( seq -- assoc )
+ [ dup ] H{ } map>assoc ;
: remove-exclusions ( alist -- alist )
- exclusions [ nip sets:in? ] curry assoc-reject ;
+ exclusions unique assoc-diff ;
: process-canonical ( data -- hash hash )
(process-decomposed) [ first* ] filter
! Extra properties {{[a,b],prop}}
: parse-properties ( -- assoc )
- "vocab:unicode/data/PropList.txt" load-data-file [
+ "vocab:unicode/UCD/PropList.txt" load-data-file [
[
".." split1 [ dup ] unless*
[ hex> ] bi@ 2array
! Special casing data
: load-special-casing ( -- special-casing )
- "vocab:unicode/data/SpecialCasing.txt" load-data-file
+ "vocab:unicode/UCD/SpecialCasing.txt" load-data-file
[ length 5 = ] filter
[ [ set-code-point ] each ] H{ } make ;