! Copyright (C) 2008 Daniel Ehrenberg.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays assocs combinators
-combinators.short-circuit kernel locals make math math.order
-math.parser namespaces sequences simple-flat-file splitting
-strings unicode.data ;
+combinators.short-circuit combinators.smart fry kernel make
+math math.order math.parser namespaces sequences
+simple-flat-file splitting strings unicode.data ;
IN: unicode.collation
<PRIVATE
SYMBOL: ducet
-TUPLE: weight primary secondary tertiary ignorable? ;
+TUPLE: weight-levels primary secondary tertiary ignorable? ;
+: <weight-levels> ( primary secondary tertiary -- weight-levels> )
+ weight-levels new
+ swap >>tertiary
+ swap >>secondary
+ swap >>primary ; inline
: parse-weight ( string -- weight )
"]" split but-last [
- weight new swap rest unclip CHAR: * = swapd >>ignorable?
+ weight-levels new swap rest unclip CHAR: * = swapd >>ignorable?
swap "." split first3 [ hex> ] tri@
[ >>primary ] [ >>secondary ] [ >>tertiary ] tri*
] map ;
: parse-keys ( string -- chars )
- " " split [ hex> ] "" map-as ;
+ split-words [ hex> ] "" map-as ;
: parse-ducet ( file -- ducet )
- data [ [ parse-keys ] [ parse-weight ] bi* ] H{ } assoc-map-as ;
+ load-data-file [ [ parse-keys ] [ parse-weight ] bi* ] H{ } assoc-map-as ;
-"vocab:unicode/collation/allkeys.txt" parse-ducet ducet set-global
+"vocab:unicode/UCA/allkeys.txt" parse-ducet ducet set-global
-! Fix up table for long contractions
-: help-one ( assoc key -- )
- ! Need to be more general? Not for DUCET, apparently
- 2 head 2dup swap key? [ 2drop ] [
- [ [ 1string of ] with { } map-as concat ]
- [ swap set-at ] 2bi
- ] if ;
+! https://www.unicode.org/reports/tr10/tr10-41.html#Well_Formed_DUCET
+! WF5 - Well-formedness 5 condition:
+! https://www.unicode.org/reports/tr10/tr10-41.html#WF5
+! { "0CC6" "0CC2" "0CD5" } ! 0CD5 is not a non-starter, don't add 2-gram "0CC6" "0CC2"to ducet
+! { "0DD9" "0DCF" "0DCA" } ! already in allkeys.txt file
+! { "0FB2" "0F71" "0F80" } ! added below
+! { "0FB3" "0F71" "0F80" } ! added below
+! This breaks the unicode tests that ship in CollationTest_SHIFTED.txt
+! but it's supposedly more correct.
+: fixup-ducet-for-tibetan ( -- )
+ {
+ {
+ { 0x0FB2 0x0F71 } ! CE(0FB2) CE(0F71)
+ {
+ T{ weight-levels
+ { primary 12719 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12741 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ {
+ { 0x0FB3 0x0F71 } ! CE(0FB3) CE(0F71)
+ {
+ T{ weight-levels
+ { primary 12722 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12741 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+
+ {
+ { 0x0FB2 0x0F71 0x0F72 } ! CE(0FB2) CE(0F71 0F72)
+ {
+ T{ weight-levels
+ { primary 12719 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12743 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ {
+ { 0x0FB2 0x0F73 } ! CE(0FB2) CE(0F71 0F72)
+ {
+ T{ weight-levels
+ { primary 12719 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12743 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ {
+ { 0x0FB2 0x0F71 0x0F74 } ! CE(0FB2) CE(0F71 0F74)
+ {
+ T{ weight-levels
+ { primary 12719 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12747 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ {
+ { 0x0FB2 0x0F75 } ! CE(0FB2) CE(0F71 0F74)
+ {
+ T{ weight-levels
+ { primary 12719 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12747 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ {
+ { 0x0FB3 0x0F71 0x0F72 } ! CE(0FB3) CE(0F71 0F72)
+ {
+ T{ weight-levels
+ { primary 12722 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12743 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ {
+ { 0x0FB3 0x0F73 } ! CE(0FB3) CE(0F71 0F72)
+ {
+ T{ weight-levels
+ { primary 12722 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12743 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ {
+ { 0x0FB3 0x0F71 0x0F74 } ! CE(0FB3) CE(0F71 0F74)
+ {
+ T{ weight-levels
+ { primary 12722 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12747 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ {
+ { 0x0FB3 0x0F75 } ! CE(0FB3) CE(0F71 0F74)
+ {
+ T{ weight-levels
+ { primary 12722 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ T{ weight-levels
+ { primary 12747 }
+ { secondary 32 }
+ { tertiary 2 }
+ }
+ }
+ }
+ } ducet get-global '[ swap >string _ set-at ] assoc-each ;
-: insert-helpers ( assoc -- )
- dup keys [ length 3 >= ] filter
- [ help-one ] with each ;
+! These values actually break the collation unit tests in CollationTest_SHIFTED.txt
+! So we disable those tests in favor of supposedly better collation for Tibetan.
+! https://www.unicode.org/reports/tr10/tr10-41.html#Well_Formed_DUCET
-ducet get-global insert-helpers
+fixup-ducet-for-tibetan
+: tangut-block? ( char -- ? )
+ {
+ [ 0x17000 0x18AFF between? ] ! Tangut and Tangut Components
+ [ 0x18D00 0x18D8F between? ] ! Tangut Supplement
+ } 1|| ; inline
+
+: nushu-block? ( char -- ? )
+ 0x1b170 0x1B2FF between? ; inline
+
+: khitan-block? ( char -- ? )
+ 0x18b00 0x18cff between? ; inline
+
+! https://wiki.computercraft.cc/Module:Unicode_data
+! Unicode TR10 - Computing Implicit Weights
: base ( char -- base )
{
- { [ dup 0x3400 0x4DB5 between? ] [ drop 0xFB80 ] } ! Extension A
- { [ dup 0x20000 0x2A6D6 between? ] [ drop 0xFB80 ] } ! Extension B
- { [ dup 0x4E00 0x9FC3 between? ] [ drop 0xFB40 ] } ! CJK
+ { [ dup 0x03400 0x04DBF between? ] [ drop 0xFB80 ] } ! Extension A
+ { [ dup 0x20000 0x2A6DF between? ] [ drop 0xFB80 ] } ! Extension B
+ { [ dup 0x2A700 0x2B739 between? ] [ drop 0xFB80 ] } ! Extension C
+ { [ dup 0x2B740 0x2B81D between? ] [ drop 0xFB80 ] } ! Extension D
+ { [ dup 0x2B820 0x2CEA1 between? ] [ drop 0xFB80 ] } ! Extension E
+ { [ dup 0x2CEB0 0x2EBE0 between? ] [ drop 0xFB80 ] } ! Extension F
+ { [ dup 0x30000 0x3134A between? ] [ drop 0xFB80 ] } ! Extension G
+ { [ dup 0x31350 0x323AF between? ] [ drop 0xFB80 ] } ! Extension H
+ { [ dup 0x2F800 0x2FA1D between? ] [ drop 0xFB80 ] } ! CJK Compatibility
+ { [ dup 0x04E00 0x09FFF between? ] [ drop 0xFB40 ] } ! CJK
+ { [ dup 0x0F900 0x0FAD9 between? ] [ drop 0xFB40 ] } ! CJK
[ drop 0xFBC0 ] ! Other
} cond ;
-: AAAA ( char -- weight )
- [ base ] [ -15 shift ] bi + 0x20 2 f weight boa ;
+: tangut-AAAA ( char -- weight-levels )
+ drop 0xfb00 0x0020 0x0002 <weight-levels> ; inline
+
+: tangut-BBBB ( char -- weight-levels )
+ 0x17000 - 0x8000 bitor 0 0 <weight-levels> ; inline
+
+: nushu-AAAA ( char -- weight-levels )
+ drop 0xfb01 0x0020 0x0002 <weight-levels> ; inline
+
+: nushu-BBBB ( char -- weight-levels )
+ 0x1B170 - 0x8000 bitor 0 0 <weight-levels> ; inline
-: BBBB ( char -- weight )
- 0x7FFF bitand 0x8000 bitor 0 0 f weight boa ;
+: khitan-AAAA ( char -- weight-levels )
+ drop 0xfb02 0x0020 0x0002 <weight-levels> ; inline
-: illegal? ( char -- ? )
- { [ "Noncharacter_Code_Point" property? ] [ category "Cs" = ] } 1|| ;
+: khitan-BBBB ( char -- weight-levels )
+ 0x18b00 - 0x8000 bitor 0 0 <weight-levels> ; inline
-: derive-weight ( char -- weights )
- first dup illegal?
- [ drop { } ]
- [ [ AAAA ] [ BBBB ] bi 2array ] if ;
+: AAAA ( char -- weight-levels )
+ [ base ] [ -15 shift ] bi + 0x0020 0x0002 <weight-levels> ; inline
+
+: BBBB ( char -- weight-levels )
+ 0x7FFF bitand 0x8000 bitor 0 0 <weight-levels> ; inline
+
+: derive-weight ( 1string -- weight-levels-pair )
+ first
+ {
+ { [ dup tangut-block? ] [ [ tangut-AAAA ] [ tangut-BBBB ] bi 2array ] }
+ { [ dup nushu-block? ] [ [ nushu-AAAA ] [ nushu-BBBB ] bi 2array ] }
+ { [ dup khitan-block? ] [ [ khitan-AAAA ] [ khitan-BBBB ] bi 2array ] }
+ [ [ AAAA ] [ BBBB ] bi 2array ]
+ } cond ;
: building-last ( -- char )
- building get empty? [ 0 ] [ building get last last ] if ;
+ building get [ 0 ] [ last last ] if-empty ;
+! https://www.unicode.org/reports/tr10/tr10-41.html#Collation_Graphemes
: blocked? ( char -- ? )
combining-class dup { 0 f } member?
[ drop building-last non-starter? ]
: add ( char -- )
dup blocked? [ 1string , ] [
- dup possible-bases dup length iota
+ dup possible-bases dup length <iota>
[ ?combine ] 2with any?
[ drop ] [ 1string , ] if
] if ;
: string>graphemes ( string -- graphemes )
[ [ add ] each ] { } make ;
+: char>weight-levels ( 1string -- weight-levels )
+ ducet get-global ?at [ derive-weight ] unless ; inline
+
: graphemes>weights ( graphemes -- weights )
[
- dup weight? [ 1array ] ! From tailoring
- [ dup ducet get-global at [ ] [ derive-weight ] ?if ] if
+ dup weight-levels?
+ [ 1array ] ! From tailoring
+ [ char>weight-levels ] if
] { } map-as concat ;
-: append-weights ( weights quot -- )
+: append-weights ( weight-levels quot -- seq )
[ [ ignorable?>> ] reject ] dip
- map [ zero? ] reject % 0 , ; inline
+ map [ zero? ] reject ; inline
-: variable-weight ( weight -- )
- dup ignorable?>> [ primary>> ] [ drop 0xFFFF ] if , ;
+: variable-weight ( weight-levels -- obj )
+ dup ignorable?>> [ primary>> ] [ drop 0xFFFF ] if ;
-: weights>bytes ( weights -- byte-array )
+: weights>bytes ( weights -- array )
[
{
- [ [ primary>> ] append-weights ]
- [ [ secondary>> ] append-weights ]
- [ [ tertiary>> ] append-weights ]
- [ [ variable-weight ] each ]
+ [ [ primary>> ] append-weights { 0 } ]
+ [ [ secondary>> ] append-weights { 0 } ]
+ [ [ tertiary>> ] append-weights { 0 } ]
+ [ [ [ secondary>> ] [ tertiary>> ] bi [ zero? ] both? ] reject [ variable-weight ] map ]
} cleave
- ] { } make ;
+ ] { } append-outputs-as ;
PRIVATE>
: completely-ignorable? ( weight -- ? )
- [ primary>> ] [ secondary>> ] [ tertiary>> ] tri
- [ zero? ] tri@ and and ;
+ {
+ [ primary>> zero? ]
+ [ secondary>> zero? ]
+ [ tertiary>> zero? ]
+ } 1&& ;
: filter-ignorable ( weights -- weights' )
f swap [