-! Copyright (C) 2008 Daniel Ehrenberg.\r
-! See http://factorcode.org/license.txt for BSD license.\r
-USING: sequences io.files io.encodings.ascii kernel splitting\r
-accessors math.parser ascii io assocs strings math namespaces make\r
-sorting combinators math.order arrays unicode.normalize unicode.data\r
-locals macros sequences.deep words unicode.breaks quotations\r
-combinators.short-circuit simple-flat-file ;\r
-IN: unicode.collation\r
-\r
-<PRIVATE\r
-SYMBOL: ducet\r
-\r
-TUPLE: weight primary secondary tertiary ignorable? ;\r
-\r
-: parse-weight ( string -- weight )\r
- "]" split but-last [\r
- weight new swap rest unclip CHAR: * = swapd >>ignorable?\r
- swap "." split first3 [ hex> ] tri@\r
- [ >>primary ] [ >>secondary ] [ >>tertiary ] tri*\r
- ] map ;\r
-\r
-: parse-keys ( string -- chars )\r
- " " split [ hex> ] "" map-as ;\r
-\r
-: parse-ducet ( file -- ducet )\r
- data [ [ parse-keys ] [ parse-weight ] bi* ] H{ } assoc-map-as ;\r
-\r
-"vocab:unicode/collation/allkeys.txt" parse-ducet ducet set-global\r
-\r
-! Fix up table for long contractions\r
-: help-one ( assoc key -- )\r
- ! Need to be more general? Not for DUCET, apparently\r
- 2 head 2dup swap key? [ 2drop ] [\r
- [ [ 1string of ] with { } map-as concat ]\r
- [ swap set-at ] 2bi\r
- ] if ;\r
-\r
-: insert-helpers ( assoc -- )\r
- dup keys [ length 3 >= ] filter\r
- [ help-one ] with each ;\r
-\r
-ducet get-global insert-helpers\r
-\r
-: base ( char -- base )\r
- {\r
- { [ dup 0x3400 0x4DB5 between? ] [ drop 0xFB80 ] } ! Extension A\r
- { [ dup 0x20000 0x2A6D6 between? ] [ drop 0xFB80 ] } ! Extension B\r
- { [ dup 0x4E00 0x9FC3 between? ] [ drop 0xFB40 ] } ! CJK\r
- [ drop 0xFBC0 ] ! Other\r
- } cond ;\r
-\r
-: AAAA ( char -- weight )\r
- [ base ] [ -15 shift ] bi + 0x20 2 f weight boa ;\r
-\r
-: BBBB ( char -- weight )\r
- 0x7FFF bitand 0x8000 bitor 0 0 f weight boa ;\r
-\r
-: illegal? ( char -- ? )\r
- { [ "Noncharacter_Code_Point" property? ] [ category "Cs" = ] } 1|| ;\r
-\r
-: derive-weight ( char -- weights )\r
- first dup illegal?\r
- [ drop { } ]\r
- [ [ AAAA ] [ BBBB ] bi 2array ] if ;\r
-\r
-: building-last ( -- char )\r
- building get empty? [ 0 ] [ building get last last ] if ;\r
-\r
-: blocked? ( char -- ? )\r
- combining-class dup { 0 f } member?\r
- [ drop building-last non-starter? ]\r
- [ building-last combining-class = ] if ;\r
-\r
-: possible-bases ( -- slice-of-building )\r
- building get dup [ first non-starter? not ] find-last\r
- drop [ 0 ] unless* tail-slice ;\r
-\r
-:: ?combine ( char slice i -- ? )\r
- i slice nth char suffix :> str\r
- str ducet get-global key? dup\r
- [ str i slice set-nth ] when ;\r
-\r
-: add ( char -- )\r
- dup blocked? [ 1string , ] [\r
- dup possible-bases dup length iota\r
- [ ?combine ] 2with any?\r
- [ drop ] [ 1string , ] if\r
- ] if ;\r
-\r
-: string>graphemes ( string -- graphemes )\r
- [ [ add ] each ] { } make ;\r
-\r
-: graphemes>weights ( graphemes -- weights )\r
- [\r
- dup weight? [ 1array ] ! From tailoring\r
- [ dup ducet get-global at [ ] [ derive-weight ] ?if ] if\r
- ] { } map-as concat ;\r
-\r
-: append-weights ( weights quot -- )\r
- [ [ ignorable?>> ] reject ] dip\r
- map [ zero? ] reject % 0 , ; inline\r
-\r
-: variable-weight ( weight -- )\r
- dup ignorable?>> [ primary>> ] [ drop 0xFFFF ] if , ;\r
-\r
-: weights>bytes ( weights -- byte-array )\r
- [\r
- {\r
- [ [ primary>> ] append-weights ]\r
- [ [ secondary>> ] append-weights ]\r
- [ [ tertiary>> ] append-weights ]\r
- [ [ variable-weight ] each ]\r
- } cleave\r
- ] { } make ;\r
-PRIVATE>\r
-\r
-: completely-ignorable? ( weight -- ? )\r
- [ primary>> ] [ secondary>> ] [ tertiary>> ] tri\r
- [ zero? ] tri@ and and ;\r
-\r
-: filter-ignorable ( weights -- weights' )\r
- f swap [\r
- [ nip ] [ primary>> zero? and ] 2bi\r
- [ swap ignorable?>> or ]\r
- [ swap completely-ignorable? or not ] 2bi\r
- ] filter nip ;\r
-\r
-: collation-key ( string -- key )\r
- nfd string>graphemes graphemes>weights\r
- filter-ignorable weights>bytes ;\r
-\r
-<PRIVATE\r
-: insensitive= ( str1 str2 levels-removed -- ? )\r
- [\r
- [ collation-key ] dip\r
- [ [ 0 = not ] trim-tail but-last ] times\r
- ] curry same? ;\r
-PRIVATE>\r
-\r
-: primary= ( str1 str2 -- ? )\r
- 3 insensitive= ;\r
-\r
-: secondary= ( str1 str2 -- ? )\r
- 2 insensitive= ;\r
-\r
-: tertiary= ( str1 str2 -- ? )\r
- 1 insensitive= ;\r
-\r
-: quaternary= ( str1 str2 -- ? )\r
- 0 insensitive= ;\r
-\r
-: w/collation-key ( str -- {str,key} )\r
- [ collation-key ] keep 2array ;\r
-\r
-: sort-strings ( strings -- sorted )\r
- [ w/collation-key ] map natural-sort values ;\r
-\r
-: string<=> ( str1 str2 -- <=> )\r
- [ w/collation-key ] compare ;\r
+! Copyright (C) 2008 Daniel Ehrenberg.
+! See http://factorcode.org/license.txt for BSD license.
+USING: sequences io.files io.encodings.ascii kernel splitting
+accessors math.parser ascii io assocs strings math namespaces make
+sorting combinators math.order arrays unicode.normalize unicode.data
+locals macros sequences.deep words unicode.breaks quotations
+combinators.short-circuit simple-flat-file ;
+IN: unicode.collation
+
+<PRIVATE
+SYMBOL: ducet
+
+TUPLE: weight primary secondary tertiary ignorable? ;
+
+: parse-weight ( string -- weight )
+ "]" split but-last [
+ weight new swap rest unclip CHAR: * = swapd >>ignorable?
+ swap "." split first3 [ hex> ] tri@
+ [ >>primary ] [ >>secondary ] [ >>tertiary ] tri*
+ ] map ;
+
+: parse-keys ( string -- chars )
+ " " split [ hex> ] "" map-as ;
+
+: parse-ducet ( file -- ducet )
+ data [ [ parse-keys ] [ parse-weight ] bi* ] H{ } assoc-map-as ;
+
+"vocab:unicode/collation/allkeys.txt" parse-ducet ducet set-global
+
+! Fix up table for long contractions
+: help-one ( assoc key -- )
+ ! Need to be more general? Not for DUCET, apparently
+ 2 head 2dup swap key? [ 2drop ] [
+ [ [ 1string of ] with { } map-as concat ]
+ [ swap set-at ] 2bi
+ ] if ;
+
+: insert-helpers ( assoc -- )
+ dup keys [ length 3 >= ] filter
+ [ help-one ] with each ;
+
+ducet get-global insert-helpers
+
+: base ( char -- base )
+ {
+ { [ dup 0x3400 0x4DB5 between? ] [ drop 0xFB80 ] } ! Extension A
+ { [ dup 0x20000 0x2A6D6 between? ] [ drop 0xFB80 ] } ! Extension B
+ { [ dup 0x4E00 0x9FC3 between? ] [ drop 0xFB40 ] } ! CJK
+ [ drop 0xFBC0 ] ! Other
+ } cond ;
+
+: AAAA ( char -- weight )
+ [ base ] [ -15 shift ] bi + 0x20 2 f weight boa ;
+
+: BBBB ( char -- weight )
+ 0x7FFF bitand 0x8000 bitor 0 0 f weight boa ;
+
+: illegal? ( char -- ? )
+ { [ "Noncharacter_Code_Point" property? ] [ category "Cs" = ] } 1|| ;
+
+: derive-weight ( char -- weights )
+ first dup illegal?
+ [ drop { } ]
+ [ [ AAAA ] [ BBBB ] bi 2array ] if ;
+
+: building-last ( -- char )
+ building get empty? [ 0 ] [ building get last last ] if ;
+
+: blocked? ( char -- ? )
+ combining-class dup { 0 f } member?
+ [ drop building-last non-starter? ]
+ [ building-last combining-class = ] if ;
+
+: possible-bases ( -- slice-of-building )
+ building get dup [ first non-starter? not ] find-last
+ drop [ 0 ] unless* tail-slice ;
+
+:: ?combine ( char slice i -- ? )
+ i slice nth char suffix :> str
+ str ducet get-global key? dup
+ [ str i slice set-nth ] when ;
+
+: add ( char -- )
+ dup blocked? [ 1string , ] [
+ dup possible-bases dup length iota
+ [ ?combine ] 2with any?
+ [ drop ] [ 1string , ] if
+ ] if ;
+
+: string>graphemes ( string -- graphemes )
+ [ [ add ] each ] { } make ;
+
+: graphemes>weights ( graphemes -- weights )
+ [
+ dup weight? [ 1array ] ! From tailoring
+ [ dup ducet get-global at [ ] [ derive-weight ] ?if ] if
+ ] { } map-as concat ;
+
+: append-weights ( weights quot -- )
+ [ [ ignorable?>> ] reject ] dip
+ map [ zero? ] reject % 0 , ; inline
+
+: variable-weight ( weight -- )
+ dup ignorable?>> [ primary>> ] [ drop 0xFFFF ] if , ;
+
+: weights>bytes ( weights -- byte-array )
+ [
+ {
+ [ [ primary>> ] append-weights ]
+ [ [ secondary>> ] append-weights ]
+ [ [ tertiary>> ] append-weights ]
+ [ [ variable-weight ] each ]
+ } cleave
+ ] { } make ;
+PRIVATE>
+
+: completely-ignorable? ( weight -- ? )
+ [ primary>> ] [ secondary>> ] [ tertiary>> ] tri
+ [ zero? ] tri@ and and ;
+
+: filter-ignorable ( weights -- weights' )
+ f swap [
+ [ nip ] [ primary>> zero? and ] 2bi
+ [ swap ignorable?>> or ]
+ [ swap completely-ignorable? or not ] 2bi
+ ] filter nip ;
+
+: collation-key ( string -- key )
+ nfd string>graphemes graphemes>weights
+ filter-ignorable weights>bytes ;
+
+<PRIVATE
+: insensitive= ( str1 str2 levels-removed -- ? )
+ [
+ [ collation-key ] dip
+ [ [ 0 = not ] trim-tail but-last ] times
+ ] curry same? ;
+PRIVATE>
+
+: primary= ( str1 str2 -- ? )
+ 3 insensitive= ;
+
+: secondary= ( str1 str2 -- ? )
+ 2 insensitive= ;
+
+: tertiary= ( str1 str2 -- ? )
+ 1 insensitive= ;
+
+: quaternary= ( str1 str2 -- ? )
+ 0 insensitive= ;
+
+: w/collation-key ( str -- {str,key} )
+ [ collation-key ] keep 2array ;
+
+: sort-strings ( strings -- sorted )
+ [ w/collation-key ] map natural-sort values ;
+
+: string<=> ( str1 str2 -- <=> )
+ [ w/collation-key ] compare ;
projects / factor.git / blobdiff