core/io/encodings/utf8/utf8-tests.factor

   1 USING: arrays io.encodings.string io.encodings.utf8 kernel
   2 sequences strings tools.test ;
   3 IN: io.encodings.utf8.tests
   4
   5 : decode-utf8-w/stream ( array -- newarray )
   6     utf8 decode >array ;
   7
   8 : encode-utf8-w/stream ( array -- newarray )
   9     >string utf8 encode >array ;
  10
  11 { { CHAR: replacement-character } } [ { 0b11110,101 0b10,111111 0b10,000000 0b11111111 } decode-utf8-w/stream ] unit-test
  12
  13 { "x" } [ "x" decode-utf8-w/stream >string ] unit-test
  14
  15 { { 0b11111000000 } } [ { 0b110,11111 0b10,000000 } decode-utf8-w/stream >array ] unit-test
  16
  17 { { CHAR: replacement-character } } [ { 0b10000000 } decode-utf8-w/stream ] unit-test
  18
  19 { { 0b1111000000111111 } } [ { 0b1110,1111 0b10,000000 0b10,111111 } decode-utf8-w/stream >array ] unit-test
  20
  21 { { 0b11110,101 0b10,111111 0b10,000000 0b10,111111 0b1110,1111 0b10,000000 0b10,111111 0b110,11111 0b10,000000 CHAR: x } }
  22 [ { 0b101111111000000111111 0b1111000000111111 0b11111000000 CHAR: x } encode-utf8-w/stream ] unit-test
  23
  24 { 3 } [ 1 "日本語" >utf8-index ] unit-test
  25 { 3 } [ 9 "日本語" utf8-index> ] unit-test
  26
  27 { 3 } [ 2 "lápis" >utf8-index ] unit-test
  28
  29 { V{ } } [ 100000 iota [ [ code-point-length ] [ 1string utf8 encode length ] bi = ] reject ] unit-test
  30
  31 { { CHAR: replacement-character } } [ { 0b110,00000 0b10,000000 } decode-utf8-w/stream ] unit-test
  32 { { CHAR: replacement-character } } [ { 0b110,00001 0b10,111111 } decode-utf8-w/stream ] unit-test
  33 { { 0x80 } } [ { 0b110,00010 0b10,000000 } decode-utf8-w/stream ] unit-test
  34
  35 { { CHAR: replacement-character } } [ { 0b1110,0000 0b10,000000 0b10,000000 } decode-utf8-w/stream ] unit-test
  36 { { CHAR: replacement-character } } [ { 0b1110,0000 0b10,011111 0b10,111111 } decode-utf8-w/stream ] unit-test
  37 { { 0x800 } } [ { 0b1110,0000 0b10,100000 0b10,000000 } decode-utf8-w/stream ] unit-test
  38
  39 { { CHAR: replacement-character } } [ { 0b11110,000 0b10,000000 0b10,000000 0b10,000000 } decode-utf8-w/stream ] unit-test
  40 { { CHAR: replacement-character } } [ { 0b11110,000 0b10,001111 0b10,111111 0b10,111111 } decode-utf8-w/stream ] unit-test
  41 { { CHAR: replacement-character } } [ { 0b11110,100 0b10,010000 0b10,000000 0b10,000000 } decode-utf8-w/stream ] unit-test
  42 { { 0x10000 } } [ { 0b11110,000 0b10,010000 0b10,000000 0b10,000000 } decode-utf8-w/stream ] unit-test
  43 { { 0x10FFFF } } [ { 0b11110,100 0b10,001111 0b10,111111 0b10,111111 } decode-utf8-w/stream ] unit-test