From 68332a6e7e600f3f8bcfbd4baf8b2afa87f3abcc Mon Sep 17 00:00:00 2001 From: John Benediktsson Date: Fri, 21 Jan 2022 13:16:36 -0800 Subject: [PATCH] io.encodings.utf16n: merge with io.encodings.utf16 --- basis/bootstrap/stage2.factor | 6 ++---- basis/core-text/core-text.factor | 2 +- basis/environment/windows/windows.factor | 2 +- basis/io/monitors/windows/windows.factor | 2 +- basis/tools/ps/windows/windows.factor | 2 +- basis/ui/backend/cocoa/views/views.factor | 2 +- basis/ui/backend/windows/windows.factor | 2 +- basis/ui/text/core-text/core-text.factor | 2 +- basis/windows/dropfiles/dropfiles.factor | 2 +- basis/windows/registry/registry-tests.factor | 2 +- basis/windows/registry/registry.factor | 2 +- basis/windows/types/types.factor | 2 +- basis/windows/uniscribe/uniscribe.factor | 2 +- core/alien/alien-docs.factor | 2 +- core/alien/strings/strings-tests.factor | 2 +- core/alien/strings/strings.factor | 2 +- core/io/encodings/utf16/utf16-docs.factor | 9 +++++++-- core/io/encodings/utf16/utf16-tests.factor | 11 ++++++++++- core/io/encodings/utf16/utf16.factor | 14 ++++++++++++-- core/io/encodings/utf16n/authors.txt | 1 - core/io/encodings/utf16n/summary.txt | 1 - core/io/encodings/utf16n/utf16n-docs.factor | 6 ------ core/io/encodings/utf16n/utf16n-tests.factor | 9 --------- core/io/encodings/utf16n/utf16n.factor | 15 --------------- 24 files changed, 46 insertions(+), 56 deletions(-) delete mode 100644 core/io/encodings/utf16n/authors.txt delete mode 100644 core/io/encodings/utf16n/summary.txt delete mode 100644 core/io/encodings/utf16n/utf16n-docs.factor delete mode 100644 core/io/encodings/utf16n/utf16n-tests.factor delete mode 100644 core/io/encodings/utf16n/utf16n.factor diff --git a/basis/bootstrap/stage2.factor b/basis/bootstrap/stage2.factor index 3e6e58a8bb..ee6c349272 100644 --- a/basis/bootstrap/stage2.factor +++ b/basis/bootstrap/stage2.factor @@ -13,10 +13,8 @@ SYMBOL: bootstrap-time : strip-encodings ( -- ) os unix? [ [ - P" resource:core/io/encodings/utf16/utf16.factor" - P" resource:core/io/encodings/utf16n/utf16n.factor" [ forget ] bi@ - "io.encodings.utf16" - "io.encodings.utf16n" [ loaded-child-vocab-names [ forget-vocab ] each ] bi@ + P" resource:core/io/encodings/utf16/utf16.factor" forget + "io.encodings.utf16" loaded-child-vocab-names [ forget-vocab ] each ] with-compilation-unit ] when ; diff --git a/basis/core-text/core-text.factor b/basis/core-text/core-text.factor index d94d850ab5..9b98890c80 100644 --- a/basis/core-text/core-text.factor +++ b/basis/core-text/core-text.factor @@ -4,7 +4,7 @@ USING: accessors alien.c-types alien.data alien.syntax arrays assocs cache classes colors combinators core-foundation core-foundation.attributed-strings core-foundation.strings core-graphics core-graphics.types core-text.fonts destructors -fonts init io.encodings.string io.encodings.utf16n kernel make +fonts init io.encodings.string io.encodings.utf16 kernel make math math.functions math.order math.vectors namespaces opengl sequences strings ; IN: core-text diff --git a/basis/environment/windows/windows.factor b/basis/environment/windows/windows.factor index 2ff126c887..4c0d00d2ca 100644 --- a/basis/environment/windows/windows.factor +++ b/basis/environment/windows/windows.factor @@ -1,6 +1,6 @@ ! Copyright (C) 2008 Doug Coleman. ! See http://factorcode.org/license.txt for BSD license. -USING: alien.strings fry io.encodings.utf16n kernel +USING: alien.strings fry io.encodings.utf16 kernel splitting windows windows.kernel32 windows.types system environment alien.data sequences windows.errors io.streams.memory io.encodings io specialized-arrays ; diff --git a/basis/io/monitors/windows/windows.factor b/basis/io/monitors/windows/windows.factor index 4c99ca18c8..958fa3ed3a 100644 --- a/basis/io/monitors/windows/windows.factor +++ b/basis/io/monitors/windows/windows.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. USING: accessors alien alien.data arrays classes.struct combinators continuations destructors fry io.backend -io.encodings.string io.encodings.utf16n io.files.windows +io.encodings.string io.encodings.utf16 io.files.windows io.monitors io.pathnames io.ports kernel literals locals make math sequences system threads windows.errors windows.kernel32 windows.types ; diff --git a/basis/tools/ps/windows/windows.factor b/basis/tools/ps/windows/windows.factor index a21d8644d2..b5bc8d15f0 100644 --- a/basis/tools/ps/windows/windows.factor +++ b/basis/tools/ps/windows/windows.factor @@ -1,7 +1,7 @@ USING: accessors alien alien.c-types alien.data alien.syntax arrays byte-arrays classes.struct combinators.short-circuit continuations destructors fry io io.encodings.string -io.encodings.utf16n kernel literals locals math sequences +io.encodings.utf16 kernel literals locals math sequences strings system tools.ps windows.errors windows.handles windows.kernel32 windows.ntdll windows.types ; IN: tools.ps.windows diff --git a/basis/ui/backend/cocoa/views/views.factor b/basis/ui/backend/cocoa/views/views.factor index dda758f11d..be51f635b7 100644 --- a/basis/ui/backend/cocoa/views/views.factor +++ b/basis/ui/backend/cocoa/views/views.factor @@ -6,7 +6,7 @@ arrays assocs classes cocoa cocoa.application cocoa.classes cocoa.pasteboard cocoa.runtime cocoa.subclassing cocoa.touchbar cocoa.types cocoa.views combinators continuations core-foundation.strings core-graphics core-graphics.types -core-text debugger io.encodings.string io.encodings.utf16n +core-text debugger io.encodings.string io.encodings.utf16 io.encodings.utf8 kernel literals math math.order math.parser math.rectangles math.vectors namespaces opengl sequences splitting threads ui.backend.cocoa.input-methods ui.commands diff --git a/basis/ui/backend/windows/windows.factor b/basis/ui/backend/windows/windows.factor index b379581df6..4281dd8394 100644 --- a/basis/ui/backend/windows/windows.factor +++ b/basis/ui/backend/windows/windows.factor @@ -4,7 +4,7 @@ USING: accessors alien alien.c-types alien.data alien.strings arrays ascii assocs assocs.extras byte-arrays calendar classes classes.struct colors combinators continuations io io.crlf -io.encodings.string io.encodings.utf16n io.encodings.utf8 kernel +io.encodings.string io.encodings.utf16 io.encodings.utf8 kernel libc literals make math math.bitwise namespaces sequences sets specialized-arrays strings threads ui ui.backend ui.clipboards ui.event-loop ui.gadgets ui.gadgets.private ui.gadgets.worlds diff --git a/basis/ui/text/core-text/core-text.factor b/basis/ui/text/core-text/core-text.factor index f8e93690b9..135d00970e 100644 --- a/basis/ui/text/core-text/core-text.factor +++ b/basis/ui/text/core-text/core-text.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2009, 2010 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. USING: accessors cache core-graphics.types core-text -core-text.fonts io.encodings.string io.encodings.utf16n kernel +core-text.fonts io.encodings.string io.encodings.utf16 kernel math math.vectors namespaces opengl sequences ui.text ui.text.private ; IN: ui.text.core-text diff --git a/basis/windows/dropfiles/dropfiles.factor b/basis/windows/dropfiles/dropfiles.factor index 194502c4eb..1c0e1dde48 100644 --- a/basis/windows/dropfiles/dropfiles.factor +++ b/basis/windows/dropfiles/dropfiles.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2017-2018 Alexander Ilin. ! See http://factorcode.org/license.txt for BSD license. USING: accessors alien alien.data alien.libraries alien.strings -init io.encodings.utf16n kernel literals math namespaces +init io.encodings.utf16 kernel literals math namespaces sequences ui.backend.windows ui.gadgets.worlds ui.gestures windows.errors windows.messages windows.shell32 windows.types windows.user32 ; diff --git a/basis/windows/registry/registry-tests.factor b/basis/windows/registry/registry-tests.factor index 839f2eecd3..cf58533774 100644 --- a/basis/windows/registry/registry-tests.factor +++ b/basis/windows/registry/registry-tests.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2010 Doug Coleman. ! Copyright (C) 2018 Alexander Ilin. ! See http://factorcode.org/license.txt for BSD license. -USING: byte-arrays io.encodings.string io.encodings.utf16n +USING: byte-arrays io.encodings.string io.encodings.utf16 kernel sequences tools.test windows.advapi32 windows.kernel32 windows.registry ; IN: windows.registry.tests diff --git a/basis/windows/registry/registry.factor b/basis/windows/registry/registry.factor index 21e326539f..8bb4d6d26e 100644 --- a/basis/windows/registry/registry.factor +++ b/basis/windows/registry/registry.factor @@ -3,7 +3,7 @@ ! See http://factorcode.org/license.txt for BSD license. USING: accessors alien.c-types alien.data byte-arrays classes.struct continuations io.encodings.string -io.encodings.utf16n kernel literals locals math sequences sets +io.encodings.utf16 kernel literals locals math sequences sets splitting windows windows.advapi32 windows.errors windows.kernel32 windows.time windows.types ; IN: windows.registry diff --git a/basis/windows/types/types.factor b/basis/windows/types/types.factor index e6fe062242..5827d5c9b9 100644 --- a/basis/windows/types/types.factor +++ b/basis/windows/types/types.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2005, 2006 Doug Coleman. ! See http://factorcode.org/license.txt for BSD license. USING: alien.c-types alien.syntax classes.struct colors -io.encodings.utf16n io.encodings.utf8 kernel math math.bitwise +io.encodings.utf16 io.encodings.utf8 kernel math math.bitwise math.functions math.vectors sequences ; FROM: alien.c-types => float short ; IN: windows.types diff --git a/basis/windows/uniscribe/uniscribe.factor b/basis/windows/uniscribe/uniscribe.factor index 08f33b0f17..326d17a9c0 100644 --- a/basis/windows/uniscribe/uniscribe.factor +++ b/basis/windows/uniscribe/uniscribe.factor @@ -3,7 +3,7 @@ USING: accessors alien.c-types alien.data arrays assocs byte-arrays cache classes.struct colors combinators destructors -fonts images init io.encodings.string io.encodings.utf16n kernel +fonts images init io.encodings.string io.encodings.utf16 kernel literals locals math math.bitwise math.functions namespaces sequences specialized-arrays windows.errors windows.fonts windows.gdi32 windows.offscreen windows.ole32 windows.types diff --git a/core/alien/alien-docs.factor b/core/alien/alien-docs.factor index 37a4efe278..3c83e17a06 100644 --- a/core/alien/alien-docs.factor +++ b/core/alien/alien-docs.factor @@ -1,6 +1,6 @@ USING: alien.accessors alien.c-types alien.libraries alien.strings alien.syntax byte-arrays cpu.x86 eval help.markup -help.syntax io io.backend io.encodings.utf16n io.encodings.utf8 +help.syntax io io.backend io.encodings.utf16 io.encodings.utf8 kernel math quotations sequences system ; IN: alien diff --git a/core/alien/strings/strings-tests.factor b/core/alien/strings/strings-tests.factor index a288a54d93..06781f1566 100644 --- a/core/alien/strings/strings-tests.factor +++ b/core/alien/strings/strings-tests.factor @@ -1,6 +1,6 @@ USING: alien alien.data alien.strings io.encodings.ascii io.encodings.latin1 io.encodings.string io.encodings.utf16 -io.encodings.utf16n io.encodings.utf8 kernel libc tools.test ; +io.encodings.utf8 kernel libc tools.test ; { "\u0000ff" } [ "\u0000ff" latin1 string>alien latin1 alien>string ] diff --git a/core/alien/strings/strings.factor b/core/alien/strings/strings.factor index a7ddb3dd80..60b7eaa6ce 100644 --- a/core/alien/strings/strings.factor +++ b/core/alien/strings/strings.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2008, 2011 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. USING: accessors alien arrays byte-arrays byte-vectors init io -io.encodings io.encodings.ascii io.encodings.utf16n +io.encodings io.encodings.ascii io.encodings.utf16 io.encodings.utf8 io.streams.memory kernel kernel.private math namespaces sequences sequences.private strings strings.private system system.private ; diff --git a/core/io/encodings/utf16/utf16-docs.factor b/core/io/encodings/utf16/utf16-docs.factor index 6b046f3e69..b6c8985388 100644 --- a/core/io/encodings/utf16/utf16-docs.factor +++ b/core/io/encodings/utf16/utf16-docs.factor @@ -4,11 +4,12 @@ USING: help.markup help.syntax ; IN: io.encodings.utf16 ARTICLE: "io.encodings.utf16" "UTF-16 encoding" -"The UTF-16 encoding is a variable-width encoding. Unicode code points are encoded as 2 or 4 byte sequences. There are three encoding descriptor classes for working with UTF-16, depending on endianness or the presence of a BOM:" +"The UTF-16 encoding is a variable-width encoding. Unicode code points are encoded as 2 or 4 byte sequences. There are four encoding descriptor classes for working with UTF-16, depending on endianness or the presence of a BOM:" { $subsections utf16 utf16le utf16be + utf16n } ; ABOUT: "io.encodings.utf16" @@ -25,4 +26,8 @@ HELP: utf16 { $class-description "The encoding descriptor for UTF-16, that is, UTF-16 with a byte order mark. This is the most useful for general input and output in UTF-16. Streams can be made which read or write wth this encoding." } { $see-also "encodings-introduction" } ; -{ utf16 utf16le utf16be } related-words +HELP: utf16n +{ $class-description "The encoding descriptor for UTF-16 without a byte order mark in native endian order. This is useful mostly for FFI calls which take input of strings of the type " { $snippet "wchar_t*" } "." } +{ $see-also "encodings-introduction" } ; + +{ utf16 utf16le utf16be utf16n } related-words diff --git a/core/io/encodings/utf16/utf16-tests.factor b/core/io/encodings/utf16/utf16-tests.factor index 0b704532d2..71a16c63b2 100644 --- a/core/io/encodings/utf16/utf16-tests.factor +++ b/core/io/encodings/utf16/utf16-tests.factor @@ -1,7 +1,9 @@ ! Copyright (C) 2008 Daniel Ehrenberg. ! See http://factorcode.org/license.txt for BSD license. -USING: arrays io.encodings.string io.encodings.utf16 strings +USING: accessors alien.data arrays io.encodings.string +io.encodings.utf16 io.streams.byte-array kernel strings tools.test ; +IN: io.encodings.utf16.tests { { CHAR: x } } [ B{ 0 CHAR: x } utf16be decode >array ] unit-test { { 0x1D11E } } [ B{ 0xD8 0x34 0xDD 0x1E } utf16be decode >array ] unit-test @@ -26,3 +28,10 @@ tools.test ; { B{ CHAR: a 0 CHAR: b 0 CHAR: c 0 } } [ "abc" utf16le encode ] unit-test { B{ 0 CHAR: a 0 CHAR: b 0 CHAR: c } } [ "abc" utf16be encode ] unit-test + +: correct-endian ( obj -- ? ) + code>> little-endian? utf16le utf16be ? = ; + +{ t } [ B{ } utf16n correct-endian ] unit-test +{ t } [ utf16n correct-endian ] unit-test + diff --git a/core/io/encodings/utf16/utf16.factor b/core/io/encodings/utf16/utf16.factor index 2777649aa9..01fc02c1bb 100644 --- a/core/io/encodings/utf16/utf16.factor +++ b/core/io/encodings/utf16/utf16.factor @@ -1,7 +1,8 @@ ! Copyright (C) 2006, 2009 Daniel Ehrenberg. ! See http://factorcode.org/license.txt for BSD license. -USING: accessors byte-arrays io io.encodings kernel math -math.private sequences sequences.private strings strings.private ; +USING: accessors alien.accessors byte-arrays io io.encodings +kernel math math.private sequences sequences.private strings +strings.private ; IN: io.encodings.utf16 SINGLETON: utf16be @@ -163,3 +164,12 @@ M: utf16 drop bom-le over stream-write utf16le ; PRIVATE> + +SINGLETON: utf16n + +: choose-utf16-endian ( -- descriptor ) + B{ 1 0 0 0 } 0 alien-unsigned-4 1 = utf16le utf16be ? ; foldable + +M: utf16n drop choose-utf16-endian ; + +M: utf16n drop choose-utf16-endian ; diff --git a/core/io/encodings/utf16n/authors.txt b/core/io/encodings/utf16n/authors.txt deleted file mode 100644 index f990dd0ed2..0000000000 --- a/core/io/encodings/utf16n/authors.txt +++ /dev/null @@ -1 +0,0 @@ -Daniel Ehrenberg diff --git a/core/io/encodings/utf16n/summary.txt b/core/io/encodings/utf16n/summary.txt deleted file mode 100644 index 4d94d1bbdb..0000000000 --- a/core/io/encodings/utf16n/summary.txt +++ /dev/null @@ -1 +0,0 @@ -UTF16 encoding with native byte order diff --git a/core/io/encodings/utf16n/utf16n-docs.factor b/core/io/encodings/utf16n/utf16n-docs.factor deleted file mode 100644 index 9ccf4834d4..0000000000 --- a/core/io/encodings/utf16n/utf16n-docs.factor +++ /dev/null @@ -1,6 +0,0 @@ -USING: help.markup help.syntax ; -IN: io.encodings.utf16n - -HELP: utf16n -{ $class-description "The encoding descriptor for UTF-16 without a byte order mark in native endian order. This is useful mostly for FFI calls which take input of strings of the type wchar_t*" } -{ $see-also "encodings-introduction" } ; diff --git a/core/io/encodings/utf16n/utf16n-tests.factor b/core/io/encodings/utf16n/utf16n-tests.factor deleted file mode 100644 index c76e090ae9..0000000000 --- a/core/io/encodings/utf16n/utf16n-tests.factor +++ /dev/null @@ -1,9 +0,0 @@ -USING: accessors alien.data io.encodings.utf16 -io.encodings.utf16n io.streams.byte-array kernel tools.test ; -IN: io.encodings.utf16n.tests - -: correct-endian ( obj -- ? ) - code>> little-endian? [ utf16le = ] [ utf16be = ] if ; - -{ t } [ B{ } utf16n correct-endian ] unit-test -{ t } [ utf16n correct-endian ] unit-test diff --git a/core/io/encodings/utf16n/utf16n.factor b/core/io/encodings/utf16n/utf16n.factor deleted file mode 100644 index 6e178c1774..0000000000 --- a/core/io/encodings/utf16n/utf16n.factor +++ /dev/null @@ -1,15 +0,0 @@ -! Copyright (C) 2008 Slava Pestov. -! See http://factorcode.org/license.txt for BSD license. -USING: io.encodings io.encodings.utf16 kernel alien.accessors ; -IN: io.encodings.utf16n - -! Native-order UTF-16 - -SINGLETON: utf16n - -: choose-utf16-endian ( -- descriptor ) - B{ 1 0 0 0 } 0 alien-unsigned-4 1 = utf16le utf16be ? ; foldable - -M: utf16n drop choose-utf16-endian ; - -M: utf16n drop choose-utf16-endian ; -- 2.34.1