u8-istream: New library for reading a text file and recoding to UTF-8.

[pspp] / tests / libpspp / u8-istream.at

diff --git a/tests/libpspp/u8-istream.at b/tests/libpspp/u8-istream.at
new file mode 100644 (file)
index 0000000..2d8baa4
--- /dev/null
+++ b/tests/libpspp/u8-istream.at
@@ -0,0 +1,142 @@
+AT_BANNER([u8_istream])
+
+AT_SETUP([read ASCII])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([supports_encodings ASCII])
+AT_CHECK([echo string | u8-istream-test read - ASCII], [0], [string
+])
+AT_CLEANUP
+
+AT_SETUP([read UTF-8])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([printf '\346\227\245\346\234\254\350\252\236\n' | u8-istream-test read - UTF-8], [0], [dnl
+UTF-8 mode
+日本語
+UTF-8 mode
+])
+AT_CLEANUP
+
+AT_SETUP([read EUC-JP])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([supports_encodings EUC-JP])
+AT_CHECK([printf '\244\241 \244\242 \244\243 \244\244 \244\245 \244\246 \244\247 \244\250 \244\251 \244\252\n' | u8-istream-test read - EUC-JP],
+  [0],
+  [ぁ あ ぃ い ぅ う ぇ え ぉ お
+])
+AT_CLEANUP
+
+AT_SETUP([read UTF-8 with character split across input buffers])
+AT_KEYWORDS([u8_istream])
+buffer_size=`u8-istream-test buffer-size`
+($PERL -e "print 'x' x ($buffer_size - 16)"
+ printf '\343\201\201\343\201\202\343\201\203\343\201\204\343\201\205\343\201\206\343\201\207\343\201\210\343\201\211\343\201\212\n') > input
+(echo "UTF-8 mode"
+ cat input
+ echo "UTF-8 mode") > expout
+AT_CHECK([u8-istream-test read input UTF-8 16], [0], [expout])
+AT_CLEANUP
+
+AT_SETUP([read UTF-8 with character split across output buffers])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([printf '\343\201\201\343\201\202\343\201\203\343\201\204\343\201\205\343\201\206\343\201\207\343\201\210\343\201\211\343\201\212\n' | u8-istream-test read - UTF-8 16], [0], [dnl
+UTF-8 mode
+ぁあぃいぅうぇえぉお
+UTF-8 mode
+])
+AT_CLEANUP
+
+AT_SETUP([read UTF-8 with character split across input and output buffers])
+AT_KEYWORDS([u8_istream])
+buffer_size=`u8-istream-test buffer-size`
+($PERL -e "print 'x' x ($buffer_size - 16)"
+ printf '\343\201\201\343\201\202\343\201\203\343\201\204\343\201\205\343\201\206\343\201\207\343\201\210\343\201\211\343\201\212\n') > input
+(echo "UTF-8 mode"
+ cat input
+ echo "UTF-8 mode") > expout
+AT_CHECK([u8-istream-test read input UTF-8 16], [0], [expout])
+AT_CLEANUP
+
+AT_SETUP([read EUC-JP with character split across input buffers])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([supports_encodings EUC-JP])
+buffer_size=`u8-istream-test buffer-size`
+($PERL -e "print 'x' x ($buffer_size - 16)"
+ printf '\244\241 \244\242 \244\243 \244\244 \244\245 \244\246 \244\247 '
+ printf '\244\250 \244\251 \244\252\n') > input
+($PERL -e "print 'x' x ($buffer_size - 16)"
+ printf '\343\201\201\040\343\201\202\040\343\201\203\040\343\201\204\040'
+ printf '\343\201\205\040\343\201\206\040\343\201\207\040\343\201\210\040'
+ printf '\343\201\211\040\343\201\212\n') > expout
+AT_CHECK([u8-istream-test read input EUC-JP], [0], [expout])
+AT_CLEANUP
+
+AT_SETUP([read EUC-JP with character split across output buffers])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([supports_encodings EUC-JP])
+AT_CHECK([printf '\244\241\244\242\244\243\244\244\244\245\244\246\244\247\244\250\244\251\244\252\n' | u8-istream-test read - EUC-JP 16],
+  [0],
+  [ぁあぃいぅうぇえぉお
+])
+AT_CLEANUP
+
+AT_SETUP([read EUC-JP with character split across input and output buffers])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([supports_encodings EUC-JP])
+buffer_size=`u8-istream-test buffer-size`
+($PERL -e "print 'x' x ($buffer_size - 16)"
+ printf 'xyz\244\241\244\242\244\243\244\244\244\245\244\246\244\247\244\250'
+ printf '\244\251\244\252\n') > input
+($PERL -e "print 'x' x ($buffer_size - 16)"
+ printf '\170\171\172\343\201\201\343\201\202\343\201\203\343\201\204\343'
+ printf '\201\205\343\201\206\343\201\207\343\201\210\343\201\211\343\201'
+ printf '\212\n') > expout
+AT_CHECK([u8-istream-test read input EUC-JP 16], [0], [expout])
+AT_CLEANUP
+
+AT_SETUP([read ASCII as Auto])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([echo string | u8-istream-test read - Auto], [0], [dnl
+Auto mode
+string
+Auto mode
+])
+AT_CLEANUP
+
+AT_SETUP([read UTF-8 as Auto])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([printf 'entr\303\251e\n' | u8-istream-test read - Auto], [0], [dnl
+Auto mode
+entrée
+UTF-8 mode
+])
+AT_CLEANUP
+
+AT_SETUP([read ISO-8859-1 as Auto,ISO-8859-1])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([supports_encodings ISO-8859-1])
+buffer_size=`u8-istream-test buffer-size`
+($PERL -e "print 'x' x int($buffer_size * 2.5)"; printf 'entr\351e\n') > input
+(echo "Auto mode"
+ $PERL -e "print 'x' x int($buffer_size * 2.5)"
+ printf 'entr\303\251e\n') > expout
+AT_CHECK([u8-istream-test read input Auto,ISO-8859-1], [0], [expout])
+AT_CLEANUP
+
+dnl UTF-16BE is not ASCII compatible so this doesn't start out in Auto mode.
+AT_SETUP([read UTF-16BE as Auto,UTF-16BE])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([supports_encodings UTF-16BE])
+AT_CHECK([printf '\0e\0n\0t\0r\0\351\0e\0\n' | u8-istream-test read - Auto,UTF-16BE], 
+  [0], [dnl
+entrée
+])
+AT_CLEANUP
+
+AT_SETUP([read EUC-JP as Auto,EUC-JP])
+AT_KEYWORDS([u8_istream])
+AT_CHECK([supports_encodings EUC-JP])
+AT_CHECK([printf 'entr\217\253\261e\n' | u8-istream-test read - Auto,EUC-JP], 
+  [0], [entrée
+])
+AT_CLEANUP
+