1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2006, 2010, 2011, 2012, 2014, 2016 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
20 #include "libpspp/compiler.h"
21 #include "libpspp/str.h"
25 void i18n_done (void);
26 void i18n_init (void);
30 /* The encoding of literal strings in PSPP source code, as seen at execution
31 time. In fact this is likely to be some extended ASCII encoding, such as
32 UTF-8 or ISO-8859-1, but ASCII is adequate for our purposes. */
33 #define C_ENCODING "ASCII"
37 char recode_byte (const char *to, const char *from, char);
39 char *recode_string (const char *to, const char *from,
40 const char *text, int len);
41 char *recode_string_pool (const char *to, const char *from,
42 const char *text, int length, struct pool *);
43 struct substring recode_substring_pool (const char *to, const char *from,
44 struct substring text, struct pool *);
45 int recode_pedantically (const char *to, const char *from,
46 struct substring text, struct pool *,
47 struct substring *out);
49 size_t recode_string_len (const char *to, const char *from,
50 const char *text, int len);
52 char *utf8_encoding_trunc (const char *, const char *encoding,
54 size_t utf8_encoding_trunc_len (const char *, const char *encoding,
57 char *utf8_encoding_concat (const char *head, const char *tail,
58 const char *encoding, size_t max_len);
59 size_t utf8_encoding_concat_len (const char *head, const char *tail,
60 const char *encoding, size_t max_len);
62 size_t utf8_count_columns (const char *, size_t);
63 size_t utf8_columns_to_bytes (const char *, size_t, size_t n_columns);
65 char *utf8_to_filename (const char *filename);
66 char *filename_to_utf8 (const char *filename);
68 bool valid_encoding (const char *enc);
70 char get_system_decimal (void);
72 const char * get_default_encoding (void);
73 void set_default_encoding (const char *enc);
75 bool set_encoding_from_locale (const char *loc);
77 const char *uc_name (ucs4_t uc, char buffer[16]);
79 unsigned int utf8_hash_case_bytes (const char *, size_t n, unsigned int basis) WARN_UNUSED_RESULT;
80 unsigned int utf8_hash_case_string (const char *, unsigned int basis) WARN_UNUSED_RESULT;
81 unsigned int utf8_hash_case_substring (struct substring, unsigned int basis)
83 int utf8_strcasecmp (const char *, const char *);
84 int utf8_sscasecmp (struct substring, struct substring);
85 int utf8_strncasecmp (const char *, size_t, const char *, size_t);
86 int utf8_strverscasecmp (const char *, const char *);
87 char *utf8_to_upper (const char *);
88 char *utf8_to_lower (const char *);
89 char *utf8_to_title (const char *);
91 /* Information about character encodings. */
93 /* ISO C defines a set of characters that a C implementation must support at
94 runtime, called the C basic execution character set, which consists of the
97 A B C D E F G H I J K L M
98 N O P Q R S T U V W X Y Z
99 a b c d e f g h i j k l m
100 n o p q r s t u v w x y z
102 ! " # % & ' () * + , - . / :
103 ; < = > ? [ \ ] ^ _ { | } ~
104 space \a \b \r \n \t \v \f \0
106 The following is true of every member of the C basic execution character
107 set in all "reasonable" encodings:
109 1. Every member of the C basic character set is encoded.
111 2. Every member of the C basic character set has the same width in
112 bytes, called the "unit width". Most encodings have a unit width of
113 1 byte, but UCS-2 and UTF-16 have a unit width of 2 bytes and UCS-4
114 and UTF-32 have a unit width of 4 bytes.
116 3. In a stateful encoding, the encoding of members of the C basic
117 character set does not vary with shift state.
119 4. When a string is read unit-by-unit, a unit that has the encoded value
120 of a member of the C basic character set, EXCEPT FOR THE DECIMAL
121 DIGITS, always represents that member. That is, if the encoding has
122 multi-unit characters, the units that encode the C basic character
123 set are never part of a multi-unit character.
125 The exception for decimal digits is due to GB18030, which uses
126 decimal digits as part of multi-byte encodings.
128 All 8-bit and wider encodings that I have been able to find follow these
129 rules. 7-bit and narrower encodings (e.g. UTF-7) do not. I'm not too
130 concerned about that. */
134 /* Maximum width of a unit, in bytes. UTF-32 with 4-byte units is the widest
135 that I am aware of. */
138 /* Information about an encoding. */
141 /* Encoding name. IANA says character set names may be up to 40 US-ASCII
145 /* True if this encoding has a unit width of 1 byte, and every character
146 used in ASCII text files has the same value in this encoding. */
147 bool is_ascii_compatible;
149 /* True if this encoding has a unit width of 1 byte and appears to be
151 bool is_ebcdic_compatible;
153 /* Character information. */
154 int unit; /* Unit width, in bytes. */
155 char cr[MAX_UNIT]; /* \r in encoding, 'unit' bytes long. */
156 char lf[MAX_UNIT]; /* \n in encoding, 'unit' bytes long. */
157 char space[MAX_UNIT]; /* ' ' in encoding, 'unit' bytes long. */
160 bool get_encoding_info (struct encoding_info *, const char *name);
161 bool is_encoding_ascii_compatible (const char *encoding);
162 bool is_encoding_ebcdic_compatible (const char *encoding);
163 bool is_encoding_supported (const char *encoding);
165 bool is_encoding_utf8 (const char *encoding);
167 /* Database of encodings, by language or region. */
169 struct encoding_category
171 const char *category; /* e.g. "Arabic" or "Western European". */
172 const char **encodings; /* Encodings within the category. */
173 size_t n_encodings; /* Number of encodings in category. */
176 struct encoding_category *get_encoding_categories (void);
177 size_t get_n_encoding_categories (void);
179 /* Return the ISO two letter code for the current LC_MESSAGES
181 char *get_language (void);