/* PSPP - a program for statistical analysis.
- Copyright (C) 2006, 2010, 2011 Free Software Foundation, Inc.
+ Copyright (C) 2006, 2010, 2011, 2012, 2014, 2016 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#ifndef I18N_H
#define I18N_H
+/*
+
+ PSPP has three ``working'' locales:
+
+ * The user interface locale.
+
+ This is the locale which is visible to the person using pspp. Error
+ messages and confidence indications are written in this locale. For
+ example ``Cannot open file'' will be written in the user interface locale.
+
+ This locale is set from the environment of the user who starts PSPP or from
+ the system locale if not set.
+
+ * The output locale.
+
+ This locale should be visible to the person reading a report generated by
+ pspp. Non-data related strings (e.g., "Page number", "Standard Deviation"
+ etc.) appear in this locale.
+
+ * The data locale.
+
+ Only the character encoding is relevant.
+
+ This locale is the one associated with the data being analysed. The only
+ important aspect of this locale is the character encoding. (It might also
+ be desirable for the LC_COLLATE category to be used for the purposes of
+ sorting data.) The dictionary pertaining to the data contains a field
+ denoting the encoding. Any string data stored in a "union value" is
+ encoded in the dictionary's character set.
+
+ Each of these locales may, at different times take separate (or identical)
+ values. So for example, a French statistician can use pspp to prepare a
+ report in the English language, using a datafile which has been created by a
+ Japanese researcher hence uses a Japanese character set.
+
+ It's rarely, if ever, necessary to interrogate the system to find out the
+ values of the 3 locales. However it's important to be aware of the source
+ (destination) locale when reading (writing) string data. When transferring
+ data between a source and a destination, the appropriate recoding must be
+ performed.
+
+ System Files
+ ============
+
+ '.sav' files contain a field which is supposed to identify the encoding of
+ the data they contain. However, many files produced by early versions of
+ spss set this to "2" (ASCII) regardless of the encoding of the data. Later
+ versions contain an additional record (the "Character Encoding Record")
+ describing the encoding. When a system file is read, the dictionary's
+ encoding is set using information gleaned from the system file. If the
+ encoding cannot be determined or would be unreliable, then it remains unset.
+
+ GUI
+ ===
+
+ The psppire graphic user interface is written using the GTK+ api, for which
+ all strings must be encoded in UTF-8. All strings passed to the GTK+/GLib
+ library functions (except for filenames) must be UTF-8 encoded otherwise
+ errors will occur. Thus, for the purposes of programming PSPPIRE, the user
+ interface locale should be assumed to be UTF-8, even if setlocale() and/or
+ nl_langinfo indicates otherwise.
+
+ Filenames
+ ---------
+
+ The GLib API has some special functions for dealing with filenames. Strings
+ returned from functions like gtk_file_chooser_dialog_get_name() are not, in
+ general, encoded in UTF-8, but in "filename" encoding. If that filename is
+ passed to another GLib function which expects a filename, no conversion is
+ necessary. If it's passed to a function for the purposes of displaying it
+ (e.g. in a window's title-bar) it must be converted to UTF-8 (there is a
+ special function for this: g_filename_display_name or g_filename_basename).
+ If however, a filename needs to be passed outside of GTK+/GLib, e.g. to
+ fopen, it must be converted to the local system encoding.
+
+ Existing Locale Handling Functions
+ ==================================
+
+ The major aspect of locale handling which the programmer has to consider is
+ that of character encoding. recode_string() is the main function for
+ changing the encoding of strings.
+
+ To minimise the number of conversions required, and to simplify design, PSPP
+ attempts to store all internal strings in UTF-8 encoding. Thus, when reading
+ system and portable files (or any other data source), the following items are
+ immediately converted to UTF-8
+
+ * Variable names
+ * Variable labels
+ * Value labels
+
+ Conversely, when writing system files, these are converted back to the
+ encoding of that system file.
+
+ String data stored in "union value"s are left in their original encoding.
+ These are converted for display later by data_out().
+
+ Quirks
+ ======
+
+ For historical reasons, not all locale handling follows POSIX conventions.
+ This makes it difficult (impossible?) to elegantly handle issues. For
+ example, it would make sense for the GUI's datasheet to display numbers
+ formatted according to LC_NUMERIC. Instead however there is data_out(),
+ which uses settings_get_decimal_char() function instead of the locale's
+ decimal separator. Similarly, formatting of monetary values is displayed in
+ a PSPP/SPSS-specific fashion instead of using LC_MONETARY.
+*/
+
+#include "libpspp/compiler.h"
+#include "libpspp/str.h"
#include <stdbool.h>
#include <unistr.h>
const char *text, int length, struct pool *);
struct substring recode_substring_pool (const char *to, const char *from,
struct substring text, struct pool *);
+int recode_pedantically (const char *to, const char *from,
+ struct substring text, struct pool *,
+ struct substring *out);
size_t recode_string_len (const char *to, const char *from,
const char *text, int len);
size_t utf8_encoding_concat_len (const char *head, const char *tail,
const char *encoding, size_t max_len);
+size_t utf8_count_columns (const char *, size_t);
+size_t utf8_columns_to_bytes (const char *, size_t, size_t n_columns);
+
char *utf8_to_filename (const char *filename);
char *filename_to_utf8 (const char *filename);
bool set_encoding_from_locale (const char *loc);
const char *uc_name (ucs4_t uc, char buffer[16]);
+
+unsigned int utf8_hash_case_bytes (const char *, size_t n, unsigned int basis) WARN_UNUSED_RESULT;
+unsigned int utf8_hash_case_string (const char *, unsigned int basis) WARN_UNUSED_RESULT;
+unsigned int utf8_hash_case_substring (struct substring, unsigned int basis)
+ WARN_UNUSED_RESULT;
+int utf8_strcasecmp (const char *, const char *);
+int utf8_sscasecmp (struct substring, struct substring);
+int utf8_strncasecmp (const char *, size_t, const char *, size_t);
+int utf8_strverscasecmp (const char *, const char *);
+char *utf8_to_upper (const char *);
+char *utf8_to_lower (const char *);
+char *utf8_to_title (const char *);
\f
/* Information about character encodings. */
a b c d e f g h i j k l m
n o p q r s t u v w x y z
0 1 2 3 4 5 6 7 8 9
- ! " # % & ' ( ) * + , - . / :
+ ! " # % & ' () * + , - . / :
; < = > ? [ \ ] ^ _ { | } ~
space \a \b \r \n \t \v \f \0
used in ASCII text files has the same value in this encoding. */
bool is_ascii_compatible;
+ /* True if this encoding has a unit width of 1 byte and appears to be
+ EBCDIC-based. */
+ bool is_ebcdic_compatible;
+
/* Character information. */
int unit; /* Unit width, in bytes. */
char cr[MAX_UNIT]; /* \r in encoding, 'unit' bytes long. */
char lf[MAX_UNIT]; /* \n in encoding, 'unit' bytes long. */
+ char space[MAX_UNIT]; /* ' ' in encoding, 'unit' bytes long. */
};
bool get_encoding_info (struct encoding_info *, const char *name);
bool is_encoding_ascii_compatible (const char *encoding);
+bool is_encoding_ebcdic_compatible (const char *encoding);
bool is_encoding_supported (const char *encoding);
+bool is_encoding_utf8 (const char *encoding);
+\f
+/* Database of encodings, by language or region. */
+
+struct encoding_category
+ {
+ const char *category; /* e.g. "Arabic" or "Western European". */
+ const char **encodings; /* Encodings within the category. */
+ size_t n_encodings; /* Number of encodings in category. */
+ };
+
+struct encoding_category *get_encoding_categories (void);
+size_t get_n_encoding_categories (void);
+
+/* Return the ISO two letter code for the current LC_MESSAGES
+ locale category. */
+char *get_language (void);
+
#endif /* i18n.h */