/* Generate Unicode conforming character classification tables and
line break properties tables and word break property tables and
decomposition/composition and case mapping tables from a UnicodeData file.
- Copyright (C) 2000-2002, 2004, 2007-2010 Free Software Foundation, Inc.
+ Copyright (C) 2000-2002, 2004, 2007-2011 Free Software Foundation, Inc.
Written by Bruno Haible <bruno@clisp.org>, 2000-2002.
This program is free software: you can redistribute it and/or modify
};
/* Returns the line breaking classification for ch, as a bit mask. */
-static int
+static int64_t
get_lbp (unsigned int ch)
{
- int attr = 0;
+ int64_t attr = 0;
if (unicode_attributes[ch].name != NULL)
{
|| ch == 0x000B /* line tabulation */
|| ch == 0x2028 /* LINE SEPARATOR */
|| ch == 0x2029 /* PARAGRAPH SEPARATOR */)
- attr |= 1 << LBP_BK;
+ attr |= (int64_t) 1 << LBP_BK;
if (ch == 0x2060 /* WORD JOINER */
|| ch == 0xFEFF /* ZERO WIDTH NO-BREAK SPACE */)
- attr |= 1 << LBP_WJ;
+ attr |= (int64_t) 1 << LBP_WJ;
/* zero width space */
if (ch == 0x200B /* ZERO WIDTH SPACE */)
- attr |= 1 << LBP_ZW;
+ attr |= (int64_t) 1 << LBP_ZW;
/* non-breaking (glue) */
if (ch == 0x00A0 /* NO-BREAK SPACE */
|| ch == 0x0F0C /* TIBETAN MARK DELIMITER TSHEG BSTAR */
|| ch == 0x0F12 /* TIBETAN MARK RGYA GRAM SHAD */
|| (ch >= 0x035C && ch <= 0x0362) /* COMBINING DOUBLE ... */)
- attr |= 1 << LBP_GL;
+ attr |= (int64_t) 1 << LBP_GL;
/* space */
if (ch == 0x0020 /* SPACE */)
- attr |= 1 << LBP_SP;
+ attr |= (int64_t) 1 << LBP_SP;
/* break opportunity before and after */
if (ch == 0x2014 /* EM DASH */)
- attr |= 1 << LBP_B2;
+ attr |= (int64_t) 1 << LBP_B2;
/* break opportunity after */
if (ch == 0x1680 /* OGHAM SPACE MARK */
|| ch == 0x12471 /* CUNEIFORM PUNCTUATION SIGN VERTICAL COLON */
|| ch == 0x12472 /* CUNEIFORM PUNCTUATION SIGN DIAGONAL COLON */
|| ch == 0x12473 /* CUNEIFORM PUNCTUATION SIGN DIAGONAL TRICOLON */)
- attr |= 1 << LBP_BA;
+ attr |= (int64_t) 1 << LBP_BA;
/* break opportunity before */
if (ch == 0x00B4 /* ACUTE ACCENT */
|| ch == 0xA874 /* PHAGS-PA SINGLE HEAD MARK */
|| ch == 0xA875 /* PHAGS-PA DOUBLE HEAD MARK */
|| ch == 0x1806 /* MONGOLIAN TODO SOFT HYPHEN */)
- attr |= 1 << LBP_BB;
+ attr |= (int64_t) 1 << LBP_BB;
/* hyphen */
if (ch == 0x002D /* HYPHEN-MINUS */)
- attr |= 1 << LBP_HY;
+ attr |= (int64_t) 1 << LBP_HY;
/* contingent break opportunity */
if (ch == 0xFFFC /* OBJECT REPLACEMENT CHARACTER */)
- attr |= 1 << LBP_CB;
+ attr |= (int64_t) 1 << LBP_CB;
/* closing punctuation */
if ((unicode_attributes[ch].category[0] == 'P'
|| ch == 0xFF0E /* FULLWIDTH FULL STOP */
|| ch == 0xFF61 /* HALFWIDTH IDEOGRAPHIC FULL STOP */
|| ch == 0xFF64 /* HALFWIDTH IDEOGRAPHIC COMMA */)
- attr |= 1 << LBP_CL;
+ attr |= (int64_t) 1 << LBP_CL;
/* exclamation/interrogation */
if (ch == 0x0021 /* EXCLAMATION MARK */
|| ch == 0x2CF9 /* COPTIC OLD NUBIAN FULL STOP */
|| ch == 0x2CFE /* COPTIC FULL STOP */
|| ch == 0x2E2E /* REVERSED QUESTION MARK */
-#if REVISION_22
- || ch == 0xA60C /* VAI SYLLABLE LENGTHENER */
-#endif
|| ch == 0xA60E /* VAI FULL STOP */
|| ch == 0xA876 /* PHAGS-PA MARK SHAD */
|| ch == 0xA877 /* PHAGS-PA MARK DOUBLE SHAD */
|| ch == 0xFE57 /* SMALL EXCLAMATION MARK */
|| ch == 0xFF01 /* FULLWIDTH EXCLAMATION MARK */
|| ch == 0xFF1F /* FULLWIDTH QUESTION MARK */)
- attr |= 1 << LBP_EX;
+ attr |= (int64_t) 1 << LBP_EX;
/* inseparable */
if (ch == 0x2024 /* ONE DOT LEADER */
|| ch == 0x2025 /* TWO DOT LEADER */
|| ch == 0x2026 /* HORIZONTAL ELLIPSIS */
|| ch == 0xFE19 /* PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS */)
- attr |= 1 << LBP_IN;
+ attr |= (int64_t) 1 << LBP_IN;
/* non starter */
if (ch == 0x17D6 /* KHMER SIGN CAMNUC PII KUUH */
|| ch == 0xFF9F /* HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK */
|| strstr (unicode_attributes[ch].name, "HIRAGANA LETTER SMALL ") != NULL
|| strstr (unicode_attributes[ch].name, "KATAKANA LETTER SMALL ") != NULL)
- attr |= 1 << LBP_NS;
+ attr |= (int64_t) 1 << LBP_NS;
/* opening punctuation */
if ((unicode_attributes[ch].category[0] == 'P'
|| ch == 0x00A1 /* INVERTED EXCLAMATION MARK */
|| ch == 0x00BF /* INVERTED QUESTION MARK */
|| ch == 0x2E18 /* INVERTED INTERROBANG */)
- attr |= 1 << LBP_OP;
+ attr |= (int64_t) 1 << LBP_OP;
/* ambiguous quotation */
if ((unicode_attributes[ch].category[0] == 'P'
|| ch == 0x2E07 /* RAISED DOTTED INTERPOLATION MARKER */
|| ch == 0x2E08 /* DOTTED TRANSPOSITION MARKER */
|| ch == 0x2E0B /* RAISED SQUARE */)
- attr |= 1 << LBP_QU;
+ attr |= (int64_t) 1 << LBP_QU;
/* infix separator (numeric) */
if (ch == 0x002C /* COMMA */
|| ch == 0xFE10 /* PRESENTATION FORM FOR VERTICAL COMMA */
|| ch == 0xFE13 /* PRESENTATION FORM FOR VERTICAL COLON */
|| ch == 0xFE14 /* PRESENTATION FORM FOR VERTICAL SEMICOLON */)
- attr |= 1 << LBP_IS;
+ attr |= (int64_t) 1 << LBP_IS;
/* numeric */
if ((unicode_attributes[ch].category[0] == 'N'
&& strstr (unicode_attributes[ch].name, "FULLWIDTH") == NULL)
|| ch == 0x066B /* ARABIC DECIMAL SEPARATOR */
|| ch == 0x066C /* ARABIC THOUSANDS SEPARATOR */)
- attr |= 1 << LBP_NU;
+ attr |= (int64_t) 1 << LBP_NU;
/* postfix (numeric) */
if (ch == 0x0025 /* PERCENT SIGN */
|| ch == 0x0609 /* ARABIC-INDIC PER MILLE SIGN */
|| ch == 0x060A /* ARABIC-INDIC PER TEN THOUSAND SIGN */
|| ch == 0x0D79 /* MALAYALAM DATE MARK */)
- attr |= 1 << LBP_PO;
+ attr |= (int64_t) 1 << LBP_PO;
/* prefix (numeric) */
if ((unicode_attributes[ch].category[0] == 'S'
|| ch == 0x2116 /* NUMERO SIGN */
|| ch == 0x2212 /* MINUS SIGN */
|| ch == 0x2213 /* MINUS-OR-PLUS SIGN */)
- if (!(attr & (1 << LBP_PO)))
- attr |= 1 << LBP_PR;
+ if (!(attr & ((int64_t) 1 << LBP_PO)))
+ attr |= (int64_t) 1 << LBP_PR;
/* symbols allowing breaks */
if (ch == 0x002F /* SOLIDUS */)
- attr |= 1 << LBP_SY;
+ attr |= (int64_t) 1 << LBP_SY;
if (ch >= 0xAC00 && ch <= 0xD7A3 && ((ch - 0xAC00) % 28) == 0)
- attr |= 1 << LBP_H2;
+ attr |= (int64_t) 1 << LBP_H2;
if (ch >= 0xAC00 && ch <= 0xD7A3 && ((ch - 0xAC00) % 28) != 0)
- attr |= 1 << LBP_H3;
+ attr |= (int64_t) 1 << LBP_H3;
if ((ch >= 0x1100 && ch <= 0x1159) || ch == 0x115F)
- attr |= 1 << LBP_JL;
+ attr |= (int64_t) 1 << LBP_JL;
if (ch >= 0x1160 && ch <= 0x11A2)
- attr |= 1 << LBP_JV;
+ attr |= (int64_t) 1 << LBP_JV;
if (ch >= 0x11A8 && ch <= 0x11F9)
- attr |= 1 << LBP_JT;
+ attr |= (int64_t) 1 << LBP_JT;
/* complex context (South East Asian) */
if (((unicode_attributes[ch].category[0] == 'C'
|| (ch >= 0x1000 && ch <= 0x109F)
|| (ch >= 0x1780 && ch <= 0x17FF)
|| (ch >= 0x1950 && ch <= 0x19DF)))
- attr |= 1 << LBP_SA;
+ attr |= (int64_t) 1 << LBP_SA;
/* attached characters and combining marks */
if ((unicode_attributes[ch].category[0] == 'M'
|| (unicode_attributes[ch].category[0] == 'C'
&& (unicode_attributes[ch].category[1] == 'c'
|| unicode_attributes[ch].category[1] == 'f')))
- if (!(attr & ((1 << LBP_BK) | (1 << LBP_BA) | (1 << LBP_GL) | (1 << LBP_SA) | (1 << LBP_WJ) | (1 << LBP_ZW))))
- attr |= 1 << LBP_CM;
+ if (!(attr & (((int64_t) 1 << LBP_BK) | ((int64_t) 1 << LBP_BA) | ((int64_t) 1 << LBP_GL) | ((int64_t) 1 << LBP_SA) | ((int64_t) 1 << LBP_WJ) | ((int64_t) 1 << LBP_ZW))))
+ attr |= (int64_t) 1 << LBP_CM;
/* ideographic */
if ((ch >= 0x2E80 && ch <= 0x2FFF) /* CJK RADICAL, KANGXI RADICAL, IDEOGRAPHIC DESCRIPTION */
|| (ch >= 0x2F800 && ch <= 0x2FA1D) /* CJK COMPATIBILITY IDEOGRAPH */
|| strstr (unicode_attributes[ch].name, "FULLWIDTH LATIN ") != NULL
|| (ch >= 0x3000 && ch <= 0x33FF
- && !(attr & ((1 << LBP_CM) | (1 << LBP_NS) | (1 << LBP_OP) | (1 << LBP_CL))))
+ && !(attr & (((int64_t) 1 << LBP_CM) | ((int64_t) 1 << LBP_NS) | ((int64_t) 1 << LBP_OP) | ((int64_t) 1 << LBP_CL))))
/* Extra characters for compatibility with Unicode LineBreak.txt. */
|| ch == 0xFE30 /* PRESENTATION FORM FOR VERTICAL TWO DOT LEADER */
|| ch == 0xFE31 /* PRESENTATION FORM FOR VERTICAL EM DASH */
|| ch == 0xFFE2 /* FULLWIDTH NOT SIGN */
|| ch == 0xFFE3 /* FULLWIDTH MACRON */
|| ch == 0xFFE4 /* FULLWIDTH BROKEN BAR */)
- if (!(attr & ((1 << LBP_NS) | (1 << LBP_CM))))
+ if (!(attr & (((int64_t) 1 << LBP_NS) | ((int64_t) 1 << LBP_CM))))
{
/* ambiguous (ideograph) ? */
if ((unicode_width[ch] != NULL
&& ch >= 0x2000)
|| ch == 0x24EA /* CIRCLED DIGIT ZERO */
|| (ch >= 0x2780 && ch <= 0x2793) /* DINGBAT ... CIRCLED DIGIT ... */)
- attr |= 1 << LBP_AI;
+ attr |= (int64_t) 1 << LBP_AI;
else
- attr |= 1 << LBP_ID;
+ attr |= (int64_t) 1 << LBP_ID;
}
/* ordinary alphabetic and symbol characters */
|| ch == 0x2062 /* INVISIBLE TIMES */
|| ch == 0x2063 /* INVISIBLE SEPARATOR */
|| ch == 0x2064 /* INVISIBLE PLUS */)
- if (!(attr & ((1 << LBP_GL) | (1 << LBP_B2) | (1 << LBP_BA) | (1 << LBP_BB) | (1 << LBP_HY) | (1 << LBP_CB) | (1 << LBP_CL) | (1 << LBP_EX) | (1 << LBP_IN) | (1 << LBP_NS) | (1 << LBP_OP) | (1 << LBP_QU) | (1 << LBP_IS) | (1 << LBP_NU) | (1 << LBP_PO) | (1 << LBP_PR) | (1 << LBP_SY) | (1 << LBP_H2) | (1 << LBP_H3) | (1 << LBP_JL) | (1 << LBP_JV) | (1 << LBP_JT) | (1 << LBP_SA) | (1 << LBP_ID))))
+ if (!(attr & (((int64_t) 1 << LBP_GL) | ((int64_t) 1 << LBP_B2) | ((int64_t) 1 << LBP_BA) | ((int64_t) 1 << LBP_BB) | ((int64_t) 1 << LBP_HY) | ((int64_t) 1 << LBP_CB) | ((int64_t) 1 << LBP_CL) | ((int64_t) 1 << LBP_EX) | ((int64_t) 1 << LBP_IN) | ((int64_t) 1 << LBP_NS) | ((int64_t) 1 << LBP_OP) | ((int64_t) 1 << LBP_QU) | ((int64_t) 1 << LBP_IS) | ((int64_t) 1 << LBP_NU) | ((int64_t) 1 << LBP_PO) | ((int64_t) 1 << LBP_PR) | ((int64_t) 1 << LBP_SY) | ((int64_t) 1 << LBP_H2) | ((int64_t) 1 << LBP_H3) | ((int64_t) 1 << LBP_JL) | ((int64_t) 1 << LBP_JV) | ((int64_t) 1 << LBP_JT) | ((int64_t) 1 << LBP_SA) | ((int64_t) 1 << LBP_ID))))
{
/* ambiguous (alphabetic) ? */
if ((unicode_width[ch] != NULL
&& ch != 0x21E7 /* UPWARDS WHITE ARROW */
&& ch != 0x24FF /* NEGATIVE CIRCLED DIGIT ZERO */
&& ch != 0x273D /* HEAVY TEARDROP-SPOKED ASTERISK */)
-#if !REVISION_22
- || ch == 0x00A1 /* INVERTED EXCLAMATION MARK */
|| ch == 0x00A7 /* SECTION SIGN */
|| ch == 0x00A8 /* DIAERESIS */
|| ch == 0x00AA /* FEMININE ORDINAL INDICATOR */
|| ch == 0x00BC /* VULGAR FRACTION ONE QUARTER */
|| ch == 0x00BD /* VULGAR FRACTION ONE HALF */
|| ch == 0x00BE /* VULGAR FRACTION THREE QUARTERS */
- || ch == 0x00BF /* INVERTED QUESTION MARK */
|| ch == 0x00D7 /* MULTIPLICATION SIGN */
|| ch == 0x00F7 /* DIVISION SIGN */
|| ch == 0x02C7 /* CARON */
|| ch == 0x02DA /* RING ABOVE */
|| ch == 0x02DB /* OGONEK */
|| ch == 0x02DD /* DOUBLE ACUTE ACCENT */
-#endif
|| ch == 0x24EA /* CIRCLED DIGIT ZERO */
|| (ch >= 0x2780 && ch <= 0x2793) /* DINGBAT ... CIRCLED DIGIT ... */
/* Extra characters for compatibility with Unicode LineBreak.txt. */
|| ch == 0x2574 /* BOX DRAWINGS LIGHT LEFT */
|| ch == 0x2616 /* WHITE SHOGI PIECE */
|| ch == 0x2617 /* BLACK SHOGI PIECE */)
- attr |= 1 << LBP_AI;
+ attr |= (int64_t) 1 << LBP_AI;
else
- attr |= 1 << LBP_AL;
- attr &= ~(1 << LBP_CM);
+ attr |= (int64_t) 1 << LBP_AL;
+ attr &= ~((int64_t) 1 << LBP_CM);
}
}
if (attr == 0)
/* unknown */
- attr |= 1 << LBP_XX;
+ attr |= (int64_t) 1 << LBP_XX;
return attr;
}
for (i = 0; i < 0x110000; i++)
{
- int attr = get_lbp (i);
- if (attr != 1 << LBP_XX)
+ int64_t attr = get_lbp (i);
+ if (attr != (int64_t) 1 << LBP_XX)
{
fprintf (stream, "0x%04X", i);
#define PRINT_BIT(attr,bit) \
for (i = 0; i < 0x110000; i++)
{
- int attr = get_lbp (i);
+ int64_t attr = get_lbp (i);
/* Now attr should contain exactly one bit. */
if (attr == 0 || ((attr & (attr - 1)) != 0))
abort ();
- if (attr != 1 << LBP_XX)
+ if (attr != (int64_t) 1 << LBP_XX)
{
unsigned int log2_attr;
for (log2_attr = 0; attr > 1; attr >>= 1, log2_attr++);
/* The grapheme break property from the GraphemeBreakProperty.txt file. */
int unicode_org_gbp[0x110000];
+/* Output the unit test data for the grapheme break property. */
+static void
+output_gbp_test (const char *filename)
+{
+ FILE *stream;
+ bool need_comma;
+ unsigned int ch;
+
+ stream = fopen (filename, "w");
+ if (stream == NULL)
+ {
+ fprintf (stderr, "cannot open '%s' for writing\n", filename);
+ exit (1);
+ }
+
+ fprintf (stream, "/* DO NOT EDIT! GENERATED AUTOMATICALLY! */\n");
+ fprintf (stream, "/* Test the Unicode grapheme break property functions.\n");
+ fprintf (stream, " Copyright (C) 2010 Free Software Foundation, Inc.\n");
+ fprintf (stream, "\n");
+ fprintf (stream, " This program is free software: you can redistribute it and/or modify\n");
+ fprintf (stream, " it under the terms of the GNU General Public License as published by\n");
+ fprintf (stream, " the Free Software Foundation; either version 3 of the License, or\n");
+ fprintf (stream, " (at your option) any later version.\n");
+ fprintf (stream, "\n");
+ fprintf (stream, " This program is distributed in the hope that it will be useful,\n");
+ fprintf (stream, " but WITHOUT ANY WARRANTY; without even the implied warranty of\n");
+ fprintf (stream, " MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n");
+ fprintf (stream, " GNU General Public License for more details.\n");
+ fprintf (stream, "\n");
+ fprintf (stream, " You should have received a copy of the GNU General Public License\n");
+ fprintf (stream, " along with this program. If not, see <http://www.gnu.org/licenses/>. */\n");
+ fprintf (stream, "\n");
+
+ need_comma = false;
+ for (ch = 0; ch < 0x110000; ch++)
+ {
+ int gbp = unicode_org_gbp[ch];
+ const char *gbp_string;
+
+ while (ch + 1 < 0x110000 && unicode_org_gbp[ch + 1] == gbp)
+ ch++;
+
+ switch (gbp)
+ {
+#define CASE(x) case x: gbp_string = #x; break;
+ CASE (GBP_OTHER)
+ CASE (GBP_CR)
+ CASE (GBP_LF)
+ CASE (GBP_CONTROL)
+ CASE (GBP_EXTEND)
+ CASE (GBP_PREPEND)
+ CASE (GBP_SPACINGMARK)
+ CASE (GBP_L)
+ CASE (GBP_V)
+ CASE (GBP_T)
+ CASE (GBP_LV)
+ CASE (GBP_LVT)
+#undef CASE
+ default:
+ abort ();
+ }
+
+ if (need_comma)
+ fprintf (stream, ",\n");
+ fprintf (stream, "{ 0x%04X, %s }", ch + 1, gbp_string);
+
+ need_comma = true;
+ }
+ fprintf (stream, "\n");
+
+ if (ferror (stream) || fclose (stream))
+ {
+ fprintf (stderr, "error writing to '%s'\n", filename);
+ exit (1);
+ }
+}
+
/* Output the per-character grapheme break property table. */
static void
output_gbp_table (const char *filename, const char *version)
debug_output_org_wbrk_tables ("uniwbrk/wbrkprop_org.txt");
output_wbrk_tables ("uniwbrk/wbrkprop.h", version);
+ output_gbp_test ("../tests/unigbrk/test-uc-gbrk-prop.h");
output_gbp_table ("unigbrk/gbrkprop.h", version);
output_decomposition_tables ("uninorm/decomposition-table1.h", "uninorm/decomposition-table2.h", version);
/gfs/petix/Volumes/ExtData/www-archive/software/i18n/unicode/ftp.unicode.org/ArchiveVersions/5.1.0/ucd/CompositionExclusions.txt \
/gfs/petix/Volumes/ExtData/www-archive/software/i18n/unicode/ftp.unicode.org/ArchiveVersions/5.1.0/ucd/SpecialCasing.txt \
/gfs/petix/Volumes/ExtData/www-archive/software/i18n/unicode/ftp.unicode.org/ArchiveVersions/5.1.0/ucd/CaseFolding.txt \
- 5.1.0
+ 5.1.0 \
+ && diff unilbrk/lbrkprop_org.txt unilbrk/lbrkprop.txt \
+ && diff uniwbrk/wbrkprop_org.txt uniwbrk/wbrkprop.txt
"
* End:
*/
projects / pspp / blobdiff