/* PSPP - a program for statistical analysis.
- Copyright (C) 2010, 2011 Free Software Foundation, Inc.
+ Copyright (C) 2010, 2011, 2013 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include "libpspp/cast.h"
#include "gl/c-ctype.h"
+#include "gl/c-strtod.h"
#include "gl/xmemdup0.h"
enum
case '<': return T_LT;
case '>': return T_GT;
case '~': return T_NOT;
+ default: return T_MACRO_PUNCT;
}
NOT_REACHED ();
else
p = xmemdup0 (s.string, s.length);
- number = strtod (p, NULL);
+ number = c_strtod (p, NULL);
if (p != buf)
free (p);
#undef SCAN_TYPE
default:
- return token_type_to_name (type);
+ return token_type_to_name ((enum token_type) type);
}
}
case SEG_DO_REPEAT_COMMAND:
case SEG_INLINE_DATA:
case SEG_DOCUMENT:
+ case SEG_MACRO_BODY:
token->type = T_STRING;
ss_alloc_substring (&token->string, s);
return SCAN_DONE;
ss_alloc_substring (&token->string, s);
return SCAN_DONE;
+ case SEG_MACRO_ID:
+ token->type = T_MACRO_ID;
+ ss_alloc_substring (&token->string, s);
+ return SCAN_DONE;
+
case SEG_PUNCT:
if (s.length == 1 && s.string[0] == '-')
{
else
{
token->type = scan_punct__ (s);
+ if (token->type == T_MACRO_PUNCT)
+ ss_alloc_substring (&token->string, s);
return SCAN_DONE;
}
ss_alloc_substring (&token->string, s);
return SCAN_DONE;
- case SEG_UNEXPECTED_DOT:
- token->type = SCAN_UNEXPECTED_DOT;
- return SCAN_DONE;
-
case SEG_UNEXPECTED_CHAR:
return scan_unexpected_char (&s, token);
-
- case SEG_N_TYPES:
- NOT_REACHED ();
}
NOT_REACHED ();
scanner_init (struct scanner *scanner, struct token *token)
{
scanner->state = S_START;
- token_init (token);
+ *token = (struct token) { .type = T_STOP };
}
/* Adds the segment with type TYPE and UTF-8 text S to SCANNER. TOKEN must be
NOT_REACHED ();
}
+\f
+/* Initializes SLEX for parsing INPUT, which is LENGTH bytes long, in the
+ specified MODE.
+
+ SLEX has no internal state to free, but it retains a reference to INPUT, so
+ INPUT must not be modified or freed while SLEX is still in use. */
+void
+string_lexer_init (struct string_lexer *slex, const char *input, size_t length,
+ enum segmenter_mode mode)
+{
+ *slex = (struct string_lexer) {
+ .input = input,
+ .length = length,
+ .offset = 0,
+ .segmenter = segmenter_init (mode, true),
+ };
+}
+
+/* */
+bool
+string_lexer_next (struct string_lexer *slex, struct token *token)
+{
+ struct segmenter saved_segmenter;
+ size_t saved_offset = 0;
+
+ struct scanner scanner;
+
+ scanner_init (&scanner, token);
+ for (;;)
+ {
+ const char *s = slex->input + slex->offset;
+ size_t left = slex->length - slex->offset;
+ enum segment_type type;
+ int n;
+
+ n = segmenter_push (&slex->segmenter, s, left, true, &type);
+ assert (n >= 0);
+
+ slex->offset += n;
+ switch (scanner_push (&scanner, type, ss_buffer (s, n), token))
+ {
+ case SCAN_BACK:
+ slex->segmenter = saved_segmenter;
+ slex->offset = saved_offset;
+ /* Fall through. */
+ case SCAN_DONE:
+ return token->type != T_STOP;
+
+ case SCAN_MORE:
+ break;
+
+ case SCAN_SAVE:
+ saved_segmenter = slex->segmenter;
+ saved_offset = slex->offset;
+ break;
+ }
+ }
+}