From 30495b8d5d776f3a10f878ca55b213d863e72631 Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Fri, 11 Jul 2025 17:05:30 -0700 Subject: [PATCH] work on docs --- rust/pspp/src/lex/mod.rs | 6 +++--- rust/pspp/src/lex/scan/mod.rs | 16 +++++++--------- rust/pspp/src/lex/segment/mod.rs | 6 +++--- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/rust/pspp/src/lex/mod.rs b/rust/pspp/src/lex/mod.rs index 2acbe0dfff..34bd92de50 100644 --- a/rust/pspp/src/lex/mod.rs +++ b/rust/pspp/src/lex/mod.rs @@ -14,12 +14,12 @@ // You should have received a copy of the GNU General Public License along with // this program. If not, see . -//! PSPP syntax scanning. +//! # PSPP tokenization //! //! PSPP divides traditional "lexical analysis" or "tokenization" into two //! phases: a lower-level phase called "segmentation" and a higher-level phase -//! called "scanning". [segment] implements the segmentation phase and -//! this module the scanning phase. +//! called "scanning". The [segment] module implements the segmentation phase +//! and the [scan] module the scanning phase. //! //! Scanning accepts as input a stream of segments, which are UTF-8 strings each //! labeled with a segment type. It outputs a stream of "scan tokens", which diff --git a/rust/pspp/src/lex/scan/mod.rs b/rust/pspp/src/lex/scan/mod.rs index b08af3d6e1..c15b9fb3f7 100644 --- a/rust/pspp/src/lex/scan/mod.rs +++ b/rust/pspp/src/lex/scan/mod.rs @@ -14,17 +14,15 @@ // You should have received a copy of the GNU General Public License along with // this program. If not, see . -//! PSPP lexical analysis. +//! # High-level lexical analysis //! -//! PSPP divides traditional "lexical analysis" or "tokenization" into two -//! phases: a lower-level phase called "segmentation" and a higher-level phase -//! called "scanning". [mod segment] implements the segmentation phase and [mod -//! scan] the scanning phase. +//! This module implements higher-level lexical analysis using the segments +//! output by the lower-level [segmentation phase](super::segment). //! -//! Scanning accepts as input a stream of segments, which are UTF-8 strings each -//! labeled with a segment type. It outputs a stream of "scan tokens", which -//! are the same as the tokens used by the PSPP parser with a few additional -//! types. +//! Scanning accepts as input a stream of segments, which are UTF-8 strings +//! labeled with a [segment type](super::segment::Segment). It outputs a stream +//! of [ScanToken]s, which are either the [Token] used by the PSPP parser or an +//! error. use crate::identifier::{Identifier, ReservedWord}; diff --git a/rust/pspp/src/lex/segment/mod.rs b/rust/pspp/src/lex/segment/mod.rs index e51ab5a3fb..21ab7a96d4 100644 --- a/rust/pspp/src/lex/segment/mod.rs +++ b/rust/pspp/src/lex/segment/mod.rs @@ -14,12 +14,12 @@ // You should have received a copy of the GNU General Public License along with // this program. If not, see . -//! Syntax segmentation. +//! # Low-level lexical analysis //! //! PSPP divides traditional "lexical analysis" or "tokenization" into two //! phases: a lower-level phase called "segmentation" and a higher-level phase -//! called "scanning". This module implements the segmentation phase. -//! [`super::scan`] contains declarations for the scanning phase. +//! called "scanning". This module implements the segmentation phase; the +//! [scan](super::scan) module implements the higher-level scanning phase. //! //! Segmentation accepts a stream of UTF-8 bytes as input. It outputs a label //! (a segment type) for each byte or contiguous sequence of bytes in the input. -- 2.30.2