work on docs

diff --git a/rust/pspp/src/lex/mod.rs b/rust/pspp/src/lex/mod.rs
index 2acbe0dfff3f5165577a10507825a57f874eb240..34bd92de50f411483dd5b0a692387a31df3a7409 100644 (file)
--- a/rust/pspp/src/lex/mod.rs
+++ b/rust/pspp/src/lex/mod.rs
@@ -14,12 +14,12 @@
 // You should have received a copy of the GNU General Public License along with
 // this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//! PSPP syntax scanning.
+//! # PSPP tokenization
 //!
 //! PSPP divides traditional "lexical analysis" or "tokenization" into two
 //! phases: a lower-level phase called "segmentation" and a higher-level phase
-//! called "scanning".  [segment] implements the segmentation phase and
-//! this module the scanning phase.
+//! called "scanning".  The [segment] module implements the segmentation phase
+//! and the [scan] module the scanning phase.
 //!
 //! Scanning accepts as input a stream of segments, which are UTF-8 strings each
 //! labeled with a segment type.  It outputs a stream of "scan tokens", which

diff --git a/rust/pspp/src/lex/scan/mod.rs b/rust/pspp/src/lex/scan/mod.rs
index b08af3d6e183e87ba154293effdb73ce72dc745e..c15b9fb3f7402f3aa69b7b3b8967f9056faaf334 100644 (file)
--- a/rust/pspp/src/lex/scan/mod.rs
+++ b/rust/pspp/src/lex/scan/mod.rs
@@ -14,17 +14,15 @@
 // You should have received a copy of the GNU General Public License along with
 // this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//! PSPP lexical analysis.
+//! # High-level lexical analysis
 //!
-//! PSPP divides traditional "lexical analysis" or "tokenization" into two
-//! phases: a lower-level phase called "segmentation" and a higher-level phase
-//! called "scanning".  [mod segment] implements the segmentation phase and [mod
-//! scan] the scanning phase.
+//! This module implements higher-level lexical analysis using the segments
+//! output by the lower-level [segmentation phase](super::segment).
 //!
-//! Scanning accepts as input a stream of segments, which are UTF-8 strings each
-//! labeled with a segment type.  It outputs a stream of "scan tokens", which
-//! are the same as the tokens used by the PSPP parser with a few additional
-//! types.
+//! Scanning accepts as input a stream of segments, which are UTF-8 strings
+//! labeled with a [segment type](super::segment::Segment).  It outputs a stream
+//! of [ScanToken]s, which are either the [Token] used by the PSPP parser or an
+//! error.
 
 use crate::identifier::{Identifier, ReservedWord};
 

diff --git a/rust/pspp/src/lex/segment/mod.rs b/rust/pspp/src/lex/segment/mod.rs
index e51ab5a3fb419d53c457a4784cd71a025f75dfdf..21ab7a96d404241dace6fa63546ec3699d0edf84 100644 (file)
--- a/rust/pspp/src/lex/segment/mod.rs
+++ b/rust/pspp/src/lex/segment/mod.rs
@@ -14,12 +14,12 @@
 // You should have received a copy of the GNU General Public License along with
 // this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Syntax segmentation.
+//! # Low-level lexical analysis
 //!
 //! PSPP divides traditional "lexical analysis" or "tokenization" into two
 //! phases: a lower-level phase called "segmentation" and a higher-level phase
-//! called "scanning".  This module implements the segmentation phase.
-//! [`super::scan`] contains declarations for the scanning phase.
+//! called "scanning".  This module implements the segmentation phase; the
+//! [scan](super::scan) module implements the higher-level scanning phase.
 //!
 //! Segmentation accepts a stream of UTF-8 bytes as input.  It outputs a label
 //! (a segment type) for each byte or contiguous sequence of bytes in the input.