sys-file-reader: Break multiple response set decoding into two stages.

[pspp] / tests / data / sys-file-reader.at

diff --git a/tests/data/sys-file-reader.at b/tests/data/sys-file-reader.at
index 4b7a2b541fc306e0b903670085932916b859bbd4..c0530acc9929441cd685b89982e045f44bed953c 100644 (file)
--- a/tests/data/sys-file-reader.at
+++ b/tests/data/sys-file-reader.at
@@ -140,11 +140,11 @@ File label: PSPP synthetic test file: ôõöø
 
 Variable,Description,,Position
 num1,Format: F8.0,,1
-num2,Numeric variable 2's label (ùúû),,2
+num2,Label: Numeric variable 2's label (ùúû),,2
 ,Format: F8.0,,
 num3,Format: F8.0,,3
 ,Missing Values: 1,,
-num4,Another numeric variable label,,4
+num4,Label: Another numeric variable label,,4
 ,Format: F8.0,,
 ,Missing Values: 1,,
 num5,Format: F8.0,,5
@@ -160,11 +160,11 @@ num9,Format: F8.0,,9
 numàèìñò,Format: F8.0,,10
 ,Missing Values: LOWEST THRU 1; 5,,
 str1,Format: A4,,11
-str2,String variable 2's label,,12
+str2,Label: String variable 2's label,,12
 ,Format: A4,,
 str3,Format: A4,,13
 ,"Missing Values: ""MISS""",,
-str4,Another string variable label,,14
+str4,Label: Another string variable label,,14
 ,Format: A4,,
 ,"Missing Values: ""OTHR""",,
 str5,Format: A4,,15
@@ -179,7 +179,7 @@ str9,Format: A10,,19
 ,"Missing Values: ""abcdefgh""; ""01234567""",,
 str10,Format: A11,,20
 ,"Missing Values: ""abcdefgh""; ""01234567""; ""0       """,,
-str11,25-byte string,,21
+str11,Label: 25-byte string,,21
 ,Format: A25,,
 
 Table: Data List
@@ -232,7 +232,7 @@ LIST.
   AT_CHECK([grep -v Measure pspp.csv | grep -v Display], [0], [dnl
 Variable,Description,,Position
 num1,Format: F8.0,,1
-num2,Numeric variable 2's label,,2
+num2,Label: Numeric variable 2's label,,2
 ,Format: F8.0,,
 
 Table: Data List
@@ -288,7 +288,7 @@ LIST.
   AT_CHECK([grep -v Measure pspp.csv | grep -v Display], [0], [dnl
 Variable,Description,,Position
 num1,Format: F8.0,,1
-num2,Numeric variable 2's label,,2
+num2,Label: Numeric variable 2's label,,2
 ,Format: F8.0,,
 
 Table: Data List
@@ -599,13 +599,14 @@ dnl Machine integer info record.
 7; 7; 1;
 COUNT(
   "$a=C 10 my mcgroup "; i8 0x82; i8 0xa0; " b c"; i8 10;
-  "$b=D2 55 0  g e f d"; i8 10;
+  "$b=D2 55 0  g e f d"; i8 10; i8 10;
   "$c=D4 "; i8 0x82; i8 0xcd; i8 0x82; i8 0xa2; " 10 mdgroup #2 h i j"; i8 10);
 
 7; 19; 1;
 COUNT(
+  i8 10;
   "$d=E 1 2 34 13 third mdgroup k l m"; i8 10;
-  "$e=E 11 6 choice 0  n o p"; i8 10);
+  "$e=E 11 6 choice 0  n o p"; i8 10; i8 10; i8 10; i8 10);
 
 dnl Character encoding record.
 7; 20; 1; 9; "shift_jis";
@@ -614,8 +615,8 @@ dnl Dictionary termination record.
 999; 0;
 ])
 for variant in \
-       "be fdf260a05220e08c748967dcb90d8b15" \
-       "le 4c9b0c0636bc0aa0cc16684c8188d1c7"
+       "be eab38c68398324ac1663161512a8516a" \
+       "le afc6173eed7f4c4e63dc4252bc18f6a2"
 do
   set $variant
   AT_CHECK_UNQUOTED([sack --$[1] sys-file.sack > sys-file.sav], [0], [], [$[2]
@@ -1755,6 +1756,8 @@ do
 done
 AT_CLEANUP
 
+dnl SPSS-generated system file can contain duplicate variable names
+dnl (see bug #41475).
 AT_SETUP([duplicate variable name])
 AT_KEYWORDS([sack synthetic system file negative])
 AT_DATA([sys-file.sack], [dnl
@@ -1777,9 +1780,20 @@ do
   AT_CHECK_UNQUOTED([sack --$[1] sys-file.sack > sys-file.sav], [0], [], [$[2]
 ])
   AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
+DISPLAY DICTIONARY.
 ])
-  AT_CHECK([pspp -O format=csv sys-file.sps], [1], 
-   [error: `sys-file.sav' near offset 0xd4: Duplicate variable name `VAR1'.
+  AT_CHECK([pspp -O format=csv sys-file.sps], [0],
+   [warning: `sys-file.sav' near offset 0xd4: Renaming variable with duplicate name `VAR1' to `VAR001'.
+
+Variable,Description,,Position
+var1,Format: F8.0,,1
+,Measure: Scale,,
+,Display Alignment: Right,,
+,Display Width: 8,,
+var001,Format: F8.0,,2
+,Measure: Scale,,
+,Display Alignment: Right,,
+,Display Width: 8,,
 ])
 done
 AT_CLEANUP
@@ -2078,9 +2092,20 @@ do
   AT_CHECK_UNQUOTED([sack --$[1] sys-file.sack > sys-file.sav], [0], [], [$[2]
 ])
   AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
+DISPLAY DICTIONARY.
 ])
-  AT_CHECK([pspp -O format=csv sys-file.sps], [1], 
-   [error: `sys-file.sav': Weighting variable must be numeric (not string variable `STR1').
+  AT_CHECK([pspp -O format=csv sys-file.sps], [0],
+   [warning: `sys-file.sav': Ignoring string variable `STR1' set as weighting variable.
+
+Variable,Description,,Position
+num1,Format: F8.0,,1
+,Measure: Scale,,
+,Display Alignment: Right,,
+,Display Width: 8,,
+str1,Format: A4,,2
+,Measure: Nominal,,
+,Display Alignment: Left,,
+,Display Width: 4,,
 ])
 done
 AT_CLEANUP
@@ -2121,7 +2146,7 @@ AT_KEYWORDS([sack synthetic system file negative])
 AT_DATA([sys-file.sack], [dnl
 dnl File header.
 "$FL2"; s60 "$(#) SPSS DATA FILE PSPP synthetic test file";
-2; 2; 1; >>3<<; -1; 100.0; "01 Jan 11"; "20:53:52"; s64 ""; i8 0 *3;
+2; 3; 1; >>3<<; -1; 100.0; "01 Jan 11"; "20:53:52"; s64 ""; i8 0 *3;
 
 dnl Numeric variable.
 2; 0; 0; 0; 0x050800 *2; s8 "NUM1";
@@ -2134,8 +2159,8 @@ dnl End of dictionary.
 999; 0;
 ])
 for variant in \
-       "be 0c395354df56ea5ff374aafcc535d633" \
-       "le d977f684ea9d4648ed40f8c6dddde9f7"
+       "be e47d2d0e65846a9b534e004f0ae65083" \
+       "le e5e6e00f34c4697c7bee21f15f86c7e3"
 do
   set $variant
   AT_CHECK_UNQUOTED([sack --$[1] sys-file.sack > sys-file.sav], [0], [], [$[2]
@@ -2450,30 +2475,102 @@ AT_KEYWORDS([sack synthetic system file negative multiple response])
 AT_DATA([sys-file.sack], [dnl
 dnl File header.
 "$FL2"; s60 "$(#) SPSS DATA FILE PSPP synthetic test file";
-2; 1; 1; 0; -1; 100.0; "01 Jan 11"; "20:53:52"; s64 ""; i8 0 *3;
+2; dnl Layout code
+16; dnl Nominal case size
+0; dnl Not compressed
+0; dnl Not weighted
+0; dnl No cases.
+100.0; dnl Bias.
+"01 Jan 11"; "20:53:52"; s64 "PSPP synthetic test file";
+i8 0 *3;
 
-dnl Numeric variable, no label or missing values.
-2; 0; 0; 0; 0x050800 *2; s8 "NUM1";
+dnl $a
+2; 0; 0; 0; 0x050800 *2; i8 0x82; i8 0xa0; s6 "";
+2; 0; 0; 0; 0x050800 *2; s8 "B";
+2; 0; 0; 0; 0x050800 *2; s8 "C";
 
-dnl Multiple response sets.
-7; 7; 1; COUNT("a=C");
-7; 19; 1; COUNT("xyz=D");
+dnl $b
+2; 0; 0; 0; 0x050800 *2; s8 "D";
+2; 0; 0; 0; 0x050800 *2; s8 "E";
+2; 0; 0; 0; 0x050800 *2; s8 "F";
+2; 0; 0; 0; 0x050800 *2; s8 "G";
+
+dnl $c
+2; 4; 0; 0; 0x010400 *2; s8 "H";
+2; 4; 0; 0; 0x010400 *2; s8 "I";
+2; 4; 0; 0; 0x010400 *2; s8 "J";
+
+dnl $d
+2; 0; 0; 0; 0x050800 *2; s8 "K";
+2; 0; 0; 0; 0x050800 *2; s8 "L";
+2; 0; 0; 0; 0x050800 *2; s8 "M";
+
+dnl $e
+2; 6; 0; 0; 0x010600 *2; s8 "N";
+2; 6; 0; 0; 0x010600 *2; s8 "O";
+2; 6; 0; 0; 0x010600 *2; s8 "P";
+
+dnl Machine integer info record.
+7; 3; 4; 8; 1; 2; 3; -1; 1; 1; ENDIAN; 932;
+
+7; 7; 1;
+COUNT(
+  "$a=C 10 my mcgroup "; i8 0x82; i8 0xa0; " b c"; i8 10;
+  "b=D2 55 0  g e f d"; i8 10;
+  "$c=D4 "; i8 0x82; i8 0xcd; i8 0x82; i8 0xa2; " 10 mdgroup #2 h i j"; i8 10);
 
+7; 19; 1;
+COUNT(
+  "$d=E 1 2 34 13 third mdgroup k l m"; i8 10;
+  "e=E 11 6 choice 0  n o p"; i8 10);
+
+dnl Character encoding record.
+7; 20; 1; 9; "shift_jis";
+
+dnl Dictionary termination record.
 999; 0;
 ])
 for variant in \
-       "be 15a9bf44d0cd6186a60629b77079c5a5" \
-       "le 161c99aca5e7a3684df096137e72ce5b"
+       "be 8832b331e09557a1ac0bf4e31611428a" \
+       "le 1c1dcd9930864632402b5c635395cb51"
 do
   set $variant
   AT_CHECK_UNQUOTED([sack --$[1] sys-file.sack > sys-file.sav], [0], [], [$[2]
 ])
   AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
+MRSETS /DISPLAY NAME=ALL.
 ])
   AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: `a' does not begin with `$' at offset 2 in MRSETS record.
+warning: `sys-file.sav': Multiple response set name `b' does not begin with `$'.
+
+warning: `sys-file.sav': Multiple response set name `e' does not begin with `$'.
 
-warning: `sys-file.sav' near offset 0xeb: `xyz' does not begin with `$' at offset 4 in MRSETS record.
+Table: Multiple Response Sets
+Name,Variables,Details
+$a,"あ
+b
+c
+","Multiple category set
+Label: my mcgroup
+"
+$c,"h
+i
+j
+","Multiple dichotomy set
+Label: mdgroup #2
+Label source: Provided by user
+Counted value: `はい'
+Category label source: Variable labels
+"
+$d,"k
+l
+m
+","Multiple dichotomy set
+Label: third mdgroup
+Label source: Provided by user
+Counted value: 34
+Category label source: Value labels of counted value
+"
 ])
 done
 AT_CLEANUP
@@ -2563,7 +2660,7 @@ do
   AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
 ])
   AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Unexpected label source value `2' following `E' at offset 7 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Unexpected label source value following `E' at offset 7 in MRSETS record.
 
 warning: `sys-file.sav' near offset 0xd8: Expecting digit at offset 7 in MRSETS record.
 ])
@@ -2747,7 +2844,7 @@ do
   AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
 warning: `sys-file.sav' near offset 0xd8: Missing new-line parsing variable names at offset 13 in MRSETS record.
 
-warning: `sys-file.sav' near offset 0xd8: MRSET $a has only 1 variables.
+warning: `sys-file.sav': MRSET $a has only one variable.
 ])
 done
 AT_CLEANUP
@@ -2777,9 +2874,9 @@ do
   AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
 ])
   AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Duplicate variable name NUM1 at offset 18 in MRSETS record.
+warning: `sys-file.sav': MRSET $a contains duplicate variable name NUM1.
 
-warning: `sys-file.sav' near offset 0xd8: MRSET $a has only 1 variables.
+warning: `sys-file.sav': MRSET $a has only one variable.
 ])
 done
 AT_CLEANUP
@@ -2810,9 +2907,9 @@ do
   AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
 ])
   AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xf8: MRSET $a contains both string and numeric variables.
+warning: `sys-file.sav': MRSET $a contains both string and numeric variables.
 
-warning: `sys-file.sav' near offset 0xf8: MRSET $a has only 1 variables.
+warning: `sys-file.sav': MRSET $a has only one variable.
 ])
 done
 AT_CLEANUP
@@ -2842,12 +2939,12 @@ do
   AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
 ])
   AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: MRSET $a has only 1 variables.
+warning: `sys-file.sav': MRSET $a has only one variable.
 ])
 done
 AT_CLEANUP
 
-AT_SETUP([only one variable in mrset])
+AT_SETUP([zero or one variable in mrset])
 AT_KEYWORDS([sack synthetic system file negative multiple response])
 AT_DATA([sys-file.sack], [dnl
 dnl File header.
@@ -2858,13 +2955,13 @@ dnl Numeric variable, no label or missing values.
 2; 0; 0; 0; 0x050800 *2; s8 "NUM1";
 
 dnl Multiple response sets.
-7; 7; 1; COUNT("$a=C 0  NUM1"; i8 10);
+7; 7; 1; COUNT("$a=C 0  NUM1"; i8 10; "$b=C 0  "; i8 10);
 
 999; 0;
 ])
 for variant in \
-       "be 3a891e0a467afb3d622629c70f329ada" \
-       "le 432998ec08370510411af4f5207c015e"
+       "be d94549fece75560fe5b47f32713b63a1" \
+       "le 63028521d460df9e998b40a75a93c330"
 do
   set $variant
   AT_CHECK_UNQUOTED([sack --$[1] sys-file.sack > sys-file.sav], [0], [], [$[2]
@@ -2872,7 +2969,9 @@ do
   AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
 ])
   AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: MRSET $a has only 1 variables.
+warning: `sys-file.sav': MRSET $a has only one variable.
+
+warning: `sys-file.sav': MRSET $b has no variables.
 ])
 done
 AT_CLEANUP
@@ -3811,7 +3910,7 @@ AT_CLEANUP
 
 AT_SETUP([zcompressed data - wrong compressed_ofs])
 AT_KEYWORDS([sack synthetic system file negative zlib])
-zcompressed_sack | sed 's/.*\bcompressed_ofs.*/i64 >>0x191<<;/' > sys-file.sack
+zcompressed_sack | sed 's/.*@%:@ compressed_ofs.*/i64 >>0x191<<;/' > sys-file.sack
 for variant in \
        "be 652e28f8d3f8e4ce47ad18d0f30e7bb9" \
        "le ebf2c647f2d7c47858d4f5ed683526e6"
@@ -3914,7 +4013,7 @@ AT_CLEANUP
 AT_SETUP([zcompressed data - compression expands data too much])
 AT_KEYWORDS([sack synthetic system file negative zlib])
 zcompressed_sack | sed 's/.*uncompressed_size.*/>>50<<;/
-s/.*\bcompressed_size.*/>>100<<;/' > sys-file.sack
+s/.*@%:@ compressed_size.*/>>100<<;/' > sys-file.sack
 for variant in \
        "be e11cadde5f0855c965a1cb388dedc36e" \
        "le 37953e71462b6554c5644fec8b539164"
@@ -3931,7 +4030,7 @@ AT_CLEANUP
 
 AT_SETUP([zcompressed data - compressed sizes don't add up])
 AT_KEYWORDS([sack synthetic system file negative zlib])
-zcompressed_sack | sed 's/.*\bcompressed_size.*/>>88<<;/' > sys-file.sack
+zcompressed_sack | sed 's/.*@%:@ compressed_size.*/>>88<<;/' > sys-file.sack
 for variant in \
        "be 366eaf85be1f26fb6549e2f8ee393628" \
        "le a756e5125e6a908cb4990f66cc419bef"