dnl PSPP - a program for statistical analysis.
dnl Copyright (C) 2017 Free Software Foundation, Inc.
dnl
dnl This program is free software: you can redistribute it and/or modify
dnl it under the terms of the GNU General Public License as published by
dnl the Free Software Foundation, either version 3 of the License, or
dnl (at your option) any later version.
dnl
dnl This program is distributed in the hope that it will be useful,
dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
dnl GNU General Public License for more details.
dnl
dnl You should have received a copy of the GNU General Public License
dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
dnl
AT_BANNER([DATA LIST])

AT_SETUP([DATA LIST LIST with empty fields])
AT_DATA([data-list.pspp], [dnl
DATA LIST LIST NOTABLE /A B C (F1.0).
BEGIN DATA.
,,
,,3
,2,
,2,3
1,,
1,,3
1,2,
1,2,3
END DATA.

LIST.
])
AT_CHECK([pspp -O format=csv data-list.pspp], [0], [dnl
Table: Data List
A,B,C
.,.,.
.,.,3
.,2,.
.,2,3
1,.,.
1,.,3
1,2,.
1,2,3
])
AT_CLEANUP


AT_SETUP([DATA LIST LIST with explicit delimiters])
AT_DATA([data-list.pspp], [dnl
data list list ('|','X') /A B C D.
begin data.
1|23X45|2.03x
2X22|34|23|
3|34|34X34
end data.

list.
])
AT_CHECK([pspp -O format=csv data-list.pspp], [0], [dnl
Table: Reading free-form data from INLINE.
Variable,Format
A,F8.0
B,F8.0
C,F8.0
D,F8.0

data-list.pspp:3.9-3.13: warning: Data for variable D is not valid as format F: Number followed by garbage.

Table: Data List
A,B,C,D
1.00,23.00,45.00,.  @&t@
2.00,22.00,34.00,23.00
3.00,34.00,34.00,34.00
])
AT_CLEANUP

AT_SETUP([DATA LIST FREE with SKIP])
AT_DATA([data-list.pspp], [dnl
data list free skip=1/A B C D.
begin data.
# This record is ignored.
,1,2,x
,4,,5
6
7,
8 9
0,1 ,,,
,,,,
2

3
4
5
end data.
list.
])
AT_CHECK([pspp -O format=csv data-list.pspp], [0], [dnl
data-list.pspp:4.6: warning: Data for variable D is not valid as format F: Field contents are not numeric.

Table: Data List
A,B,C,D
.  ,1.00,2.00,.  @&t@
.  ,4.00,.  ,5.00
6.00,7.00,8.00,9.00
.00,1.00,.  ,.  @&t@
.  ,.  ,.  ,.  @&t@
2.00,3.00,4.00,5.00
])
AT_CLEANUP

AT_SETUP([DATA LIST LIST with SKIP and tab delimiter])
AT_DATA([data-list.pspp], [dnl
data list list (tab) notable skip=2/A B C D.
begin data.
# These records
# are skipped.
1	2	3	4
1	2	3	@&t@
1	2		4
1	2		@&t@
1		3	4
1		3	@&t@
1			4
1			@&t@
	2	3	4
	2	3	@&t@
	2		4
	2		@&t@
		3	4
		3	@&t@
			4
			@&t@
end data.
list.
])
AT_CHECK([pspp -O format=csv data-list.pspp], [0], [dnl
Table: Data List
A,B,C,D
1.00,2.00,3.00,4.00
1.00,2.00,3.00,.  @&t@
1.00,2.00,.  ,4.00
1.00,2.00,.  ,.  @&t@
1.00,.  ,3.00,4.00
1.00,.  ,3.00,.  @&t@
1.00,.  ,.  ,4.00
1.00,.  ,.  ,.  @&t@
.  ,2.00,3.00,4.00
.  ,2.00,3.00,.  @&t@
.  ,2.00,.  ,4.00
.  ,2.00,.  ,.  @&t@
.  ,.  ,3.00,4.00
.  ,.  ,3.00,.  @&t@
.  ,.  ,.  ,4.00
.  ,.  ,.  ,.  @&t@
])
AT_CLEANUP

dnl Results of this test were confirmed with SPSS 21:
dnl http://lists.gnu.org/archive/html/pspp-dev/2013-09/msg00003.html
AT_SETUP([DATA LIST FREE with explicit delimiter at end of line])
AT_DATA([data-list.pspp], [dnl
DATA LIST FREE(',')/x y z.
BEGIN DATA.
1,2,3
4,5,6
7,8,9
END DATA.
LIST.

DATA LIST FREE(',')/x y z.
BEGIN DATA.
11,12,13,
14,15,16,
17,18,19,
END DATA.
LIST.

DATA LIST FREE(TAB)/x y z.
BEGIN DATA.
21	22	23
24	25	26
27	28	29
END DATA.
LIST.

DATA LIST FREE(TAB)/x y z.
BEGIN DATA.
31	32	33	@&t@
34	35	36	@&t@
37	38	39	@&t@
END DATA.
LIST.
])
AT_CHECK([pspp -O format=csv data-list.pspp], [0], [dnl
Table: Data List
x,y,z
1.00,2.00,3.00
4.00,5.00,6.00
7.00,8.00,9.00

Table: Data List
x,y,z
11.00,12.00,13.00
14.00,15.00,16.00
17.00,18.00,19.00

Table: Data List
x,y,z
21.00,22.00,23.00
24.00,25.00,26.00
27.00,28.00,29.00

Table: Data List
x,y,z
31.00,32.00,33.00
34.00,35.00,36.00
37.00,38.00,39.00
])
AT_CLEANUP

AT_SETUP([DATA LIST FIXED with multiple records per case])
AT_DATA([data-list.pspp], [dnl
data list fixed notable
        /1 start 1-20 (adate)
        /2 end 1-20 (adate)
        /3 count 1-3.
begin data.
07-22-2007
10-06-2007
x
07-14-1789
08-26-1789
xy
01-01-1972
12-31-1999
682
end data.
list.
])
AT_CHECK([pspp -O format=csv data-list.pspp], [0], [dnl
data-list.pspp:8.1-8.3: warning: Data for variable count is not valid as format F: Field contents are not numeric.

data-list.pspp:11.1-11.3: warning: Data for variable count is not valid as format F: Field contents are not numeric.

Table: Data List
start,end,count
07/22/2007,10/06/2007,.
07/14/1789,08/26/1789,.
01/01/1972,12/31/1999,682
])
AT_CLEANUP

AT_SETUP([DATA LIST FIXED with empty trailing record])
AT_DATA([data-list.pspp], [dnl
data list fixed notable records=2/x 1 y 2.
begin data.
12

34

56

78

90

end data.
list.
])
AT_CHECK([pspp -O format=csv data-list.pspp], [0], [dnl
Table: Data List
x,y
1,2
3,4
5,6
7,8
9,0
])
AT_CLEANUP

dnl Test that PSPP accepts LF and CR LF as line ends, but
dnl treats isolated CR as linear whitespace.
AT_SETUP([DATA LIST with various line-ends])
AT_DATA([data-list.sps], [dnl
data list list notable file='input.txt'/a b c.
list.
])
printf '1 2 3\n4 5 6\r\n7\r8\r9\r\n10 11 12\n13 14 15 \r\n16\r\r17\r18\n' > input.txt
dnl Make sure that input.txt actually received the data that we expect.
dnl It might not have, if we're running on a system that translates \n
dnl into some other sequence.
AT_CHECK([cksum input.txt], [0], [1732021750 50 input.txt
])
AT_CHECK([pspp -o pspp.csv data-list.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
a,b,c
1.00,2.00,3.00
4.00,5.00,6.00
7.00,8.00,9.00
10.00,11.00,12.00
13.00,14.00,15.00
16.00,17.00,18.00
])
AT_CLEANUP

AT_SETUP([DATA LIST properly expands tabs in input])
AT_DATA([data-list.sps], [dnl
data list notable /X 1-50 (a).
begin data.
	1	12	123	1234	12345    .
end data.
print /x.
print outfile='print.txt' /x.
write outfile='write.txt' /x.
execute.
])
AT_CHECK([sed -n '/12345/l' data-list.sps], [0], [dnl
\t1\t12\t123\t1234\t12345    .$
])
AT_CHECK([pspp -o pspp.csv data-list.sps])
dnl The CSV driver drops leading spaces so they don't appear here:
AT_CHECK([cat pspp.csv], [0], [dnl
1       12      123     1234    12345    . @&t@
])
dnl But they do appear in print.txt.  The PRINT command also puts a space
dnl at the beginning of the line and after the variable:
AT_CHECK([cat print.txt], [0], [dnl
         1       12      123     1234    12345    . @&t@
])
dnl WRITE doesn't add spaces at the beginning or end of lines:
AT_CHECK([cat write.txt], [0], [dnl
        1       12      123     1234    12345    .
])
AT_CLEANUP

AT_SETUP([DATA LIST FREE and LIST report missing delimiters])
AT_DATA([data-list.sps], [dnl
DATA LIST FREE NOTABLE/s (a10).
LIST.
BEGIN DATA.
'y'z
END DATA.
])
AT_CHECK([pspp -O format=csv data-list.sps], [0], [dnl
data-list.sps:4: warning: Missing delimiter following quoted string.

Table: Data List
s
y
z
])
AT_CLEANUP

AT_SETUP([DATA LIST FREE and LIST assume a width if omitted])
AT_DATA([data-list.sps], [dnl
DATA LIST FREE TABLE/s (a) d (datetime) f (f).
])
AT_CHECK([pspp -O format=csv data-list.sps], [0], [dnl
Table: Reading free-form data from INLINE.
Variable,Format
s,A1
d,DATETIME17.0
f,F1.0
])
AT_CLEANUP

AT_SETUP([DATA LIST Decimal comma])
AT_DATA([data-list.sps], [dnl
SET DECIMAL=COMMA.

DATA LIST NOTABLE LIST /A *.
BEGIN DATA
1
2
3
3,5
4
4,5
5
6
END DATA

LIST /FORMAT=NUMBERED.
])

AT_CHECK([pspp -O format=csv data-list.sps], [0], [dnl
Table: Data List
Case Number,A
1,"1,00"
2,"2,00"
3,"3,00"
4,"3,50"
5,"4,00"
6,"4,50"
7,"5,00"
8,"6,00"
])

AT_CLEANUP

AT_SETUP([DATA LIST syntax errors])
AT_DATA([insert.sps], [dnl
INSERT FILE='data-list.sps' ERROR=IGNORE.
])
AT_DATA([data-list.sps], [dnl
DATA LIST FILE=**.
DATA LIST ENCODING=**.
DATA LIST RECORDS=1 RECORDS=2.
DATA LIST RECORDS=0.
DATA LIST SKIP=-1.
DATA LIST END=**.
INPUT PROGRAM.
DATA LIST END=xyzzy END=xyzzy.
END INPUT PROGRAM.
INPUT PROGRAM.
DATA LIST END=**.
END INPUT PROGRAM.
DATA LIST XYZZY.
DATA LIST FREE LIST.
DATA LIST LIST (**).
DATA LIST **.
DATA LIST ENCODING='xyzzy'/x.
INPUT PROGRAM.
DATA LIST LIST END=xyzzy/x.
END INPUT PROGRAM.
DATA LIST FIXED/0.
DATA LIST FIXED/ **.
DATA LIST FIXED/x 1.5.
DATA LIST FIXED/x -1.
DATA LIST FIXED/x 5-3.
DATA LIST FIXED/x y 1-3.
DATA LIST FIXED/x 1-5 (xyzzy).
DATA LIST FIXED/x 1-5 (**).
DATA LIST FIXED/x 1 (F,5).
DATA LIST FIXED/x (2F8.0).
DATA LIST FIXED/x **.
DATA LIST FIXED/x 1 x 2.
INPUT PROGRAM.
DATA LIST FIXED/x 1.
DATA LIST FIXED/x 1 (a).
END INPUT PROGRAM.
INPUT PROGRAM.
DATA LIST FIXED/y 2 (a).
DATA LIST FIXED/y 3-4 (a).
END INPUT PROGRAM.
DATA LIST FIXED RECORDS=1/x y(F2/F3).
DATA LIST FIXED RECORDS=1//.
DATA LIST FIXED RECORDS=1/.
])

AT_CHECK([pspp --testing-mode -O format=csv insert.sps], [1], [dnl
"data-list.sps:1.16-1.17: error: DATA LIST: Syntax error expecting a file name or handle name.
    1 | DATA LIST FILE=**.
      |                ^~"

"data-list.sps:2.20-2.21: error: DATA LIST: Syntax error expecting string.
    2 | DATA LIST ENCODING=**.
      |                    ^~"

"data-list.sps:3.21-3.27: error: DATA LIST: Subcommand RECORDS may only be specified once.
    3 | DATA LIST RECORDS=1 RECORDS=2.
      |                     ^~~~~~~"

"data-list.sps:4.20: error: DATA LIST: Syntax error expecting one of the following: FILE, ENCODING, RECORDS, SKIP, END, NOTABLE, TABLE, FIXED, FREE, LIST.
    4 | DATA LIST RECORDS=0.
      |                    ^"

"data-list.sps:5.16-5.17: error: DATA LIST: Syntax error expecting non-negative integer for SKIP.
    5 | DATA LIST SKIP=-1.
      |                ^~"

"data-list.sps:6.11-6.13: error: DATA LIST: The END subcommand may only be used within INPUT PROGRAM.
    6 | DATA LIST END=**.
      |           ^~~"

"data-list.sps:8.21-8.23: error: DATA LIST: Subcommand END may only be specified once.
    8 | DATA LIST END=xyzzy END=xyzzy.
      |                     ^~~"

"data-list.sps:11.15-11.16: error: DATA LIST: Syntax error expecting identifier.
   11 | DATA LIST END=**.
      |               ^~"

"data-list.sps:13.11-13.15: error: DATA LIST: Syntax error expecting one of the following: FILE, ENCODING, RECORDS, SKIP, END, NOTABLE, TABLE, FIXED, FREE, LIST.
   13 | DATA LIST XYZZY.
      |           ^~~~~"

"data-list.sps:14.16-14.19: error: DATA LIST: Only one of FIXED, FREE, or LIST may be specified.
   14 | DATA LIST FREE LIST.
      |                ^~~~"

"data-list.sps:15.17-15.18: error: DATA LIST: Syntax error expecting TAB or delimiter string.
   15 | DATA LIST LIST (**).
      |                 ^~"

"data-list.sps:16.11-16.12: error: DATA LIST: Syntax error expecting one of the following: FILE, ENCODING, RECORDS, SKIP, END, NOTABLE, TABLE, FIXED, FREE, LIST.
   16 | DATA LIST **.
      |           ^~"

"data-list.sps:17.11-17.26: warning: DATA LIST: Encoding should not be specified for inline data. It will be ignored.
   17 | DATA LIST ENCODING='xyzzy'/x.
      |           ^~~~~~~~~~~~~~~~"

"data-list.sps:17.29: error: DATA LIST: SPSS-like or Fortran-like format specification expected after variable names.
   17 | DATA LIST ENCODING='xyzzy'/x.
      |                             ^"

"data-list.sps:19.16-19.24: error: DATA LIST: The END subcommand may be used only with DATA LIST FIXED.
   19 | DATA LIST LIST END=xyzzy/x.
      |                ^~~~~~~~~"

"data-list.sps:21.17: error: DATA LIST: Syntax error expecting positive integer.
   21 | DATA LIST FIXED/0.
      |                 ^"

"data-list.sps:22.18-22.19: error: DATA LIST: Syntax error expecting variable name.
   22 | DATA LIST FIXED/ **.
      |                  ^~"

"data-list.sps:23.19-23.21: error: DATA LIST: Syntax error expecting integer.
   23 | DATA LIST FIXED/x 1.5.
      |                   ^~~"

"data-list.sps:24.19-24.20: error: DATA LIST: Column positions for fields must be positive.
   24 | DATA LIST FIXED/x -1.
      |                   ^~"

"data-list.sps:25.19-25.21: error: DATA LIST: The ending column for a field must be greater than the starting column.
   25 | DATA LIST FIXED/x 5-3.
      |                   ^~~"

"data-list.sps:26.21-26.23: error: DATA LIST: The 3 columns 1-3 can't be evenly divided into 2 fields.
   26 | DATA LIST FIXED/x y 1-3.
      |                     ^~~"

"data-list.sps:27.24-27.28: error: DATA LIST: Unknown format type `xyzzy'.
   27 | DATA LIST FIXED/x 1-5 (xyzzy).
      |                        ^~~~~"

"data-list.sps:28.24-28.25: error: DATA LIST: Syntax error expecting `)'.
   28 | DATA LIST FIXED/x 1-5 (**).
      |                        ^~"

"data-list.sps:29.19-29.25: error: DATA LIST: Input format F1.5 specifies 5 decimal places, but width 1 allows at most 1 decimals.
   29 | DATA LIST FIXED/x 1 (F,5).
      |                   ^~~~~~~"

"data-list.sps:30.20-30.25: error: DATA LIST: Number of variables specified (1) differs from number of variable formats (2).
   30 | DATA LIST FIXED/x (2F8.0).
      |                    ^~~~~~"

"data-list.sps:31.19-31.20: error: DATA LIST: SPSS-like or Fortran-like format specification expected after variable names.
   31 | DATA LIST FIXED/x **.
      |                   ^~"

"data-list.sps:32.21: error: DATA LIST: x is a duplicate variable name.
   32 | DATA LIST FIXED/x 1 x 2.
      |                     ^"

Table: Reading 1 record from INLINE.
Variable,Record,Columns,Format
x,1,1-1,F1.0

"data-list.sps:35.17-35.23: error: DATA LIST: There is already a variable x of a different type.
   35 | DATA LIST FIXED/x 1 (a).
      |                 ^~~~~~~"

Table: Reading 1 record from INLINE.
Variable,Record,Columns,Format
y,1,2-2,A1

"data-list.sps:39.17-39.25: error: DATA LIST: There is already a string variable y of a different width.
   39 | DATA LIST FIXED/y 3-4 (a).
      |                 ^~~~~~~~~"

"data-list.sps:41.27-41.36: error: DATA LIST: Cannot place variable y on record 2 when RECORDS=1 is specified.
   41 | DATA LIST FIXED RECORDS=1/x y(F2/F3).
      |                           ^~~~~~~~~~"

"data-list.sps:42.27: error: DATA LIST: Cannot advance to record 2 when RECORDS=1 is specified.
   42 | DATA LIST FIXED RECORDS=1//.
      |                           ^"

"data-list.sps:43.26: error: DATA LIST: No fields were specified.  At least one is required.
   43 | DATA LIST FIXED RECORDS=1/.
      |                          ^"
])
AT_CLEANUP

AT_SETUP([DATA LIST FIXED manual example 1])
AT_DATA([data-list.sps], [dnl
DATA LIST TABLE /NAME 1-10 (A) INFO1 TO INFO3 12-17 (1).

BEGIN DATA.
John Smith 102311
Bob Arnold 122015
Bill Yates  918 6
END DATA.

LIST.
])
AT_CHECK([pspp data-list.sps -O box=unicode], [0], [dnl
  Reading 1 record from INLINE.
╭────────┬──────┬───────┬──────╮
│Variable│Record│Columns│Format│
├────────┼──────┼───────┼──────┤
│NAME    │     1│1-10   │A10   │
│INFO1   │     1│12-13  │F2.1  │
│INFO2   │     1│14-15  │F2.1  │
│INFO3   │     1│16-17  │F2.1  │
╰────────┴──────┴───────┴──────╯

           Data List
╭──────────┬─────┬─────┬─────╮
│   NAME   │INFO1│INFO2│INFO3│
├──────────┼─────┼─────┼─────┤
│John Smith│  1.0│  2.3│  1.1│
│Bob Arnold│  1.2│  2.0│  1.5│
│Bill Yates│   .9│  1.8│   .6│
╰──────────┴─────┴─────┴─────╯
])
AT_CLEANUP

AT_SETUP([DATA LIST FIXED manual example 2])
AT_DATA([data-list.sps], [dnl
DATA LIST
        /ID 1-5 NAME 7-36 (A) SURNAME 38-67 (A) MINITIAL 69 (A)
        /Q01 TO Q50 7-56
        /.
])
AT_CHECK([pspp data-list.sps -O box=unicode], [0], [dnl
 Reading 3 records from INLINE.
╭────────┬──────┬───────┬──────╮
│Variable│Record│Columns│Format│
├────────┼──────┼───────┼──────┤
│ID      │     1│1-5    │F5.0  │
│NAME    │     1│7-36   │A30   │
│SURNAME │     1│38-67  │A30   │
│MINITIAL│     1│69-69  │A1    │
│Q01     │     2│7-7    │F1.0  │
│Q02     │     2│8-8    │F1.0  │
│Q03     │     2│9-9    │F1.0  │
│Q04     │     2│10-10  │F1.0  │
│Q05     │     2│11-11  │F1.0  │
│Q06     │     2│12-12  │F1.0  │
│Q07     │     2│13-13  │F1.0  │
│Q08     │     2│14-14  │F1.0  │
│Q09     │     2│15-15  │F1.0  │
│Q10     │     2│16-16  │F1.0  │
│Q11     │     2│17-17  │F1.0  │
│Q12     │     2│18-18  │F1.0  │
│Q13     │     2│19-19  │F1.0  │
│Q14     │     2│20-20  │F1.0  │
│Q15     │     2│21-21  │F1.0  │
│Q16     │     2│22-22  │F1.0  │
│Q17     │     2│23-23  │F1.0  │
│Q18     │     2│24-24  │F1.0  │
│Q19     │     2│25-25  │F1.0  │
│Q20     │     2│26-26  │F1.0  │
│Q21     │     2│27-27  │F1.0  │
│Q22     │     2│28-28  │F1.0  │
│Q23     │     2│29-29  │F1.0  │
│Q24     │     2│30-30  │F1.0  │
│Q25     │     2│31-31  │F1.0  │
│Q26     │     2│32-32  │F1.0  │
│Q27     │     2│33-33  │F1.0  │
│Q28     │     2│34-34  │F1.0  │
│Q29     │     2│35-35  │F1.0  │
│Q30     │     2│36-36  │F1.0  │
│Q31     │     2│37-37  │F1.0  │
│Q32     │     2│38-38  │F1.0  │
│Q33     │     2│39-39  │F1.0  │
│Q34     │     2│40-40  │F1.0  │
│Q35     │     2│41-41  │F1.0  │
│Q36     │     2│42-42  │F1.0  │
│Q37     │     2│43-43  │F1.0  │
│Q38     │     2│44-44  │F1.0  │
│Q39     │     2│45-45  │F1.0  │
│Q40     │     2│46-46  │F1.0  │
│Q41     │     2│47-47  │F1.0  │
│Q42     │     2│48-48  │F1.0  │
│Q43     │     2│49-49  │F1.0  │
│Q44     │     2│50-50  │F1.0  │
│Q45     │     2│51-51  │F1.0  │
│Q46     │     2│52-52  │F1.0  │
│Q47     │     2│53-53  │F1.0  │
│Q48     │     2│54-54  │F1.0  │
│Q49     │     2│55-55  │F1.0  │
│Q50     │     2│56-56  │F1.0  │
╰────────┴──────┴───────┴──────╯
])
AT_CLEANUP

# Checks for a bug where DATA LIST was allowed inside LOOP, which
# cleared the dataset and caused LOOP to assert-fail because its
# transformation stack entry had been popped.
AT_SETUP([DATA LIST inside LOOP])
AT_DATA([data-list.sps], [dnl
DATA LIST LIST NOTABLE/x y z.
LOOP.
DATA LIST.
END LOOP.
])
AT_CHECK([pspp data-list.sps], [1], [dnl
data-list.sps:3.1-3.9: error: DATA LIST: DATA LIST is not allowed inside DO IF
or LOOP (except inside INPUT PROGRAM).
    3 | DATA LIST.
      | ^~~~~~~~~
])
AT_CLEANUP