« back to all changes in this revision
Viewing changes to tests/csv.lisp
- browse files at revision 6
- compare with another revision
- download diff
- download tarball
- view history from revision 6
- Committer: Package Import Robot
- Author(s): Dimitri Fontaine
- Date: 2014-08-04 19:57:54 UTC
- mfrom: (1.1.2)
- Revision ID: package-import@ubuntu.com-20140804195754-vo64b5r1daxwg8ld
Tags: 20140211-1
Quicklisp release update.
- files added:
- .gitignore
- .pc
- debian/source
- tests
- files removed:
- csv-src.lisp
- files modified:
- debian/changelog
added
removed
tests/csv.lisp
1
(defpackage :cl-csv-test
2
(:use :cl :cl-user :cl-csv :lisp-unit2 :iter))
3
4
(in-package :cl-csv-test)
5
(cl-interpol:enable-interpol-syntax)
6
7
(defun run-all-tests ()
8
(run-tests
9
:package :cl-csv-test
10
:name :cl-csv
11
:run-contexts #'with-summary-context ))
12
13
(defmacro assert-length (exp it &rest them)
14
`(assert-eql ,exp (length ,it) ,@them))
15
16
(defparameter +test-csv-quoted-path+
17
(asdf:system-relative-pathname :cl-csv "tests/test-csv-quoted.csv"))
18
(defparameter +test-csv-unquoted-path+
19
(asdf:system-relative-pathname :cl-csv "tests/test-csv-unquoted.csv"))
20
(defparameter +test-csv-unquoted-no-trailing-path+
21
(asdf:system-relative-pathname :cl-csv "tests/test-csv-unquoted-no-trailing.csv"))
22
(defparameter +test-multiline+
23
(asdf:system-relative-pathname :cl-csv "tests/test-multiline-data.csv"))
24
25
(defparameter +test-files+
26
(list
27
+test-csv-quoted-path+
28
+test-csv-unquoted-path+
29
+test-csv-unquoted-no-trailing-path+) )
30
31
(defparameter *test-csv1-rows*
32
'(("first name" "last name" "job \"title\"" "number of hours" "id")
33
("Russ" "Tyndall" "Software Developer's, \"Position\"" "26.2" "1")
34
("Adam" "Smith" "Economist" "37.5" "2")
35
("John" "Doe" "Anonymous Human" "42.1" "3")
36
("Chuck" "Darwin" "Natural Philosipher" "17.68" "4")
37
("Bill" "Shakespear" "Bard" "12.2" "5")
38
("James" "Kirk" "Starship Captain" "13.1" "6")
39
("Bob" "Anon" "" "13.1" "6")
40
("Mr" "Iñtërnâtiônàlizætiøn" "" "1.1" "0")))
41
42
(defparameter *test-csv1*
43
"\"first name\",\"last name\",\"job \"\"title\"\"\",\"number of hours\",\"id\"
44
\"Russ\",\"Tyndall\",\"Software Developer's, \"\"Position\"\"\",\"26.2\",\"1\"
45
\"Adam\",\"Smith\",\"Economist\",\"37.5\",\"2\"
46
\"John\",\"Doe\",\"Anonymous Human\",\"42.1\",\"3\"
47
\"Chuck\",\"Darwin\",\"Natural Philosipher\",\"17.68\",\"4\"
48
\"Bill\",\"Shakespear\",\"Bard\",\"12.2\",\"5\"
49
\"James\",\"Kirk\",\"Starship Captain\",\"13.1\",\"6\"
50
\"Bob\",\"Anon\",\"\",\"13.1\",\"6\"
51
\"Mr\",\"Iñtërnâtiônàlizætiøn\",\"\",\"1.1\",\"0\"
52
")
53
54
(defparameter *test-csv1-v2*
55
"first name,last name,\"job \"\"title\"\"\",number of hours,id
56
Russ,Tyndall,\"Software Developer's, \"\"Position\"\"\",26.2,1
57
Adam,Smith,Economist,37.5,2
58
John,Doe,Anonymous Human,42.1,3
59
Chuck,Darwin,Natural Philosipher,17.68,4
60
Bill,Shakespear,Bard,12.2,5
61
James,Kirk,Starship Captain,13.1,6
62
Bob,Anon,,13.1,6
63
Mr,Iñtërnâtiônàlizætiøn,,1.1,0
64
")
65
66
(defparameter *test-csv-no-trailing-newline*
67
"first name,last name,\"job \"\"title\"\"\",number of hours,id
68
Russ,Tyndall,\"Software Developer's, \"\"Position\"\"\",26.2,1")
69
70
(defparameter *test-csv-data-with-newlines*
71
"first name,last name,\"job \"\"title\"\"\",number of hours,id
72
Russ,Tyndall,\"Software Developer's,
73
\"\"Position\"\"\",26.2,1")
74
75
(defparameter *test-csv-data-waiting-next-error*
76
"\"Which of the following is an appropriate calming technique or statement:
77
A. \"\"I can help you.\"\"
78
B. \"\"Shut up.\"\"
79
C. \"\"If you don't calm down I'm not sending anyone.\"\"
80
D. \"\"Ma'am, ma'am\ ma'am!\"\"\",A")
81
82
(define-test parsing-1 (:tags '(parsing))
83
(assert-equal *test-csv1-rows* (read-csv *test-csv1*))
84
(assert-equal *test-csv1-rows* (read-csv *test-csv1-v2*)))
85
86
(define-test writing-1 (:tags '(writing))
87
(assert-equal *test-csv1* (write-csv *test-csv1-rows* :always-quote t)))
88
89
(define-test parsing-errors (:tags '(parsing errors))
90
(assert-error 'csv-parse-error
91
(read-csv-row
92
"first name, a test\" broken quote, other stuff"))
93
(assert-error 'csv-parse-error
94
(read-csv-row
95
"first name,\"a test broken quote\" what are these chars, other stuff"))
96
(assert-error 'csv-parse-error
97
(read-csv-row
98
"first name,\"a test unfinished quote, other stuff"))
99
(assert-eql 3 (length (read-csv-row "first name, \"a test broken quote\", other stuff
b'")))'
100
)
101
102
(define-test no-trailing-parse (:tags '(parsing errors))
103
(let* ((data (read-csv *test-csv-no-trailing-newline*))
104
(str (write-csv data :always-quote t))
105
(data2 (read-csv str)))
106
(assert-equal 2 (length data))
107
(assert-equal 5 (length (first data)))
108
(assert-equal 5 (length (second data)))
109
(assert-equal data data2)))
110
111
(define-test data-with-newlines (:tags '(whitespace parsing writing))
112
(let* ((data (read-csv *test-csv-data-with-newlines*))
113
(str (write-csv data :always-quote t))
114
(data2 (read-csv str)))
115
(assert-equal 2 (length data))
116
(assert-equal 5 (length (first data)))
117
(assert-equal 5 (length (second data)))
118
(assert-equal
119
"Software Developer's,
120
\"Position\""
121
(third (second data)))
122
(assert-equal data data2)))
123
124
(define-test data-with-whitespace-trim (:tags '(whitespace parsing trim))
125
(assert-equal
126
'("first" "last" " other " "" nil nil)
127
(read-csv-row " first , last , ' other ','',, "
128
:unquoted-empty-string-is-nil t
129
:quoted-empty-string-is-nil nil
130
:trim-outer-whitespace t
131
:quote #\'))
132
(assert-equal
133
'(" first " " last " " other " "" nil " ")
134
(read-csv-row " first , last ,' other ','',, "
135
:unquoted-empty-string-is-nil t
136
:quoted-empty-string-is-nil nil
137
:trim-outer-whitespace nil
138
:quote #\'))
139
140
(assert-error 'csv-parse-error
141
(read-csv-row " first , last , ' other ','',, "
142
:unquoted-empty-string-is-nil t
143
:quoted-empty-string-is-nil nil
144
:trim-outer-whitespace nil
145
:quote #\')
146
"whitespace before quoted values is a parse error if we are
147
not trimming ")
148
(assert-error 'csv-parse-error
149
(read-csv-row " first , last ,' other ' ,'',, "
150
:unquoted-empty-string-is-nil t
151
:quoted-empty-string-is-nil nil
152
:trim-outer-whitespace nil
153
:quote #\')
154
"whitespace after quoted values is a parse error if we are
155
not trimming ")
156
)
157
158
(define-test data-with-whitespace-nilling (:tags '(whitespace parsing trim))
159
(assert-equal
160
'("first" "last" " other " nil nil nil)
161
(read-csv-row " first , last , ' other ' ,'',, "
162
:quoted-empty-string-is-nil t
163
:unquoted-empty-string-is-nil t
164
:quote #\'))
165
(assert-equal
166
'("first" "last" " other " "" "" "")
167
(read-csv-row " first , last ,' other ','',, "
168
:quoted-empty-string-is-nil nil
169
:unquoted-empty-string-is-nil nil
170
:quote #\'))
171
172
(assert-equal
173
'("first" "last" " other " nil "" "")
174
(read-csv-row " first , last , ' other ','',, "
175
:quoted-empty-string-is-nil T
176
:unquoted-empty-string-is-nil nil
177
:quote #\')
178
"whitespace before quoted values is a parse error if we are
179
not trimming ")
180
(assert-equal
181
'("first" "last" " other " "" nil nil)
182
(read-csv-row " first , last ,' other ' ,'',, "
183
:quoted-empty-string-is-nil nil
184
:unquoted-empty-string-is-nil t
185
:quote #\')
186
"whitespace after quoted values is a parse error if we are
187
not trimming ")
188
)
189
190
191
(define-test files (:tags '(parsing files))
192
(iter (for csv in +test-files+)
193
(for data = (read-csv csv))
194
(assert-equal *test-csv1-rows* data csv)))
195
196
(define-test multi-line-file (:tags '(parsing files))
197
(let ((data (read-csv +test-multiline+)))
198
(assert-equal 2 (length data) data)
199
(assert-equal "test
200
of
201
multiline" (nth 3 (first data)) ))
202
)
203
204
(define-test dont-always-quote-and-newline (:tags '(writing whitespace quotation))
205
(let* ((row '("Russ" "Tyndall" "Software Developer's, \"Position\"" "26.2" "1" ","))
206
(res (write-csv-row row :always-quote nil :newline #?"\n")))
207
(assert-equal #?"Russ,Tyndall,\"Software Developer's, \"\"Position\"\"\",26.2,1,\",\"\n"
208
res)))
209
210
(define-test dont-always-quote-and-newline-2 (:tags '(writing whitespace quotation))
211
(let* ((row '("," #?"a\r\nnewline\r\ntest\r\n"))
212
(res (write-csv-row row :always-quote nil :newline #?"\n")))
213
(assert-equal #?"\",\",\"a\r\nnewline\r\ntest\r\n\"\n"
214
res)))
215
216
(define-test cause-error (:tags '(parsing errors))
217
(let ((data (read-csv *test-csv-data-waiting-next-error*)))
218
(assert-true data)))
219
220
(define-test chars-in-test (:tags '(utils parsing))
221
(assert-true (cl-csv::chars-in "a" "abcdef"))
222
(assert-false (cl-csv::chars-in "qu" "abcdef"))
223
(assert-true (cl-csv::chars-in "qu" "asdfqasdf"))
224
(assert-true (cl-csv::chars-in "qu" "asdfuasdf"))
225
(assert-true (cl-csv::chars-in (list "q" "u") "asdfuasdf"))
226
(assert-true (cl-csv::chars-in (list #\q #\u) "asdfuasdf"))
227
(assert-true (cl-csv::chars-in (list "q" #\u) "asdfqasdf")))
228
229
(define-test iterate-clauses (:tags '(utils iterate))
230
(iter
231
(for (a b c) in-csv "1,2,3
232
4,5,6")
233
(assert-equal (if (first-time-p) "1" "4") a)
234
(assert-equal (if (first-time-p) "2" "5") b)
235
(assert-equal (if (first-time-p) "3" "6") c)
236
(for i from 0)
237
(finally (assert-equal 1 i)))
238
239
;; test SKIPPING-HEADER option
240
(iter
241
(for (a b c) in-csv "1,2,3
242
4,5,6" SKIPPING-HEADER T)
243
(assert-equal "4" a)
244
(assert-equal "5" b)
245
(assert-equal "6" c)
246
(for i from 0)
247
(finally (assert-equal 0 i)))
248
249
;; test SEPARATOR
250
(iter
251
(for (a b c) in-csv "1|2|3
252
4|5|6" SKIPPING-HEADER T SEPARATOR #\|)
253
(assert-equal "4" a)
254
(assert-equal "5" b)
255
(assert-equal "6" c)
256
(for i from 0)
257
(finally (assert-equal 0 i))))
258
259
(define-test sampling-iterate (:tags '(parsing iterate))
260
(assert-length
261
9 (iter (for row in-csv *test-csv1*)
262
(cl-csv:sampling row)))
263
(assert-length
264
2 (iter (for row in-csv *test-csv1*)
265
(cl-csv:sampling row into sample size 2)
266
(finally (return sample))))
267
(assert-length
268
2 (read-csv-sample *test-csv1* 2))
269
(assert-length
270
3 (iter (for row in-csv *test-csv1* skipping-header t)
271
(cl-csv::sampling row size 3)))
272
(assert-length
273
9 (iter (for row in-csv *test-csv1*)
274
(cl-csv:sampling row into sample size 25)
275
(finally (return sample)))))
276
277
(define-test csv-signal-enabling (:tags '(signals))
278
(assert-signal
279
'csv-row-read
280
(assert-signal
281
'csv-data-read
282
(let ((*enable-signals* t))
283
(cl-csv:read-csv "1,2,3"))))
284
(assert-no-signal
285
'csv-row-read
286
(assert-no-signal
287
'csv-data-read
288
(let ((*enable-signals* nil))
289
(cl-csv:read-csv "1,2,3")))))
290
291
(define-test csv-filter (:tags '(signals))
292
(assert-equal
293
'(1 2 3)
294
(let ((*enable-signals* t))
295
(handler-bind ((csv-data-read
296
(lambda (c) (invoke-restart 'filter (parse-integer (cl-csv::data c))))))
297
(cl-csv:read-csv-row "1,2,3"))))
298
(assert-equal
299
'(1 2 3)
300
(let ((*enable-signals* t))
301
(handler-bind ((csv-row-read
302
(lambda (c) (invoke-restart 'filter (mapcar #'parse-integer (cl-csv::row c))))))
303
(cl-csv:read-csv-row "1,2,3")))))
304
305
(defun displaced-sub-string (s &key (start 0) (end (length s)))
306
(make-array (- end start)
307
:element-type (array-element-type s)
308
:displaced-to s
309
:displaced-index-offset start))
310
311
(define-test csv-continue-signals (:tags '(signals))
312
(handler-bind ((csv-parse-error #'continue))
313
(assert-equal
314
'(("1" "2" "3")
315
("3" "4" "5"))
316
(cl-csv:read-csv "1,2,3
317
2,3',4
318
3,4,5" :quote #\'))))
319
320
(define-test early-end-of-stream (:tags '(errors parsing))
321
(let ((line #?|"1","2|))
322
(assert-error
323
'cl-csv:csv-parse-error
324
(cl-csv:read-csv-row line)))
325
(let ((line ""))
326
(assert-error
327
'end-of-file
328
(cl-csv:read-csv-row line)))
329
330
(let ((line #?|"1","2|))
331
(assert-equal
332
'("1" "2")
333
(handler-bind ((cl-csv:csv-parse-error
334
(lambda (c)
335
(declare (ignore c))
336
(invoke-restart 'cl-csv::finish-item))))
337
(cl-csv:read-csv-row line))))
338
)
339
340
(define-test read-into-buffer-until-test (:tags '(read-until))
341
;; \r\l newline
342
(with-input-from-string (in #?"test this\r\n thing")
343
(let* ((s (make-string 80))
344
(l (cl-csv::read-into-buffer-until s in :nl #?"\r\n")))
345
(assert-eql 11 l)
346
(assert-equal #?"test this\r\n" (displaced-sub-string s :end l))))
347
;; newline
348
(with-input-from-string (in #?"t\nest this\n thing")
349
(let* ((s (make-string 80))
350
(l (cl-csv::read-into-buffer-until s in :nl #\newline)))
351
(assert-eql 2 l)
352
(assert-equal #?"t\n" (displaced-sub-string s :end l))
353
(let ((l (cl-csv::read-into-buffer-until s in :nl #\newline)))
354
(assert-eql 9 l)
355
(assert-equal #?"est this\n" (displaced-sub-string s :end l)))))
356
;; EOF
357
(with-input-from-string (in #?"test this thing")
358
(let* ((s (make-string 80))
359
(l (cl-csv::read-into-buffer-until s in :nl #\newline)))
360
(assert-eql (length "test this thing") l)
361
(assert-equal "test this thing" (displaced-sub-string s :end l))
362
))
363
;; filled buffer
364
(with-input-from-string (in #?"test this thing")
365
(let* ((s (make-string 4))
366
(l (cl-csv::read-into-buffer-until s in :nl #\newline)))
367
(assert-eql 4 l)
368
(assert-equal "test" (displaced-sub-string s :end l))
369
(assert-eql 4 (cl-csv::read-into-buffer-until s in :nl #\newline))
370
(assert-eql 4 (cl-csv::read-into-buffer-until s in :nl #\newline))
371
(assert-eql 3 (cl-csv::read-into-buffer-until s in :nl #\newline))
372
(assert-error 'end-of-file (cl-csv::read-into-buffer-until s in :nl #\newline))
373
)))
374
375
(define-test buffer-spanning-new-lines
376
(:tags '(read-until whitespace parsing))
377
(with-input-from-string (in "testRNtest")
378
(let* ((s (make-string 5))
379
len)
380
(setf len
381
(cl-csv::read-into-buffer-until s in :nl "RN"))
382
(assert-eql 5 len)
383
(setf len
384
(cl-csv::read-into-buffer-until
385
s in :nl "RN"
386
:nl-match 0))
387
(assert-eql 1 len )
388
(setf len
389
(cl-csv::read-into-buffer-until s in :nl "RN"))
390
(assert-eql 4 len))))
391
392
(define-test buffer-spanning-new-lines2
393
(:tags '(read-until newlines whitespace parsing))
394
;; ** newline
395
(with-input-from-string (in "test**tes**te**test")
396
(let* ((s (make-string 5))
397
len (nl-idx -1))
398
(flet ((rebind ( &optional new-nl-idx)
399
(when new-nl-idx
400
(setf nl-idx new-nl-idx))
401
(multiple-value-setq
402
(len)
403
(cl-csv::read-into-buffer-until
404
s in :nl "**"
405
:nl-match nl-idx))))
406
(rebind)
407
(assert-eql 5 len :first s)
408
(rebind 0)
409
(assert-eql 1 len :second)
410
(rebind -1)
411
(assert-eql 5 len :third)
412
(rebind)
413
(assert-eql 4 len :third))
414
)))
415
416
(define-test different-newlines (:tags '(read-until newlines whitespace parsing))
417
(with-input-from-string (in "a|b|c|d**1|2|3|4")
418
(let* ((cl-csv::*buffer-size* 8)
419
(rows (cl-csv:read-csv in :newline "**" :separator "|")))
420
(assert-equal 2 (length rows))
421
(assert-equal '("a" "b" "c" "d") (first rows))
422
(assert-equal '("1" "2" "3" "4") (second rows)))
423
)
424
(with-input-from-string (in "a|b|c|d**1|2|3|4")
425
(let ((rows (cl-csv:read-csv in :newline "**" :separator "|")))
426
(assert-equal 2 (length rows))
427
(assert-equal '("a" "b" "c" "d") (first rows))
428
(assert-equal '("1" "2" "3" "4") (second rows)))
429
)
430
(with-input-from-string (in "a|b|c|d*1|2|3|4")
431
(let ((rows (cl-csv:read-csv in :newline #\* :separator #\|)))
432
(assert-equal 2 (length rows))
433
(assert-equal '("a" "b" "c" "d") (first rows))
434
(assert-equal '("1" "2" "3" "4") (second rows)))
435
)
436
(with-input-from-string (in "a|b|c|d*1|2|3|4")
437
(let ((rows (cl-csv:read-csv in :newline "*" :separator "|")))
438
(assert-equal 2 (length rows))
439
(assert-equal '("a" "b" "c" "d") (first rows))
440
(assert-equal '("1" "2" "3" "4") (second rows)))
441
))
442
443
444
Loggerhead is a web-based interface for Breezy
Version: 2.0.1