test_tablib.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. """Tests for Tablib."""
  4. import json
  5. import unittest
  6. import sys
  7. import os
  8. import tablib
  9. from tablib.compat import markup, unicode, is_py3
  10. class TablibTestCase(unittest.TestCase):
  11. """Tablib test cases."""
  12. def setUp(self):
  13. """Create simple data set with headers."""
  14. global data, book
  15. data = tablib.Dataset()
  16. book = tablib.Databook()
  17. self.headers = ('first_name', 'last_name', 'gpa')
  18. self.john = ('John', 'Adams', 90)
  19. self.george = ('George', 'Washington', 67)
  20. self.tom = ('Thomas', 'Jefferson', 50)
  21. self.founders = tablib.Dataset(headers=self.headers, title='Founders')
  22. self.founders.append(self.john)
  23. self.founders.append(self.george)
  24. self.founders.append(self.tom)
  25. def tearDown(self):
  26. """Teardown."""
  27. pass
  28. def test_empty_append(self):
  29. """Verify append() correctly adds tuple with no headers."""
  30. new_row = (1, 2, 3)
  31. data.append(new_row)
  32. # Verify width/data
  33. self.assertTrue(data.width == len(new_row))
  34. self.assertTrue(data[0] == new_row)
  35. def test_empty_append_with_headers(self):
  36. """Verify append() correctly detects mismatch of number of
  37. headers and data.
  38. """
  39. data.headers = ['first', 'second']
  40. new_row = (1, 2, 3, 4)
  41. self.assertRaises(tablib.InvalidDimensions, data.append, new_row)
  42. def test_set_headers_with_incorrect_dimension(self):
  43. """Verify headers correctly detects mismatch of number of
  44. headers and data.
  45. """
  46. data.append(self.john)
  47. def set_header_callable():
  48. data.headers = ['first_name']
  49. self.assertRaises(tablib.InvalidDimensions, set_header_callable)
  50. def test_add_column(self):
  51. """Verify adding column works with/without headers."""
  52. data.append(['kenneth'])
  53. data.append(['bessie'])
  54. new_col = ['reitz', 'monke']
  55. data.append_col(new_col)
  56. self.assertEqual(data[0], ('kenneth', 'reitz'))
  57. self.assertEqual(data.width, 2)
  58. # With Headers
  59. data.headers = ('fname', 'lname')
  60. new_col = [21, 22]
  61. data.append_col(new_col, header='age')
  62. self.assertEqual(data['age'], new_col)
  63. def test_add_column_no_data_no_headers(self):
  64. """Verify adding new column with no headers."""
  65. new_col = ('reitz', 'monke')
  66. data.append_col(new_col)
  67. self.assertEqual(data[0], tuple([new_col[0]]))
  68. self.assertEqual(data.width, 1)
  69. self.assertEqual(data.height, len(new_col))
  70. def test_add_column_with_header_ignored(self):
  71. """Verify append_col() ignores the header if data.headers has
  72. not previously been set
  73. """
  74. new_col = ('reitz', 'monke')
  75. data.append_col(new_col, header='first_name')
  76. self.assertEqual(data[0], tuple([new_col[0]]))
  77. self.assertEqual(data.width, 1)
  78. self.assertEqual(data.height, len(new_col))
  79. self.assertEqual(data.headers, None)
  80. def test_add_column_with_header_and_headers_only_exist(self):
  81. """Verify append_col() with header correctly detects mismatch when
  82. headers exist but there is no existing row data
  83. """
  84. data.headers = ['first_name']
  85. #no data
  86. new_col = ('allen')
  87. def append_col_callable():
  88. data.append_col(new_col, header='middle_name')
  89. self.assertRaises(tablib.InvalidDimensions, append_col_callable)
  90. def test_add_column_with_header_and_data_exists(self):
  91. """Verify append_col() works when headers and rows exists"""
  92. data.headers = self.headers
  93. data.append(self.john)
  94. new_col = [10];
  95. data.append_col(new_col, header='age')
  96. self.assertEqual(data.height, 1)
  97. self.assertEqual(data.width, len(self.john) + 1)
  98. self.assertEqual(data['age'], new_col)
  99. self.assertEqual(len(data.headers), len(self.headers) + 1)
  100. def test_add_callable_column(self):
  101. """Verify adding column with values specified as callable."""
  102. new_col = lambda x: x[0]
  103. self.founders.append_col(new_col, header='first_again')
  104. def test_header_slicing(self):
  105. """Verify slicing by headers."""
  106. self.assertEqual(self.founders['first_name'],
  107. [self.john[0], self.george[0], self.tom[0]])
  108. self.assertEqual(self.founders['last_name'],
  109. [self.john[1], self.george[1], self.tom[1]])
  110. self.assertEqual(self.founders['gpa'],
  111. [self.john[2], self.george[2], self.tom[2]])
  112. def test_get_col(self):
  113. """Verify getting columns by index"""
  114. self.assertEqual(
  115. self.founders.get_col(list(self.headers).index('first_name')),
  116. [self.john[0], self.george[0], self.tom[0]])
  117. self.assertEqual(
  118. self.founders.get_col(list(self.headers).index('last_name')),
  119. [self.john[1], self.george[1], self.tom[1]])
  120. self.assertEqual(
  121. self.founders.get_col(list(self.headers).index('gpa')),
  122. [self.john[2], self.george[2], self.tom[2]])
  123. def test_data_slicing(self):
  124. """Verify slicing by data."""
  125. # Slice individual rows
  126. self.assertEqual(self.founders[0], self.john)
  127. self.assertEqual(self.founders[:1], [self.john])
  128. self.assertEqual(self.founders[1:2], [self.george])
  129. self.assertEqual(self.founders[-1], self.tom)
  130. self.assertEqual(self.founders[3:], [])
  131. # Slice multiple rows
  132. self.assertEqual(self.founders[:], [self.john, self.george, self.tom])
  133. self.assertEqual(self.founders[0:2], [self.john, self.george])
  134. self.assertEqual(self.founders[1:3], [self.george, self.tom])
  135. self.assertEqual(self.founders[2:], [self.tom])
  136. def test_delete(self):
  137. """Verify deleting from dataset works."""
  138. # Delete from front of object
  139. del self.founders[0]
  140. self.assertEqual(self.founders[:], [self.george, self.tom])
  141. # Verify dimensions, width should NOT change
  142. self.assertEqual(self.founders.height, 2)
  143. self.assertEqual(self.founders.width, 3)
  144. # Delete from back of object
  145. del self.founders[1]
  146. self.assertEqual(self.founders[:], [self.george])
  147. # Verify dimensions, width should NOT change
  148. self.assertEqual(self.founders.height, 1)
  149. self.assertEqual(self.founders.width, 3)
  150. # Delete from invalid index
  151. self.assertRaises(IndexError, self.founders.__delitem__, 3)
  152. def test_csv_export(self):
  153. """Verify exporting dataset object as CSV."""
  154. # Build up the csv string with headers first, followed by each row
  155. csv = ''
  156. for col in self.headers:
  157. csv += col + ','
  158. csv = csv.strip(',') + '\r\n'
  159. for founder in self.founders:
  160. for col in founder:
  161. csv += str(col) + ','
  162. csv = csv.strip(',') + '\r\n'
  163. self.assertEqual(csv, self.founders.csv)
  164. def test_tsv_export(self):
  165. """Verify exporting dataset object as TSV."""
  166. # Build up the tsv string with headers first, followed by each row
  167. tsv = ''
  168. for col in self.headers:
  169. tsv += col + '\t'
  170. tsv = tsv.strip('\t') + '\r\n'
  171. for founder in self.founders:
  172. for col in founder:
  173. tsv += str(col) + '\t'
  174. tsv = tsv.strip('\t') + '\r\n'
  175. self.assertEqual(tsv, self.founders.tsv)
  176. def test_html_export(self):
  177. """HTML export"""
  178. html = markup.page()
  179. html.table.open()
  180. html.thead.open()
  181. html.tr(markup.oneliner.th(self.founders.headers))
  182. html.thead.close()
  183. for founder in self.founders:
  184. html.tr(markup.oneliner.td(founder))
  185. html.table.close()
  186. html = str(html)
  187. self.assertEqual(html, self.founders.html)
  188. def test_html_export_none_value(self):
  189. """HTML export"""
  190. html = markup.page()
  191. html.table.open()
  192. html.thead.open()
  193. html.tr(markup.oneliner.th(['foo','', 'bar']))
  194. html.thead.close()
  195. html.tr(markup.oneliner.td(['foo','', 'bar']))
  196. html.table.close()
  197. html = str(html)
  198. headers = ['foo', None, 'bar'];
  199. d = tablib.Dataset(['foo', None, 'bar'], headers=headers)
  200. self.assertEqual(html, d.html)
  201. def test_unicode_append(self):
  202. """Passes in a single unicode character and exports."""
  203. if is_py3:
  204. new_row = ('å', 'é')
  205. else:
  206. exec("new_row = (u'å', u'é')")
  207. data.append(new_row)
  208. data.json
  209. data.yaml
  210. data.csv
  211. data.tsv
  212. data.xls
  213. data.xlsx
  214. data.ods
  215. data.html
  216. def test_book_export_no_exceptions(self):
  217. """Test that various exports don't error out."""
  218. book = tablib.Databook()
  219. book.add_sheet(data)
  220. book.json
  221. book.yaml
  222. book.xls
  223. book.xlsx
  224. book.ods
  225. def test_json_import_set(self):
  226. """Generate and import JSON set serialization."""
  227. data.append(self.john)
  228. data.append(self.george)
  229. data.headers = self.headers
  230. _json = data.json
  231. data.json = _json
  232. self.assertEqual(json.loads(_json), json.loads(data.json))
  233. def test_json_import_book(self):
  234. """Generate and import JSON book serialization."""
  235. data.append(self.john)
  236. data.append(self.george)
  237. data.headers = self.headers
  238. book.add_sheet(data)
  239. _json = book.json
  240. book.json = _json
  241. self.assertEqual(json.loads(_json), json.loads(book.json))
  242. def test_yaml_import_set(self):
  243. """Generate and import YAML set serialization."""
  244. data.append(self.john)
  245. data.append(self.george)
  246. data.headers = self.headers
  247. _yaml = data.yaml
  248. data.yaml = _yaml
  249. self.assertEqual(_yaml, data.yaml)
  250. def test_yaml_import_book(self):
  251. """Generate and import YAML book serialization."""
  252. data.append(self.john)
  253. data.append(self.george)
  254. data.headers = self.headers
  255. book.add_sheet(data)
  256. _yaml = book.yaml
  257. book.yaml = _yaml
  258. self.assertEqual(_yaml, book.yaml)
  259. def test_csv_import_set(self):
  260. """Generate and import CSV set serialization."""
  261. data.append(self.john)
  262. data.append(self.george)
  263. data.headers = self.headers
  264. _csv = data.csv
  265. data.csv = _csv
  266. self.assertEqual(_csv, data.csv)
  267. def test_csv_import_set_with_spaces(self):
  268. """Generate and import CSV set serialization when row values have
  269. spaces."""
  270. data.append(('Bill Gates', 'Microsoft'))
  271. data.append(('Steve Jobs', 'Apple'))
  272. data.headers = ('Name', 'Company')
  273. _csv = data.csv
  274. data.csv = _csv
  275. self.assertEqual(_csv, data.csv)
  276. def test_csv_import_set_with_newlines(self):
  277. """Generate and import CSV set serialization when row values have
  278. newlines."""
  279. data.append(('Markdown\n=======',
  280. 'A cool language\n\nwith paragraphs'))
  281. data.append(('reStructedText\n==============',
  282. 'Another cool language\n\nwith paragraphs'))
  283. data.headers = ('title', 'body')
  284. _csv = data.csv
  285. data.csv = _csv
  286. self.assertEqual(_csv, data.csv)
  287. def test_tsv_import_set(self):
  288. """Generate and import TSV set serialization."""
  289. data.append(self.john)
  290. data.append(self.george)
  291. data.headers = self.headers
  292. _tsv = data.tsv
  293. data.tsv = _tsv
  294. self.assertEqual(_tsv, data.tsv)
  295. def test_csv_format_detect(self):
  296. """Test CSV format detection."""
  297. _csv = (
  298. '1,2,3\n'
  299. '4,5,6\n'
  300. '7,8,9\n'
  301. )
  302. _bunk = (
  303. '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶'
  304. )
  305. self.assertTrue(tablib.formats.csv.detect(_csv))
  306. self.assertFalse(tablib.formats.csv.detect(_bunk))
  307. def test_tsv_format_detect(self):
  308. """Test TSV format detection."""
  309. _tsv = (
  310. '1\t2\t3\n'
  311. '4\t5\t6\n'
  312. '7\t8\t9\n'
  313. )
  314. _bunk = (
  315. '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶'
  316. )
  317. self.assertTrue(tablib.formats.tsv.detect(_tsv))
  318. self.assertFalse(tablib.formats.tsv.detect(_bunk))
  319. def test_json_format_detect(self):
  320. """Test JSON format detection."""
  321. _json = '[{"last_name": "Adams","age": 90,"first_name": "John"}]'
  322. _bunk = (
  323. '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶'
  324. )
  325. self.assertTrue(tablib.formats.json.detect(_json))
  326. self.assertFalse(tablib.formats.json.detect(_bunk))
  327. def test_yaml_format_detect(self):
  328. """Test YAML format detection."""
  329. _yaml = '- {age: 90, first_name: John, last_name: Adams}'
  330. _tsv = 'foo\tbar'
  331. _bunk = (
  332. '¡¡¡¡¡¡---///\n\n\n¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶'
  333. )
  334. self.assertTrue(tablib.formats.yaml.detect(_yaml))
  335. self.assertFalse(tablib.formats.yaml.detect(_bunk))
  336. self.assertFalse(tablib.formats.yaml.detect(_tsv))
  337. def test_auto_format_detect(self):
  338. """Test auto format detection."""
  339. _yaml = '- {age: 90, first_name: John, last_name: Adams}'
  340. _json = '[{"last_name": "Adams","age": 90,"first_name": "John"}]'
  341. _csv = '1,2,3\n4,5,6\n7,8,9\n'
  342. _tsv = '1\t2\t3\n4\t5\t6\n7\t8\t9\n'
  343. _bunk = '¡¡¡¡¡¡---///\n\n\n¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶'
  344. self.assertEqual(tablib.detect(_yaml)[0], tablib.formats.yaml)
  345. self.assertEqual(tablib.detect(_csv)[0], tablib.formats.csv)
  346. self.assertEqual(tablib.detect(_tsv)[0], tablib.formats.tsv)
  347. self.assertEqual(tablib.detect(_json)[0], tablib.formats.json)
  348. self.assertEqual(tablib.detect(_bunk)[0], None)
  349. def test_transpose(self):
  350. """Transpose a dataset."""
  351. transposed_founders = self.founders.transpose()
  352. first_row = transposed_founders[0]
  353. second_row = transposed_founders[1]
  354. self.assertEqual(transposed_founders.headers,
  355. ["first_name","John", "George", "Thomas"])
  356. self.assertEqual(first_row,
  357. ("last_name","Adams", "Washington", "Jefferson"))
  358. self.assertEqual(second_row,
  359. ("gpa",90, 67, 50))
  360. def test_row_stacking(self):
  361. """Row stacking."""
  362. to_join = tablib.Dataset(headers=self.founders.headers)
  363. for row in self.founders:
  364. to_join.append(row=row)
  365. row_stacked = self.founders.stack(to_join)
  366. for column in row_stacked.headers:
  367. original_data = self.founders[column]
  368. expected_data = original_data + original_data
  369. self.assertEqual(row_stacked[column], expected_data)
  370. def test_column_stacking(self):
  371. """Column stacking"""
  372. to_join = tablib.Dataset(headers=self.founders.headers)
  373. for row in self.founders:
  374. to_join.append(row=row)
  375. column_stacked = self.founders.stack_cols(to_join)
  376. for index, row in enumerate(column_stacked):
  377. original_data = self.founders[index]
  378. expected_data = original_data + original_data
  379. self.assertEqual(row, expected_data)
  380. self.assertEqual(column_stacked[0],
  381. ("John", "Adams", 90, "John", "Adams", 90))
  382. def test_sorting(self):
  383. """Sort columns."""
  384. sorted_data = self.founders.sort(col="first_name")
  385. self.assertEqual(sorted_data.title, 'Founders')
  386. first_row = sorted_data[0]
  387. second_row = sorted_data[2]
  388. third_row = sorted_data[1]
  389. expected_first = self.founders[1]
  390. expected_second = self.founders[2]
  391. expected_third = self.founders[0]
  392. self.assertEqual(first_row, expected_first)
  393. self.assertEqual(second_row, expected_second)
  394. self.assertEqual(third_row, expected_third)
  395. def test_wipe(self):
  396. """Purge a dataset."""
  397. new_row = (1, 2, 3)
  398. data.append(new_row)
  399. # Verify width/data
  400. self.assertTrue(data.width == len(new_row))
  401. self.assertTrue(data[0] == new_row)
  402. data.wipe()
  403. new_row = (1, 2, 3, 4)
  404. data.append(new_row)
  405. self.assertTrue(data.width == len(new_row))
  406. self.assertTrue(data[0] == new_row)
  407. def test_formatters(self):
  408. """Confirm formatters are being triggered."""
  409. def _formatter(cell_value):
  410. return str(cell_value).upper()
  411. self.founders.add_formatter('last_name', _formatter)
  412. for name in [r['last_name'] for r in self.founders.dict]:
  413. self.assertTrue(name.isupper())
  414. def test_unicode_csv(self):
  415. """Check if unicode in csv export doesn't raise."""
  416. data = tablib.Dataset()
  417. if sys.version_info[0] > 2:
  418. data.append(['\xfc', '\xfd'])
  419. else:
  420. exec("data.append([u'\xfc', u'\xfd'])")
  421. data.csv
  422. def test_csv_column_select(self):
  423. """Build up a CSV and test selecting a column"""
  424. data = tablib.Dataset()
  425. data.csv = self.founders.csv
  426. headers = data.headers
  427. self.assertTrue(isinstance(headers[0], unicode))
  428. orig_first_name = self.founders[self.headers[0]]
  429. csv_first_name = data[headers[0]]
  430. self.assertEqual(orig_first_name, csv_first_name)
  431. def test_csv_column_delete(self):
  432. """Build up a CSV and test deleting a column"""
  433. data = tablib.Dataset()
  434. data.csv = self.founders.csv
  435. target_header = data.headers[0]
  436. self.assertTrue(isinstance(target_header, unicode))
  437. del data[target_header]
  438. self.assertTrue(target_header not in data.headers)
  439. def test_csv_column_sort(self):
  440. """Build up a CSV and test sorting a column by name"""
  441. data = tablib.Dataset()
  442. data.csv = self.founders.csv
  443. orig_target_header = self.founders.headers[1]
  444. target_header = data.headers[1]
  445. self.founders.sort(orig_target_header)
  446. data.sort(target_header)
  447. self.assertEqual(self.founders[orig_target_header], data[target_header])
  448. def test_unicode_renders_markdown_table(self):
  449. # add another entry to test right field width for
  450. # integer
  451. self.founders.append(('Old', 'Man', 100500))
  452. self.assertEqual(
  453. """
  454. first_name|last_name |gpa
  455. ----------|----------|------
  456. John |Adams |90
  457. George |Washington|67
  458. Thomas |Jefferson |50
  459. Old |Man |100500
  460. """.strip(),
  461. unicode(self.founders)
  462. )
  463. def test_databook_add_sheet_accepts_only_dataset_instances(self):
  464. class NotDataset(object):
  465. def append(self, item):
  466. pass
  467. dataset = NotDataset()
  468. dataset.append(self.john)
  469. self.assertRaises(tablib.InvalidDatasetType, book.add_sheet, dataset)
  470. def test_databook_add_sheet_accepts_dataset_subclasses(self):
  471. class DatasetSubclass(tablib.Dataset):
  472. pass
  473. # just checking if subclass of tablib.Dataset can be added to Databook
  474. dataset = DatasetSubclass()
  475. dataset.append(self.john)
  476. dataset.append(self.tom)
  477. try:
  478. book.add_sheet(dataset)
  479. except tablib.InvalidDatasetType:
  480. self.fail("Subclass of tablib.Dataset should be accepted by Databook.add_sheet")
  481. if __name__ == '__main__':
  482. unittest.main()