Lumiera  0.pre.03
»edit your freedom«
data-csv-test.cpp
Go to the documentation of this file.
1 /*
2  DataCSV(Test) - verify data table with CSV storage support
3 
4  Copyright (C)
5  2009, Hermann Vosseler <Ichthyostega@web.de>
6 
7   **Lumiera** is free software; you can redistribute it and/or modify it
8   under the terms of the GNU General Public License as published by the
9   Free Software Foundation; either version 2 of the License, or (at your
10   option) any later version. See the file COPYING for further details.
11 
12 * *****************************************************************/
13 
19 #include "lib/test/run.hpp"
20 #include "lib/test/test-helper.hpp"
21 #include "lib/test/temp-dir.hpp"
22 #include "lib/stat/data.hpp"
23 #include "lib/time/timevalue.hpp"
24 #include "lib/format-cout.hpp"
25 #include "lib/util.hpp"
26 
27 #include <sstream>
28 #include <string>
29 #include <vector>
30 
31 using util::isnil;
32 using lib::time::Time;
33 using lib::test::TempDir;
34 using std::make_tuple;
35 using std::string;
36 using std::vector;
37 
38 
39 
40 namespace lib {
41 namespace stat{
42 namespace test{
43 
44  namespace {//Setup for test
45 
46 
48  struct TableForm
49  {
50  Column<string> id{"ID"}; // ◁────── names given here must match first storage line
51  Column<double> val{"Value"};
52  Column<int> off{"Offset"};
53 
54  auto allColumns() // ◁────────── mandatory function; defines actual sequence of columns
55  { return std::tie(id
56  ,val
57  ,off
58  );
59  }
60  };
61 
63 
64  }//(End)Test setup
65 
66  using error::LUMIERA_ERROR_STATE;
67 
68 
69 
70  /***********************************************************/
77  class DataCSV_test : public Test
78  {
79  void
80  run (Arg)
81  {
82  simpleUsage();
83  verify_rowHandling();
84  verify_CSV_Format();
85  verify_persistentDataFile();
86  demonnstrate_CSV_Notation();
87  }
88 
89 
91  void
93  {
94  TestTab tab;
95  CHECK (isnil (tab));
96  tab.newRow();
97  CHECK (not isnil (tab));
98  CHECK (1 == tab.size());
99  CHECK ( "" == string{tab.id});
100  CHECK (0.0 == tab.val);
101  CHECK ( 0 == tab.off);
102  tab.id = "one";
103  tab.val = 1.0;
104 
105  tab.dupRow();
106  CHECK (2 == tab.size());
107  CHECK ("one" == string{tab.id});
108  CHECK ( 1.0 == tab.val);
109  CHECK ( 0 == tab.off);
110 
111  tab.id = "two";
112  tab.val = 5.0;
113  tab.off = -23;
114  CHECK ("two" == string{tab.id});
115  CHECK ( 5.0 == tab.val);
116  CHECK ( -23 == tab.off);
117 
118  CHECK (tab.off.header == "Offset");
119  CHECK (tab.off.data == vector({0,-23}));
120  }
121 
122 
123 
124 
125  void
126  verify_rowHandling()
127  {
128  TestTab tab;
129  CHECK (3 == tab.columnCnt);
130 
131  CHECK (isnil (tab));
132  CHECK (0 == tab.size());
133  CHECK (0 == tab.id.data.size());
134  CHECK (0 == tab.val.data.size());
135  CHECK (0 == tab.off.data.size());
136  CHECK ("ID" == tab.id.header);
137  CHECK ("Value" == tab.val.header);
138  CHECK ("Offset" == tab.off.header);
139 
140  VERIFY_ERROR (STATE, tab.id.get() );
141  VERIFY_ERROR (STATE, tab.val.get());
142  VERIFY_ERROR (STATE, tab.off.get());
143  VERIFY_ERROR (STATE, tab.off = 5 );
144  VERIFY_ERROR (STATE,(tab.off == 5));
145 
146  // direct access to the data is possible and tolerated
147  tab.val.data.push_back (5.5);
148  CHECK (tab.val == 5.5);
149  VERIFY_ERROR (STATE, (tab.off == 5));
150  CHECK (1 == tab.val.data.size());
151  CHECK (0 == tab.off.data.size());
152  CHECK (0 == tab.id.data.size());
153  CHECK (0 == tab.size());
154  CHECK (isnil (tab));
155 
156  tab.newRow();
157  CHECK ( "" == string{tab.id});
158  CHECK (5.5 == tab.val);
159  CHECK ( 0 == tab.off);
160  CHECK (1 == tab.val.data.size());
161  CHECK (1 == tab.off.data.size());
162  CHECK (1 == tab.id.data.size());
163  CHECK (1 == tab.size());
164  CHECK (not isnil (tab));
165  CHECK (tab.off.data == vector({0}));
166  CHECK (tab.val.data == vector({5.5}));
167 
168  tab.allColumns() = make_tuple("●", 2.3, -11);
169  CHECK ("●" == string{tab.id});
170  CHECK (2.3 == tab.val);
171  CHECK (-11 == tab.off);
172 
173  tab.dupRow();
174  tab.val = 42;
175  tab.id = "◆";
176  CHECK (tab.off.data == vector({-11,-11}));
177  CHECK (tab.val.data == vector({2.3,42.0}));
178  CHECK (tab.id.data == vector<string>({"●","◆"}));
179 
180  tab.reserve(100);
181  CHECK (tab.id.data.capacity() >= 100);
182  CHECK (tab.val.data.capacity() >= 100);
183  CHECK (tab.off.data.capacity() >= 100);
184  CHECK (tab.id.data.size() == 2);
185  CHECK (tab.val.data.size() == 2);
186  CHECK (tab.off.data.size() == 2);
187  CHECK (2 == tab.size());
188  CHECK ("◆" == string{tab.id});
189  CHECK ( 42 == tab.val);
190  CHECK (-11 == tab.off);
191 
192  meta::forEach (tab.allColumns()
193  ,[](auto& col){ col.data.resize(2); }
194  );
195  CHECK (2 == tab.size());
196  CHECK ("◆" == string{tab.id});
197  CHECK ( 42 == tab.val);
198  CHECK (-11 == tab.off);
199 
200  tab.dropLastRow();
201  CHECK (1 == tab.size());
202  CHECK ("●" == string{tab.id});
203  CHECK (2.3 == tab.val);
204  CHECK (-11 == tab.off);
205  CHECK (tab.val.data.size() == 1);
206  CHECK (tab.val.data.capacity() >= 100);
207 
208  tab.clear();
209  CHECK (isnil (tab));
210  CHECK (tab.val.data.size() == 0);
211  CHECK (tab.val.data.capacity() >= 100);
212  }
213 
214 
216  void
218  {
219  double val = 1.0 / 3;
220  CHECK (util::toString(val) == "0.33333333"_expect );
221  CHECK (util::showDecimal(val) == "0.333333333333333"_expect );
222  CHECK (util::showComplete(val) == "0.33333333333333331"_expect);
223  CHECK (boost::lexical_cast<string>(val) == "0.33333333333333331"_expect);
224 
225  CHECK (format4Csv(double(1) / 3) == "0.333333333333333"_expect );
226  CHECK (format4Csv(float(1) / 3) == "0.333333"_expect );
227  CHECK (format4Csv(f128(1) / 3) == "0.333333333333333333"_expect);
228  CHECK (format4Csv(bool(1)) == "true"_expect );
229  CHECK (format4Csv(bool(0)) == "false"_expect);
230  CHECK (format4Csv("Starship-3") == "\"Starship-3\""_expect ); // 3rd test today ;-)
231  CHECK (format4Csv(Time(1,2,25,13)) == "\"13:25:02.001\""_expect);
232 
233 
234  string line;
235  int64_t ii = -100000;
236  bool boo = true;
237 
238  appendCsvField (line, ii);
239  CHECK (line == "-100000"_expect);
240  appendCsvField (line, val);
241  CHECK (line == "-100000,0.333333333333333"_expect);
242  appendCsvField (line, boo);
243  CHECK (line == "-100000,0.333333333333333,true"_expect);
244  appendCsvField (line, "Raptor");
245  CHECK (line == "-100000,0.333333333333333,true,\"Raptor\""_expect);
246 
247 
248  CsvParser parse{line};
249  CHECK (parse.isValid());
250  CHECK (*parse == "-100000"_expect);
251  CHECK (-100000 == parseAs<int>(*parse));
252  ++parse;
253  CHECK (parse.isValid());
254  CHECK (*parse == "0.333333333333333"_expect);
255  CHECK (0.333333343f == parseAs<float>(*parse));
256  ++parse;
257  CHECK (parse.isValid());
258 
259  CHECK (*parse == "true"_expect);
260  CHECK (true == parseAs<bool>(*parse));
261  ++parse;
262  CHECK (parse.isValid());
263  CHECK (*parse == "Raptor"_expect);
264  CHECK ("Raptor" == parseAs<string>(*parse));
265  ++parse;
266  CHECK (not parse.isValid());
267 
268  line = " ◐0◑. ; \t \"' \" \n ,oh my ;";
269  CsvParser horror{line};
270  CHECK ("◐0◑." == *horror); // as far as our CSV format is concerned, this is valid
271  CHECK (0 == horror.getParsedFieldCnt());
272  ++horror;
273  CHECK (1 == horror.getParsedFieldCnt());
274  CHECK ("' " == *horror);
275  ++horror;
276  CHECK ("oh" == *horror);
277  CHECK (2 == horror.getParsedFieldCnt());
278 
279  // next field is not quoted, but contains space
280  VERIFY_FAIL (",oh |↯|my ;", ++horror );
281 
282  CHECK (not horror.isValid());
283  CHECK (horror.isParseFail());
284 
285  // CsvParser is a »Lumiera Forward Iterator«
287  }
288 
289 
291  void
293  {
294  TempDir temp;
295  // prepare a data file to load into the table...
296  fs::path f = temp.makeFile("dataz.csv");
297  std::ofstream content{f};
298  content << R"("ID", "Value", "Offset")"<<endl
299  << R"( "one" , 5.5 ; +1 )"<<endl
300  << R"(;" 0 ";0)" <<endl; // ◁────── demonstrating some leeway in storage format
301  content.close();
302 
303  TestTab dat{f};
304  CHECK (2 == dat.size());
305  CHECK ("ID" == dat.id.header);
306  CHECK ("Value" == dat.val.header);
307  CHECK ("Offset" == dat.off.header);
308  //Note: data is reversed in storage — last/newest line first
309  CHECK ("one" == string{dat.id});
310  CHECK ( 5.5 == dat.val);
311  CHECK ( 1 == dat.off);
312  CHECK (dat.id.data == vector<string>({"","one"}));
313  CHECK (dat.val.data == vector<double>({0 ,5.5 }));
314  CHECK (dat.off.data == vector<int> ({0 ,1 }));
315 
316  // can modify some values....
317  dat.id = "mid";
318  dat.dupRow();
319  dat.id = "last";
320  dat.off *= -1;
321  // can render the contents as CSV
322  CHECK (dat.renderCSV() ==
323 R"("ID","Value","Offset"
324 "",0,0
325 "mid",5.5,1
326 "last",5.5,-1
327 )"_expect);
328 
329  // save complete table in current state, overwriting on disk
330  dat.save();
331 
332  // read back data rewritten on disk...
333  std::ifstream readback{f};
334  std::ostringstream inBuff;
335  inBuff << readback.rdbuf();
336  CHECK (inBuff.str() ==
337 R"("ID","Value","Offset"
338 "last",5.5,-1
339 "mid",5.5,1
340 "",0,0
341 )"_expect);
342  // note again the reversed order in storage: last line at top
343  }
344 
345 
346 
348  void
350  {
351  CHECK (CSVLine(1,"2",3.4,5555/55) == "1,\"2\",3.4,101"_expect);
352  CHECK (CSVLine(string{"himself"}) == "\"himself\""_expect);
353  CHECK (CSVLine{CSVLine{1e9}} == "1000000000"_expect);
354  CHECK (CSVLine{} == ""_expect);
355 
356  auto appended = (CSVLine{} += 5.5) += Symbol();
357  CHECK (appended == "5.5,\"⟂\""_expect);
358 
359  CHECK (CSVData({"eeny","meeny","miny","moe"}) == "\"eeny\",\"meeny\",\"miny\",\"moe\"\n"_expect);
360  CHECK (CSVData({"eeny , meeny","miny","moe"}) == "\"eeny , meeny\"\n\"miny\"\n\"moe\"\n"_expect); // you dirty dirty dishrag you
361 
362  auto csv = CSVData{{"la","la","schland"}
363  ,{{3.2,1l,88}
364  ,{"mit", string{"mia"}, Literal("ned")}
365  ,CSVLine(string(";"))
366  ,{false}
367  ,{}
368  }};
369  CHECK (csv.size() == 6);
370  CHECK (string(csv) ==
371 R"("la","la","schland"
372 3.2,1,88
373 "mit","mia","ned"
374 ";"
375 false
376 
377 )"_expect);
378 
379  VERIFY_FAIL ("Header mismatch in CSV file", TestTab{csv} );
380 
381  csv = CSVData{{"ID","Value","Offset"}
382  ,{{"Baby","toe"}
383  }};
384  VERIFY_FAIL ("unable to parse \"toe\"", TestTab{csv} );
385 
386  csv = CSVData{{"ID","Value","Offset"}
387  ,{{"Baby",1.6180,23}
388  ,{"Tiger",10101,-5}
389  }};
390  TestTab dat{csv};
391  CHECK (dat.val == 1.0101e4);
392  CHECK (dat.renderCSV() == string(csv));
393  }
394  };
395 
396  LAUNCHER (DataCSV_test, "unit calculation");
397 
398 
399 }}} // namespace lib::stat::test
Automatically use custom string conversion in C++ stream output.
Wrapper to simplify notation in tests.
Definition: csv.hpp:140
Definition: run.hpp:40
void appendCsvField(string &csv, VAL const &val)
Format and append a data value to a CSV string representation.
Definition: csv.hpp:94
inline string literal This is a marker type to indicate that
Definition: symbol.hpp:76
A string with the ability to construct or append the CSV-rendering of data fields.
Definition: csv.hpp:105
#define VERIFY_ERROR(ERROR_ID, ERRONEOUS_STATEMENT)
Macro to verify that a statement indeed raises an exception.
Manage a temporary directory for storage, with automated clean-up.
Implementation namespace for support and library code.
Lumiera&#39;s internal time value datatype.
Definition: timevalue.hpp:299
Token or Atom with distinct identity.
Definition: symbol.hpp:117
Manage a table with data records, stored persistently as CSV.
Simplistic test class runner.
Tiny helper functions and shortcuts to be used everywhere Consider this header to be effectively incl...
Trait template to detect a type usable immediately as "Lumiera Forward Iterator" in a specialised for...
Definition: trait.hpp:510
A collection of frequently used helper functions to support unit testing.
A RAII style temporary directory.
Definition: temp-dir.hpp:54
Table with data values, stored persistently as CSV file.
Definition: data.hpp:185
a family of time value like entities and their relationships.
Parser to split one line of CSV data into fields.
Definition: csv.hpp:241
#define VERIFY_FAIL(FAILURE_MSG, ERRONEOUS_STATEMENT)
Macro to verify that a statement indeed raises a std::exception, which additionally contains some FAI...