openMSX
serialize.cc
Go to the documentation of this file.
1 #include "serialize.hh"
2 #include "Base64.hh"
3 #include "HexDump.hh"
4 #include "XMLLoader.hh"
5 #include "XMLElement.hh"
6 #include "ConfigException.hh"
7 #include "XMLException.hh"
8 #include "snappy.hh"
9 #include "MemBuffer.hh"
10 #include "StringOp.hh"
11 #include "FileOperations.hh"
12 #include "Version.hh"
13 #include "Date.hh"
14 #include <cstring>
15 #include <limits>
16 
17 using std::string;
18 
19 namespace openmsx {
20 
21 template<typename Derived>
22 void ArchiveBase<Derived>::attribute(const char* name, const char* value)
23 {
24  string valueStr(value);
25  self().attribute(name, valueStr);
26 }
27 template class ArchiveBase<MemOutputArchive>;
28 template class ArchiveBase<XmlOutputArchive>;
29 
31 
33  : lastId(0)
34 {
35 }
36 
37 unsigned OutputArchiveBase2::generateID1(const void* p)
38 {
39  #ifdef linux
40  assert("Can't serialize ID of object located on the stack" &&
41  !addressOnStack(p));
42  #endif
43  ++lastId;
44  assert(polyIdMap.find(p) == end(polyIdMap));
45  polyIdMap[p] = lastId;
46  return lastId;
47 }
48 unsigned OutputArchiveBase2::generateID2(
49  const void* p, const std::type_info& typeInfo)
50 {
51  #ifdef linux
52  assert("Can't serialize ID of object located on the stack" &&
53  !addressOnStack(p));
54  #endif
55  ++lastId;
56  auto key = std::make_pair(p, std::type_index(typeInfo));
57  assert(idMap.find(key) == end(idMap));
58  idMap[key] = lastId;
59  return lastId;
60 }
61 
62 unsigned OutputArchiveBase2::getID1(const void* p)
63 {
64  auto it = polyIdMap.find(p);
65  return it != end(polyIdMap) ? it->second : 0;
66 }
67 unsigned OutputArchiveBase2::getID2(
68  const void* p, const std::type_info& typeInfo)
69 {
70  auto it = idMap.find({p, std::type_index(typeInfo)});
71  return it != end(idMap) ? it->second : 0;
72 }
73 
74 
75 template<typename Derived>
77  const char* tag, const void* data, size_t len)
78 {
79  string encoding;
80  string tmp;
81  if (false) {
82  // useful for debugging
83  encoding = "hex";
84  tmp = HexDump::encode(data, len);
85  } else if (false) {
86  encoding = "base64";
87  tmp = Base64::encode(data, len);
88  } else {
89  encoding = "gz-base64";
90  // TODO check for overflow?
91  auto dstLen = uLongf(len + len / 1000 + 12 + 1); // worst-case
92  MemBuffer<byte> buf(dstLen);
93  if (compress2(buf.data(), &dstLen,
94  reinterpret_cast<const Bytef*>(data),
95  uLong(len), 9)
96  != Z_OK) {
97  throw MSXException("Error while compressing blob.");
98  }
99  tmp = Base64::encode(buf.data(), dstLen);
100  }
101  this->self().beginTag(tag);
102  this->self().attribute("encoding", encoding);
103  Saver<string> saver;
104  saver(this->self(), tmp, false);
105  this->self().endTag(tag);
106 }
107 
110 
112 
114 {
115  auto it = idMap.find(id);
116  return it != end(idMap) ? it->second : nullptr;
117 }
118 
119 void InputArchiveBase2::addPointer(unsigned id, const void* p)
120 {
121  assert(idMap.find(id) == end(idMap));
122  idMap[id] = const_cast<void*>(p);
123 }
124 
125 template<typename Derived>
127  const char* tag, void* data, size_t len)
128 {
129  this->self().beginTag(tag);
130  string encoding;
131  this->self().attribute("encoding", encoding);
132 
133  string tmp;
134  Loader<string> loader;
135  loader(this->self(), tmp, std::make_tuple(), -1);
136  this->self().endTag(tag);
137 
138  if (encoding == "gz-base64") {
139  tmp = Base64::decode(tmp);
140  auto dstLen = uLongf(len); // TODO check for overflow?
141  if ((uncompress(reinterpret_cast<Bytef*>(data), &dstLen,
142  reinterpret_cast<const Bytef*>(tmp.data()), uLong(tmp.size()))
143  != Z_OK) ||
144  (dstLen != len)) {
145  throw MSXException("Error while decompressing blob.");
146  }
147  } else if ((encoding == "hex") || (encoding == "base64")) {
148  if (encoding == "hex") {
149  tmp = HexDump::decode(tmp);
150  } else {
151  tmp = Base64::decode(tmp);
152  }
153  if (tmp.size() != len) {
155  << "Length of decoded blob: " << tmp.size()
156  << " not identical to expected value: " << len);
157  }
158  memcpy(data, tmp.data(), len);
159  } else {
160  throw XMLException("Unsupported encoding \"" + encoding + "\" for blob");
161  }
162 }
163 
164 template class InputArchiveBase<MemInputArchive>;
165 template class InputArchiveBase<XmlInputArchive>;
166 
168 
169 void MemOutputArchive::save(const std::string& s)
170 {
171  auto size = s.size();
172  byte* buf = buffer.allocate(sizeof(size) + size);
173  memcpy(buf, &size, sizeof(size));
174  memcpy(buf + sizeof(size), s.data(), size);
175 }
176 
178 {
179  size_t size;
180  byte* data = buffer.release(size);
181  return MemBuffer<byte>(data, size);
182 }
183 
185 
186 void MemInputArchive::load(std::string& s)
187 {
188  size_t length;
189  load(length);
190  s.resize(length);
191  if (length) {
192  get(&s[0], length);
193  }
194 }
195 
197 
198 // Too small inputs don't compress very well (often the compressed size is even
199 // bigger than the input). It also takes a relatively long time (because snappy
200 // has a relatively large setup time). I choose this value semi-arbitrary. I
201 // only made it >= 52 so that the (incompressible) RP5C01 registers won't be
202 // compressed.
203 static const size_t SMALL_SIZE = 100;
204 void MemOutputArchive::serialize_blob(const char*, const void* data, size_t len)
205 {
206  // Compress in-memory blobs:
207  //
208  // This is a bit slower than memcpy, but it uses a lot less memory.
209  // Memory usage is important for the reverse feature, where we keep a
210  // lot of savestates in memory.
211  //
212  // I compared 'gzip level=1' (fastest version with lowest compression
213  // ratio) with 'lzo'. lzo was considerably faster. Compression ratio
214  // was about the same (maybe lzo was slightly better (OTOH on higher
215  // levels gzip compresses better)). So I decided to go with lzo.
216  //
217  // Later I compared 'lzo' with 'snappy', lzo compresses 6-25% better,
218  // but 'snappy' is about twice as fast. So I switched to 'snappy'.
219  if (len >= SMALL_SIZE) {
220  size_t dstLen = snappy::maxCompressedLength(len);
221  byte* buf = buffer.allocate(sizeof(dstLen) + dstLen);
222  snappy::compress(static_cast<const char*>(data), len,
223  reinterpret_cast<char*>(&buf[sizeof(dstLen)]), dstLen);
224  memcpy(buf, &dstLen, sizeof(dstLen)); // fill-in actual size
225  buffer.deallocate(&buf[sizeof(dstLen) + dstLen]); // dealloc unused portion
226  } else {
227  byte* buf = buffer.allocate(len);
228  memcpy(buf, data, len);
229  }
230 
231 }
232 
233 void MemInputArchive::serialize_blob(const char*, void* data, size_t len)
234 {
235  if (len >= SMALL_SIZE) {
236  size_t srcLen; load(srcLen);
237  snappy::uncompress(reinterpret_cast<const char*>(buffer.getCurrentPos()),
238  srcLen, reinterpret_cast<char*>(data), len);
239  buffer.skip(srcLen);
240  } else {
241  memcpy(data, buffer.getCurrentPos(), len);
242  buffer.skip(len);
243  }
244 }
245 
247 
248 XmlOutputArchive::XmlOutputArchive(const string& filename)
249  : root("serial")
250 {
251  root.addAttribute("openmsx_version", Version::full());
252  root.addAttribute("date_time", Date::toString(time(nullptr)));
253  root.addAttribute("platform", TARGET_PLATFORM);
254  auto f = FileOperations::openFile(filename, "wb");
255  if (!f) {
256  throw XMLException("Could not open compressed file \"" + filename + "\"");
257  }
258  file = gzdopen(fileno(f.get()), "wb9");
259  if (!file) {
260  throw XMLException("Could not open compressed file \"" + filename + "\"");
261  }
262  f.release();
263  current.push_back(&root);
264 }
265 
267 {
268  assert(current.back() == &root);
269  const char* header =
270  "<?xml version=\"1.0\" ?>\n"
271  "<!DOCTYPE openmsx-serialize SYSTEM 'openmsx-serialize.dtd'>\n";
272  gzwrite(file, const_cast<char*>(header), unsigned(strlen(header)));
273  string dump = root.dump();
274  gzwrite(file, const_cast<char*>(dump.data()), unsigned(dump.size()));
275  gzclose(file);
276 }
277 
279 {
280  save(string(1, c));
281 }
282 void XmlOutputArchive::save(const string& str)
283 {
284  assert(!current.empty());
285  assert(current.back()->getData().empty());
286  current.back()->setData(str);
287 }
289 {
290  assert(!current.empty());
291  assert(current.back()->getData().empty());
292  current.back()->setData(b ? "true" : "false");
293 }
294 void XmlOutputArchive::save(unsigned char b)
295 {
296  save(unsigned(b));
297 }
298 void XmlOutputArchive::save(signed char c)
299 {
300  save(int(c));
301 }
303 {
304  save(int(c));
305 }
307 {
308  saveImpl(i);
309 }
310 void XmlOutputArchive::save(unsigned u)
311 {
312  saveImpl(u);
313 }
314 void XmlOutputArchive::save(unsigned long long ull)
315 {
316  saveImpl(ull);
317 }
318 
319 void XmlOutputArchive::attribute(const char* name, const string& str)
320 {
321  assert(!current.empty());
322  assert(!current.back()->hasAttribute(name));
323  current.back()->addAttribute(name, str);
324 }
325 void XmlOutputArchive::attribute(const char* name, int i)
326 {
327  attributeImpl(name, i);
328 }
329 void XmlOutputArchive::attribute(const char* name, unsigned u)
330 {
331  attributeImpl(name, u);
332 }
333 
334 void XmlOutputArchive::beginTag(const char* tag)
335 {
336  assert(!current.empty());
337  auto& elem = current.back()->addChild(tag);
338  current.push_back(&elem);
339 }
340 void XmlOutputArchive::endTag(const char* tag)
341 {
342  assert(!current.empty());
343  assert(current.back()->getName() == tag); (void)tag;
344  current.pop_back();
345 }
346 
348 
349 XmlInputArchive::XmlInputArchive(const string& filename)
350  : elem(XMLLoader::load(filename, "openmsx-serialize.dtd"))
351 {
352  elems.emplace_back(&elem, 0);
353 }
354 
356 {
357  std::string str;
358  load(str);
359  std::istringstream is(str);
360  is >> c;
361 }
362 void XmlInputArchive::load(string& t)
363 {
364  if (!elems.back().first->getChildren().empty()) {
365  throw XMLException("No child tags expected for string types");
366  }
367  t = elems.back().first->getData();
368 }
370 {
371  if (!elems.back().first->getChildren().empty()) {
372  throw XMLException("No child tags expected for boolean types");
373  }
374  const auto& s = elems.back().first->getData();
375  if ((s == "true") || (s == "1")) {
376  b = true;
377  } else if ((s == "false") || (s == "0")) {
378  b = false;
379  } else {
380  throw XMLException("Bad value found for boolean: " + s);
381  }
382 }
383 void XmlInputArchive::load(unsigned char& b)
384 {
385  unsigned i;
386  load(i);
387  b = i;
388 }
389 void XmlInputArchive::load(signed char& c)
390 {
391  int i;
392  load(i);
393  c = i;
394 }
396 {
397  int i;
398  load(i);
399  c = i;
400 }
401 
402 // This function parses a number from a string. It's similar to the generic
403 // templatized XmlInputArchive::load() method, but _much_ faster. It does
404 // have some limitations though:
405 // - it can't handle leading whitespace
406 // - it can't handle extra characters at the end of the string
407 // - it can only handle one base (only decimal, not octal or hexadecimal)
408 // - it doesn't understand a leading '+' sign
409 // - it doesn't detect overflow or underflow (The generic implementation sets
410 // a 'bad' flag on the stream and clips the result to the min/max allowed
411 // value. Though this 'bad' flag was ignored by the openMSX code).
412 // This routine is only used to parse strings we've written ourselves (and the
413 // savestate/replay XML files are not meant to be manually edited). So the
414 // above limitations don't really matter. And we can use the speed gain.
415 template<bool IS_SIGNED> struct ConditionalNegate;
416 template<> struct ConditionalNegate<true> {
417  template<typename T> void operator()(bool negate, T& t) {
418  if (negate) t = -t; // ok to negate a signed type
419  }
420 };
421 template<> struct ConditionalNegate<false> {
422  template<typename T> void operator()(bool negate, T& /*t*/) {
423  assert(!negate); (void)negate; // can't negate unsigned type
424  }
425 };
426 template<typename T> static inline void fastAtoi(const string& str, T& t)
427 {
428  t = 0;
429  bool neg = false;
430  size_t i = 0;
431  size_t l = str.size();
432 
433  static const bool IS_SIGNED = std::numeric_limits<T>::is_signed;
434  if (IS_SIGNED) {
435  if (l == 0) return;
436  if (str[0] == '-') {
437  neg = true;
438  i = 1;
439  }
440  }
441  for (; i < l; ++i) {
442  unsigned d = str[i] - '0';
443  if (unlikely(d > 9)) {
444  throw XMLException("Invalid integer: " + str);
445  }
446  t = 10 * t + d;
447  }
448  // The following stuff does the equivalent of:
449  // if (neg) t = -t;
450  // Though this expression triggers a warning on VC++ when T is an
451  // unsigned type. This complex template stuff avoids the warning.
452  ConditionalNegate<IS_SIGNED> negateFunctor;
453  negateFunctor(neg, t);
454 }
456 {
457  std::string str;
458  load(str);
459  fastAtoi(str, i);
460 }
461 void XmlInputArchive::load(unsigned& u)
462 {
463  std::string str;
464  load(str);
465  fastAtoi(str, u);
466 }
467 void XmlInputArchive::load(unsigned long long& ull)
468 {
469  std::string str;
470  load(str);
471  fastAtoi(str, ull);
472 }
473 
474 void XmlInputArchive::beginTag(const char* tag)
475 {
476  auto* child = elems.back().first->findNextChild(
477  tag, elems.back().second);
478  if (!child) {
479  string path;
480  for (auto& e : elems) {
481  path += e.first->getName() + '/';
482  }
484  "No child tag \"" << tag <<
485  "\" found at location \"" << path << '\"');
486  }
487  elems.emplace_back(child, 0);
488 }
489 void XmlInputArchive::endTag(const char* tag)
490 {
491  const auto& elem = *elems.back().first;
492  if (elem.getName() != tag) {
493  throw XMLException("End tag \"" + elem.getName() +
494  "\" not equal to begin tag \"" + tag + "\"");
495  }
496  auto& elem2 = const_cast<XMLElement&>(elem);
497  elem2.clearName(); // mark this elem for later beginTag() calls
498  elems.pop_back();
499 }
500 
501 void XmlInputArchive::attribute(const char* name, string& t)
502 {
503  try {
504  t = elems.back().first->getAttribute(name);
505  } catch (ConfigException& ex) {
506  throw XMLException(ex.getMessage());
507  }
508 }
509 void XmlInputArchive::attribute(const char* name, int& i)
510 {
511  attributeImpl(name, i);
512 }
513 void XmlInputArchive::attribute(const char* name, unsigned& u)
514 {
515  attributeImpl(name, u);
516 }
517 bool XmlInputArchive::hasAttribute(const char* name)
518 {
519  return elems.back().first->hasAttribute(name);
520 }
521 bool XmlInputArchive::findAttribute(const char* name, unsigned& value)
522 {
523  return elems.back().first->findAttributeInt(name, value);
524 }
526 {
527  return int(elems.back().first->getChildren().size());
528 }
529 
530 } // namespace openmsx
void attribute(const char *name, T &t)
Definition: serialize.hh:785
T length(const vecN< N, T > &x)
Definition: gl_vec.hh:281
size_t maxCompressedLength(size_t inLen)
Definition: snappy.cc:616
bool findAttribute(const char *name, unsigned &value)
Definition: serialize.cc:521
void serialize_blob(const char *tag, void *data, size_t len)
Definition: serialize.cc:126
string_ref::const_iterator end(const string_ref &x)
Definition: string_ref.hh:135
void operator()(bool negate, T &t)
Definition: serialize.cc:417
#define unlikely(x)
Definition: likely.hh:15
void loadChar(char &c)
Definition: serialize.cc:355
XmlInputArchive(const std::string &filename)
Definition: serialize.cc:349
const byte * getCurrentPos() const
Return a pointer to the current position in the buffer.
void * getPointer(unsigned id)
Definition: serialize.cc:113
void save(const T &t)
Definition: serialize.hh:588
unsigned char byte
8 bit unsigned integer
Definition: openmsx.hh:27
void endTag(const char *tag)
Definition: serialize.cc:489
void serialize_blob(const char *, void *data, size_t len)
Definition: serialize.cc:233
std::string dump() const
Definition: XMLElement.cc:296
void addPointer(unsigned id, const void *p)
Definition: serialize.cc:119
XMLElement load(string_ref filename, string_ref systemID)
Definition: XMLLoader.cc:31
const std::string & getName() const
Definition: XMLElement.hh:30
bool hasAttribute(const char *name)
Definition: serialize.cc:517
void compress(const char *input, size_t inLen, char *output, size_t &outLen)
Definition: snappy.cc:603
void attribute(const char *name, const T &t)
Definition: serialize.hh:723
byte * allocate(size_t len)
Reserve space to insert the given number of bytes.
void endTag(const char *tag)
Definition: serialize.cc:340
void attributeImpl(const char *name, const T &t)
Definition: serialize.hh:719
void attribute(const char *name, T &t)
Load/store an attribute from/in the archive.
Definition: serialize.hh:205
void serialize_blob(const char *, const void *data, size_t len)
Definition: serialize.cc:204
void save(const T &t)
Definition: serialize.hh:694
const std::string & getMessage() const
Definition: MSXException.hh:14
void uncompress(const char *input, size_t inLen, char *output, size_t outLen)
Definition: snappy.cc:166
Thanks to enen for testing this on a real cartridge:
Definition: Autofire.cc:7
std::string toString(time_t time)
Definition: Date.cc:152
void deallocate(byte *pos)
Free part of a previously allocated buffer.
MemBuffer< byte > releaseBuffer()
Definition: serialize.cc:177
void operator()(bool negate, T &)
Definition: serialize.cc:422
void beginTag(const char *tag)
Definition: serialize.cc:474
int countChildren() const
Definition: serialize.cc:525
void addAttribute(string_ref name, string_ref value)
Definition: XMLElement.cc:57
static std::string full()
Definition: Version.cc:7
FILE_t openFile(const std::string &filename, const std::string &mode)
Call fopen() in a platform-independent manner.
void saveImpl(const T &t)
Definition: serialize.hh:688
XmlOutputArchive(const std::string &filename)
Definition: serialize.cc:248
byte * release(size_t &size)
Release ownership of the buffer.
void beginTag(const char *tag)
Definition: serialize.cc:334
size_t size(string_ref utf8)
void serialize_blob(const char *tag, const void *data, size_t len)
Definition: serialize.cc:76
const T * data() const
Returns pointer to the start of the memory buffer.
Definition: MemBuffer.hh:94
void skip(size_t len)
Skip the given number of bytes.
void attributeImpl(const char *name, T &t)
Definition: serialize.hh:778