From 1dad42c2f609f6a6f8f4a811da6d7d7eb1475d7b Mon Sep 17 00:00:00 2001
From: wmayer <wmayer@users.sourceforge.net>
Date: Sun, 20 Sep 2015 12:29:05 +0200
Subject: [PATCH] + fixes #0002126: Fails to load .FCStd file which contains
 many MultiByte-Char strings

---
 src/Base/InputSource.cpp | 90 +++++++++++++---------------------------
 src/Base/InputSource.h   |  2 +
 2 files changed, 30 insertions(+), 62 deletions(-)

diff --git a/src/Base/InputSource.cpp b/src/Base/InputSource.cpp
index 61cbfb400..5f669e320 100644
--- a/src/Base/InputSource.cpp
+++ b/src/Base/InputSource.cpp
@@ -57,6 +57,8 @@ using namespace std;
 StdInputStream::StdInputStream( std::istream& Stream, XERCES_CPP_NAMESPACE_QUALIFIER MemoryManager* const manager ) 
   : stream(Stream), fMemoryManager(manager)
 {
+    state.flags |= QTextCodec::IgnoreHeader;
+    state.flags |= QTextCodec::ConvertInvalidToNull;
 }
 
 
@@ -84,37 +86,19 @@ unsigned int StdInputStream::readBytes( XMLByte* const  toFill, const unsigned i
   stream.read((char *)toFill,maxToRead);
   XMLSize_t len = stream.gcount();
 
-  // See http://de.wikipedia.org/wiki/UTF-8#Kodierung
-  for (XMLSize_t i=0; i<len; i++) {
-      XMLByte& b = toFill[i];
-      int seqlen = 0;
-
-      if ((b & 0x80) == 0) {
-          seqlen = 1;
-      }
-      else if ((b & 0xE0) == 0xC0) {
-          seqlen = 2;
-          if (b == 0xC0 || b == 0xC1)
-              b = '?'; // these both values are not allowed
-      }
-      else if ((b & 0xF0) == 0xE0) {
-          seqlen = 3;
-      }
-      else if ((b & 0xF8) == 0xF0) {
-          seqlen = 4;
-      }
-      else {
-          b = '?';
-      }
-
-      for(int j = 1; j < seqlen; ++j) {
-          i++;
-          XMLByte& c = toFill[i];
-          // range of second, third or fourth byte
-          if ((c & 0xC0) != 0x80) {
-              b = '?';
-              c = '?';
-          }
+  QTextCodec *codec = QTextCodec::codecForName("UTF-8");
+  const QString text = codec->toUnicode((char *)toFill, len, &state);
+  if (state.invalidChars > 0) {
+      // In case invalid characters were found decode back to 'utf-8' and replace
+      // them with '?'
+      // First, Qt replaces invalid characters with '\0' (see ConvertInvalidToNull)
+      // but Xerces doesn't like this because it handles this as termination. Thus,
+      // we have to go through the array and replace '\0' with '?'.
+      XMLSize_t pos = 0;
+      QByteArray ba = codec->fromUnicode(text);
+      for (int i=0; i<ba.length(); i++, pos++) {
+          if (pos < len && ba[i] == '\0')
+              toFill[i] = '?';
       }
   }
 
@@ -136,37 +120,19 @@ XMLSize_t StdInputStream::readBytes( XMLByte* const  toFill, const XMLSize_t max
   stream.read((char *)toFill,maxToRead);
   XMLSize_t len = stream.gcount();
 
-  // See http://de.wikipedia.org/wiki/UTF-8#Kodierung
-  for (XMLSize_t i=0; i<len; i++) {
-      XMLByte& b = toFill[i];
-      int seqlen = 0;
-
-      if ((b & 0x80) == 0) {
-          seqlen = 1;
-      }
-      else if ((b & 0xE0) == 0xC0) {
-          seqlen = 2;
-          if (b == 0xC0 || b == 0xC1)
-              b = '?'; // these both values are not allowed
-      }
-      else if ((b & 0xF0) == 0xE0) {
-          seqlen = 3;
-      }
-      else if ((b & 0xF8) == 0xF0) {
-          seqlen = 4;
-      }
-      else {
-          b = '?';
-      }
-
-      for(int j = 1; j < seqlen; ++j) {
-          i++;
-          XMLByte& c = toFill[i];
-          // range of second, third or fourth byte
-          if ((c & 0xC0) != 0x80) {
-              b = '?';
-              c = '?';
-          }
+  QTextCodec *codec = QTextCodec::codecForName("UTF-8");
+  const QString text = codec->toUnicode((char *)toFill, len, &state);
+  if (state.invalidChars > 0) {
+      // In case invalid characters were found decode back to 'utf-8' and replace
+      // them with '?'
+      // First, Qt replaces invalid characters with '\0' (see ConvertInvalidToNull)
+      // but Xerces doesn't like this because it handles this as termination. Thus,
+      // we have to go through the array and replace '\0' with '?'.
+      XMLSize_t pos = 0;
+      QByteArray ba = codec->fromUnicode(text);
+      for (int i=0; i<ba.length(); i++, pos++) {
+          if (pos < len && ba[i] == '\0')
+              toFill[i] = '?';
       }
   }
 
diff --git a/src/Base/InputSource.h b/src/Base/InputSource.h
index 18f3694fb..deea9eb49 100644
--- a/src/Base/InputSource.h
+++ b/src/Base/InputSource.h
@@ -31,6 +31,7 @@
 #include <xercesc/util/PlatformUtils.hpp>
 #include <xercesc/util/XercesVersion.hpp>
 #include <xercesc/sax/InputSource.hpp>
+#include <QTextCodec>
 
 
 XERCES_CPP_NAMESPACE_BEGIN
@@ -75,6 +76,7 @@ private :
   // -----------------------------------------------------------------------
   std::istream            &stream;
   XERCES_CPP_NAMESPACE_QUALIFIER MemoryManager* const    fMemoryManager;
+  QTextCodec::ConverterState state;
 };