+ optimize PyStreambuf::underflow() and fix failure for large files/buffers

src/Base/Stream.cpp

@@ -524,12 +524,15 @@ IODeviceIStreambuf::seekpos(std::streambuf::pos_type pos,
 
 // ---------------------------------------------------------
 
-PyStreambuf::PyStreambuf(PyObject* o) : inp(o)
+// http://www.mr-edd.co.uk/blog/beginners_guide_streambuf
+PyStreambuf::PyStreambuf(PyObject* o, std::size_t buf_size, std::size_t put_back)
+    : inp(o)
+    , put_back(std::max(put_back, std::size_t(1)))
+    , buffer(std::max(buf_size, put_back) + put_back)
 {
     Py_INCREF(inp);
-    setg (buffer+pbSize,
-          buffer+pbSize,
-          buffer+pbSize);
+    char *end = &buffer.front() + buffer.size();
+    setg(end, end, end);
 }
 
 PyStreambuf::~PyStreambuf()
@@ -540,42 +543,40 @@ PyStreambuf::~PyStreambuf()
 std::streambuf::int_type PyStreambuf::underflow()
 {
     if (gptr() < egptr()) {
-        return *gptr();
+        return traits_type::to_int_type(*gptr());
     }
 
-    int numPutback;
-    numPutback = gptr() - eback();
-    if (numPutback > pbSize) {
-        numPutback = pbSize;
+    char *base = &buffer.front();
+    char *start = base;
+
+    if (eback() == base) { // true when this isn't the first fill
+        std::memmove(base, egptr() - put_back, put_back);
+        start += put_back;
     }
 
-    memcpy (buffer+(pbSize-numPutback), gptr()-numPutback, numPutback);
+    std::size_t n;
+    Py::Tuple arg(1);
+    long len = static_cast<long>(buffer.size() - (start - base));
+    arg.setItem(0, Py::Long(len));
+    Py::Callable meth(Py::Object(inp).getAttr("read"));
 
-    int num=0;
-    for (int i=0; i<bufSize; i++) {
-        char c;
-        Py::Tuple arg(1);
-        arg.setItem(0, Py::Int(1));
-        Py::Callable meth(Py::Object(inp).getAttr("read"));
-        try {
-            Py::Char res(meth.apply(arg));
-            c = static_cast<std::string>(res)[0];
-            num++;
-            buffer[pbSize+i] = c;
-        }
-        catch (Py::Exception& e) {
-            e.clear();
-            if (num == 0)
-                return EOF;
-            break;
+    try {
+        Py::String res(meth.apply(arg));
+        std::string c = static_cast<std::string>(res);
+        n = c.size();
+        if (n == 0) {
+            return traits_type::eof();
         }
+
+        std::memcpy(start, &(c[0]), c.size());
+    }
+    catch (Py::Exception& e) {
+        e.clear();
+        return traits_type::eof();
     }
 
-    setg (buffer+(pbSize-numPutback),
-          buffer+pbSize,
-          buffer+pbSize+num);
-
-    return *gptr();
+    setg(base, start, start + n);
+    return traits_type::to_int_type(*gptr());
 }
 
 std::streambuf::int_type

src/Base/Stream.h

@@ -241,7 +241,7 @@ protected:
 class BaseExport PyStreambuf : public std::streambuf
 {
 public:
-    PyStreambuf(PyObject* o);
+    PyStreambuf(PyObject* o, std::size_t buf_size = 256, std::size_t put_back = 8);
     virtual ~PyStreambuf();
 
 protected:
@@ -250,10 +250,9 @@ protected:
     std::streamsize xsputn (const char* s, std::streamsize num);
 
 private:
-    static const int pbSize = 4;
-    static const int bufSize = 1024;
-    char buffer[bufSize+pbSize];
     PyObject* inp;
+    const std::size_t put_back;
+    std::vector<char> buffer;
 };
 
 class BaseExport Streambuf : public std::streambuf