Fix problems with null characters in Text/Data fields. Fixes #19

capnp/includes/capnp_cpp.pxd

@@ -14,10 +14,16 @@ cdef extern from "capnp/common.h" namespace " ::capnp":
 
 cdef extern from "kj/string.h" namespace " ::kj":
     cdef cppclass StringPtr:
+        StringPtr()
         StringPtr(char *)
+        StringPtr(char *, size_t)
         char* cStr()
+        size_t size()
+        char* begin()
     cdef cppclass String:
         char* cStr()
+        size_t size()
+        char* begin()
 
 cdef extern from "kj/exception.h" namespace " ::kj":
     cdef cppclass Exception:
@@ -273,11 +279,11 @@ cdef extern from "capnp/any.h" namespace " ::capnp":
     cdef cppclass AnyPointer:
         cppclass Reader:
             DynamicStruct.Reader getAs"getAs< ::capnp::DynamicStruct>"(StructSchema)
-            String getAsText"getAs< ::capnp::Text>"()
+            StringPtr getAsText"getAs< ::capnp::Text>"()
         cppclass Builder:
             Builder(Builder)
             DynamicStruct.Builder getAs"getAs< ::capnp::DynamicStruct>"(StructSchema)
-            String getAsText"getAs< ::capnp::Text>"()
+            StringPtr getAsText"getAs< ::capnp::Text>"()
             void setAsStruct"setAs< ::capnp::DynamicStruct>"(DynamicStruct.Reader&) except +reraise_kj_exception
             void setAsText"setAs< ::capnp::Text>"(char*) except +reraise_kj_exception
 
@@ -326,6 +332,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
             Reader(float value)
             Reader(double value)
             Reader(char* value)
+            Reader(StringPtr value)
             Reader(DynamicList.Reader& value)
             Reader(DynamicEnum value)
             Reader(DynamicStruct.Reader& value)
@@ -336,7 +343,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
             uint64_t asUint"as<uint64_t>"()
             bint asBool"as<bool>"()
             double asDouble"as<double>"()
-            String asText"as< ::capnp::Text>"()
+            StringPtr asText"as< ::capnp::Text>"()
             DynamicList.Reader asList"as< ::capnp::DynamicList>"()
             DynamicStruct.Reader asStruct"as< ::capnp::DynamicStruct>"()
             AnyPointer.Reader asObject"as< ::capnp::AnyPointer>"()
@@ -350,7 +357,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
             uint64_t asUint"as<uint64_t>"()
             bint asBool"as<bool>"()
             double asDouble"as<double>"()
-            String asText"as< ::capnp::Text>"()
+            StringPtr asText"as< ::capnp::Text>"()
             DynamicList.Builder asList"as< ::capnp::DynamicList>"()
             DynamicStruct.Builder asStruct"as< ::capnp::DynamicStruct>"()
             AnyPointer.Builder asObject"as< ::capnp::AnyPointer>"()

capnp/lib/capnp.pyx

@@ -258,7 +258,7 @@ cdef public object wrap_kj_exception_for_reraise(capnp.Exception & exception):
 
 cdef public object get_exception_info(object exc_type, object exc_obj, object exc_tb):
     try:
-        return (exc_tb.tb_frame.f_code.co_filename.encode(), exc_tb.tb_lineno, (repr(exc_type) + ':' + str(exc_obj)).encode())
+        return (exc_tb.tb_frame.f_code.co_filename.encode('utf-8'), exc_tb.tb_lineno, (repr(exc_type) + ':' + str(exc_obj)).encode('utf-8'))
     except:
         return (b'', 0, b"Couldn't determine python exception")
 
@@ -574,10 +574,11 @@ cdef to_python_reader(C_DynamicValue.Reader self, object parent):
     elif type == capnp.TYPE_FLOAT:
         return self.asDouble()
     elif type == capnp.TYPE_TEXT:
-        return (<char*>self.asText().cStr())[:]
+        temp_text = self.asText()
+        return (<char*>temp_text.begin())[:temp_text.size()]
     elif type == capnp.TYPE_DATA:
-        temp = self.asData()
+        temp_data = self.asData()
-        return (<char*>temp.begin())[:temp.size()]
+        return <bytes>((<char*>temp_data.begin())[:temp_data.size()])
     elif type == capnp.TYPE_LIST:
         return _DynamicListReader()._init(self.asList(), parent)
     elif type == capnp.TYPE_STRUCT:
@@ -606,10 +607,11 @@ cdef to_python_builder(C_DynamicValue.Builder self, object parent):
     elif type == capnp.TYPE_FLOAT:
         return self.asDouble()
     elif type == capnp.TYPE_TEXT:
-        return (<char*>self.asText().cStr())[:]
+        temp_text = self.asText()
+        return (<char*>temp_text.begin())[:temp_text.size()]
     elif type == capnp.TYPE_DATA:
-        temp = self.asData()
+        temp_data = self.asData()
-        return (<char*>temp.begin())[:temp.size()]
+        return <bytes>((<char*>temp_data.begin())[:temp_data.size()])
     elif type == capnp.TYPE_LIST:
         return _DynamicListBuilder()._init(self.asList(), parent)
     elif type == capnp.TYPE_STRUCT:
@@ -656,9 +658,17 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent):
     elif value_type is bool:
         temp = C_DynamicValue.Reader(<cbool>value)
         thisptr.set(field, temp)
-    elif isinstance(value, basestring):
+    elif value_type is bytes:
-        temp = C_DynamicValue.Reader(<char*>value)
+        temp2 = new capnp.StringPtr(<char*>value, len(value))
+        temp = C_DynamicValue.Reader(deref(temp2))
         thisptr.set(field, temp)
+        del temp2
+    elif isinstance(value, basestring):
+        encoded_value = value.encode('utf-8')
+        temp2 = new capnp.StringPtr(<char*>encoded_value, len(encoded_value))
+        temp = C_DynamicValue.Reader(deref(temp2))
+        thisptr.set(field, temp)
+        del temp2
     elif value_type is list:
         builder = to_python_builder(thisptr.init(field, len(value)), parent)
         for (i, v) in enumerate(value):
@@ -693,9 +703,17 @@ cdef _setDynamicFieldPtr(_DynamicSetterClasses * thisptr, field, value, parent):
     elif value_type is bool:
         temp = C_DynamicValue.Reader(<cbool>value)
         thisptr.set(field, temp)
-    elif isinstance(value, basestring):
+    elif value_type is bytes:
-        temp = C_DynamicValue.Reader(<char*>value)
+        temp2 = new capnp.StringPtr(<char*>value, len(value))
+        temp = C_DynamicValue.Reader(deref(temp2))
         thisptr.set(field, temp)
+        del temp2
+    elif isinstance(value, basestring):
+        encoded_value = value.encode('utf-8')
+        temp2 = new capnp.StringPtr(<char*>encoded_value, len(encoded_value))
+        temp = C_DynamicValue.Reader(deref(temp2))
+        thisptr.set(field, temp)
+        del temp2
     elif value_type is list:
         builder = to_python_builder(thisptr.init(field, len(value)), parent)
         for (i, v) in enumerate(value):

test/test_regression.py

@@ -216,7 +216,7 @@ def init_all_types(builder):
     builder.float32Field = 1234.5
     builder.float64Field = -123e45
     builder.textField = "foo"
-    builder.dataField = "bar"
+    builder.dataField = b"bar"
 
     subBuilder = builder.structField
     subBuilder.voidField = None
@@ -232,7 +232,7 @@ def init_all_types(builder):
     subBuilder.float32Field = -1.25e-10
     subBuilder.float64Field = 345
     subBuilder.textField = "baz"
-    subBuilder.dataField = "qux"
+    subBuilder.dataField = b"qux"
     subSubBuilder = subBuilder.structField
     subSubBuilder.textField = "nested"
     subSubBuilder.structField.textField = "really nested"
@@ -251,7 +251,7 @@ def init_all_types(builder):
     subBuilder.float32List = [0, 1234567, 1e37, -1e37, 1e-37, -1e-37]
     subBuilder.float64List = [0, 123456789012345, 1e306, -1e306, 1e-306, -1e-306]
     subBuilder.textList = ["quux", "corge", "grault"]
-    subBuilder.dataList = ["garply", "waldo", "fred"]
+    subBuilder.dataList = [b"garply", b"waldo", b"fred"]
     listBuilder = subBuilder.init('structList', 3)
     listBuilder[0].textField = "x structlist 1"
     listBuilder[1].textField = "x structlist 2"
@@ -273,7 +273,7 @@ def init_all_types(builder):
     builder.float32List = [5555.5, float("inf"), float("-inf"), float("nan")]
     builder.float64List = [7777.75, float("inf"), float("-inf"), float("nan")]
     builder.textList = ["plugh", "xyzzy", "thud"]
-    builder.dataList = ["oops", "exhausted", "rfc3092"]
+    builder.dataList = [b"oops", b"exhausted", b"rfc3092"]
     listBuilder = builder.init('structList', 3)
     listBuilder[0].textField = "structlist 1"
     listBuilder[1].textField = "structlist 2"
@@ -306,7 +306,7 @@ def check_all_types(reader):
     assert reader.float32Field == 1234.5
     assert_almost(reader.float64Field, -123e45)
     assert reader.textField == "foo"
-    assert reader.dataField == "bar"
+    assert reader.dataField == b"bar"
 
     subReader = reader.structField
     assert subReader.voidField == None
@@ -322,7 +322,7 @@ def check_all_types(reader):
     assert_almost(subReader.float32Field, -1.25e-10)
     assert subReader.float64Field == 345
     assert subReader.textField == "baz"
-    assert subReader.dataField == "qux"
+    assert subReader.dataField == b"qux"
 
     subSubReader = subReader.structField
     assert subSubReader.textField == "nested"
@@ -343,7 +343,7 @@ def check_all_types(reader):
     check_list(subReader.float32List, [0.0, 1234567.0, 1e37, -1e37, 1e-37, -1e-37])
     check_list(subReader.float64List, [0.0, 123456789012345.0, 1e306, -1e306, 1e-306, -1e-306])
     check_list(subReader.textList, ["quux", "corge", "grault"])
-    check_list(subReader.dataList, ["garply", "waldo", "fred"])
+    check_list(subReader.dataList, [b"garply", b"waldo", b"fred"])
 
     listReader = subReader.structList
     assert len(listReader) == 3
@@ -381,7 +381,7 @@ def check_all_types(reader):
     assert math.isnan(listReader[3])
 
     check_list(reader.textList, ["plugh", "xyzzy", "thud"])
-    check_list(reader.dataList, ["oops", "exhausted", "rfc3092"])
+    check_list(reader.dataList, [b"oops", b"exhausted", b"rfc3092"])
 
     listReader = reader.structList
     len(listReader) == 3

test/test_struct.py

@@ -2,12 +2,20 @@ import pytest
 import capnp
 import os
 import tempfile
+import sys
 
 this_dir = os.path.dirname(__file__)
 
+
 @pytest.fixture
 def addressbook():
-     return capnp.load(os.path.join(this_dir, 'addressbook.capnp'))
+    return capnp.load(os.path.join(this_dir, 'addressbook.capnp'))
+
+
+@pytest.fixture
+def all_types():
+    return capnp.load(os.path.join(this_dir, 'all_types.capnp'))
+
 
 def test_which_builder(addressbook):
     addresses = addressbook.AddressBook.new_message()
@@ -21,7 +29,7 @@ def test_which_builder(addressbook):
     bob = people[1]
 
     assert bob.employment.which() == "unemployed"
-    
+
     bob.employment.unemployed = None
 
     assert bob.employment.which() == "unemployed"
@@ -31,6 +39,7 @@ def test_which_builder(addressbook):
     with pytest.raises(ValueError):
         addresses.which()
 
+
 def test_which_reader(addressbook):
     def writeAddressBook(fd):
         message = capnp._MallocMessageBuilder()
@@ -64,6 +73,7 @@ def test_which_reader(addressbook):
     with pytest.raises(ValueError):
         addresses.which()
 
+
 def test_builder_set(addressbook):
     person = addressbook.Person.new_message()
 
@@ -73,3 +83,24 @@ def test_builder_set(addressbook):
 
     with pytest.raises(ValueError):
         person.foo = 'test'
+
+
+def test_null_str(all_types):
+    msg = all_types.TestAllTypes.new_message()
+
+    msg.textField = "f\x00oo"
+    msg.dataField = b"b\x00ar"
+
+    assert msg.textField == "f\x00oo"
+    assert msg.dataField == b"b\x00ar"
+
+
+def test_unicode_str(all_types):
+    msg = all_types.TestAllTypes.new_message()
+
+    msg.textField = u"f\u00e6oo"
+
+    if sys.version_info.major == 2:
+        assert msg.textField.decode('utf-8') == u"f\u00e6oo"
+    else:
+        assert msg.textField == u"f\u00e6oo"