Get code generator plugin to a decently working state.

Still a few outstanding performance issues, and imports inside .capnp
files may not be working
This commit is contained in:
Jason Paryani 2014-10-19 20:06:00 -07:00
parent ece4e36329
commit 64e80e06bf
6 changed files with 278 additions and 95 deletions

View file

@ -6,6 +6,13 @@ import sys
from jinja2 import Environment, PackageLoader
import os
def find_type(code, id):
for node in code['nodes']:
if node['id'] == id:
return node
return None
def main():
env = Environment(loader=PackageLoader('capnp', 'templates'))
env.filters['format_name'] = lambda name: name[name.find(':')+1:]
@ -25,6 +32,18 @@ def main():
if field['discriminantValue'] != 65535:
is_union = True
field['c_name'] = field['name'][0].upper() + field['name'][1:]
if 'slot' in field:
field['type'] = field['slot']['type'].keys()[0]
if not isinstance(field['slot']['type'][field['type']], dict):
continue
sub_type = field['slot']['type'][field['type']].get('typeId', None)
if sub_type:
field['sub_type'] = find_type(code, sub_type)
sub_type = field['slot']['type'][field['type']].get('elementType', None)
if sub_type:
field['sub_type'] = sub_type
else:
field['type'] = find_type(code, field['group']['typeId'])
node['is_union'] = is_union
include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..'))

View file

@ -34,7 +34,8 @@ cdef class _DynamicStructReader:
cpdef _get(self, field)
cpdef _has(self, field)
cpdef _which(self)
cpdef _DynamicEnumField _which(self)
cpdef _which_str(self)
cpdef _get_by_field(self, _StructSchemaField field)
cpdef _has_by_field(self, _StructSchemaField field)
@ -64,13 +65,20 @@ cdef class _DynamicStructBuilder:
cpdef _has_by_field(self, _StructSchemaField field)
cpdef _init_by_field(self, _StructSchemaField field, size=?)
cpdef init_resizable_list(self, field)
cpdef _which(self)
cpdef _DynamicEnumField _which(self)
cpdef _which_str(self)
cpdef adopt(self, field, _DynamicOrphan orphan)
cpdef disown(self, field)
cpdef as_reader(self)
cpdef copy(self, num_first_segment_words=?)
cdef class _DynamicEnumField:
cdef object thisptr
cdef _init(self, proto)
cpdef _str(self)
cdef class _Schema:
cdef C_Schema thisptr
@ -101,8 +109,8 @@ cdef class _DynamicListBuilder:
cdef public object _parent
cdef _init(self, C_DynamicList.Builder other, object parent)
cdef _get(self, index)
cdef _set(self, index, value)
cpdef _get(self, int64_t index)
cpdef _set(self, index, value)
cpdef adopt(self, index, _DynamicOrphan orphan)
cpdef disown(self, index)
@ -110,7 +118,6 @@ cdef class _DynamicListBuilder:
cdef to_python_reader(C_DynamicValue.Reader self, object parent)
cdef to_python_builder(C_DynamicValue.Builder self, object parent)
cdef _to_dict(msg, bint verbose)
cdef _from_dict(_DynamicStructBuilder msg, dict d)
cdef _from_list(_DynamicListBuilder msg, list d)
cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent)
cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent)

View file

@ -6,10 +6,11 @@
# cython: c_string_type = str
# cython: c_string_encoding = default
# cython: embedsignature = True
# cython: profile=True
cimport cython
from .capnp.helpers.helpers cimport makeRpcClientWithRestorer
from capnp.helpers.helpers cimport makeRpcClientWithRestorer
from libc.stdlib cimport malloc, free
from cython.operator cimport dereference as deref
@ -396,12 +397,15 @@ cdef class _DynamicListReader:
self._parent = parent
return self
def __getitem__(self, index):
size = self.thisptr.size()
cpdef _get(self, int64_t index):
return to_python_reader(self.thisptr[index], self._parent)
def __getitem__(self, int64_t index):
cdef uint size = self.thisptr.size()
if index >= size:
raise IndexError('Out of bounds')
index = index % size
return to_python_reader(self.thisptr[index], self._parent)
return self._get(index)
def __len__(self):
return self.thisptr.size()
@ -457,6 +461,9 @@ cdef class _DynamicResizableListBuilder:
self._list.append((orphan, orphan_val))
return orphan_val
cpdef _get(self, index):
return self._list[index][1]
def __getitem__(self, index):
return self._list[index][1]
@ -500,17 +507,17 @@ cdef class _DynamicListBuilder:
self._parent = parent
return self
cdef _get(self, index):
cpdef _get(self, int64_t index):
return to_python_builder(self.thisptr[index], self._parent)
def __getitem__(self, index):
size = self.thisptr.size()
def __getitem__(self, int64_t index):
cdef uint size = self.thisptr.size()
if index >= size:
raise IndexError('Out of bounds')
index = index % size
return self._get(index)
cdef _set(self, index, value):
cpdef _set(self, index, value):
_setDynamicField(self.thisptr, index, value, self._parent)
def __setitem__(self, index, value):
@ -714,10 +721,10 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent):
elif value_type is dict:
if _DynamicSetterClasses is DynamicStruct_Builder:
builder = to_python_builder(thisptr.get(field), parent)
_from_dict(builder, value)
builder.from_dict(value)
else:
builder = to_python_builder(thisptr[field], parent)
_from_dict(builder, value)
builder.from_dict(value)
elif value is None:
temp = C_DynamicValue.Reader(VOID)
thisptr.set(field, temp)
@ -759,7 +766,7 @@ cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField
_from_list(builder, value)
elif value_type is dict:
builder = to_python_builder(thisptr.getByField(field.thisptr), parent)
_from_dict(builder, value)
builder.from_dict(value)
elif value is None:
temp = C_DynamicValue.Reader(VOID)
thisptr.setByField(field.thisptr, temp)
@ -801,7 +808,7 @@ cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent)
_from_list(builder, value)
elif value_type is dict:
builder = to_python_builder(thisptr.get(field), parent)
_from_dict(builder, value)
builder.from_dict(value)
elif value is None:
temp = C_DynamicValue.Reader(VOID)
thisptr.set(field, temp)
@ -818,22 +825,49 @@ cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent)
else:
raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value))))
cdef _DynamicListBuilder temp_list_b
cdef _DynamicListReader temp_list_r
cdef _DynamicResizableListBuilder temp_list_rb
cdef _DynamicStructBuilder temp_msg_b
cdef _DynamicStructReader temp_msg_r
cdef _to_dict(msg, bint verbose):
msg_type = type(msg)
if msg_type is _DynamicListBuilder or msg_type is _DynamicListReader or msg_type is _DynamicResizableListBuilder:
return [_to_dict(x, verbose) for x in msg]
if msg_type is _DynamicListBuilder:
temp_list_b = msg
return [_to_dict(temp_list_b._get(i), verbose) for i in range(len(msg))]
elif msg_type is _DynamicListReader:
temp_list_r = msg
return [_to_dict(temp_list_r._get(i), verbose) for i in range(len(msg))]
elif msg_type is _DynamicResizableListBuilder:
temp_list_rb = msg
return [_to_dict(temp_list_rb._get(i), verbose) for i in range(len(msg))]
if msg_type is _DynamicStructBuilder or msg_type is _DynamicStructReader:
if msg_type is _DynamicStructBuilder:
temp_msg_b = msg
ret = {}
try:
which = msg.which()
ret[which] = _to_dict(msg._get(which), verbose)
which = temp_msg_b.which()
ret[which] = _to_dict(temp_msg_b._get(which), verbose)
except ValueError:
pass
for field in msg.schema.non_union_fields:
if verbose or msg._has(field):
ret[field] = _to_dict(msg._get(field), verbose)
for field in temp_msg_b.schema.non_union_fields:
if verbose or temp_msg_b._has(field):
ret[field] = _to_dict(temp_msg_b._get(field), verbose)
return ret
elif msg_type is _DynamicStructReader:
temp_msg_r = msg
ret = {}
try:
which = temp_msg_r.which()
ret[which] = _to_dict(temp_msg_r._get(which), verbose)
except ValueError:
pass
for field in temp_msg_r.schema.non_union_fields:
if verbose or temp_msg_r._has(field):
ret[field] = _to_dict(temp_msg_r._get(field), verbose)
return ret
@ -845,22 +879,10 @@ cdef _to_dict(msg, bint verbose):
return msg
cdef _from_dict(_DynamicStructBuilder msg, dict d):
for key, val in d.iteritems():
if key != 'which':
try:
msg._set(key, val)
except Exception as e:
if 'expected isSetInUnion(field)' in str(e):
msg.init(key)
msg._set(key, val)
cdef _from_list(_DynamicListBuilder msg, list d):
cdef size_t count = 0
for val in d:
msg._set(count, val)
count += 1
for i in range(len(d)):
msg._set(i, d[i])
cdef class _DynamicEnum:
@ -903,8 +925,6 @@ cdef class _DynamicEnum:
return left >= right
cdef class _DynamicEnumField:
cdef object thisptr
cdef _init(self, proto):
self.thisptr = proto
return self
@ -914,9 +934,12 @@ cdef class _DynamicEnumField:
def __get__(self):
return self.thisptr.discriminantValue
def __str__(self):
cpdef _str(self):
return self.thisptr.name
def __str__(self):
return self._str()
def __repr__(self):
return '<%s which-enum>' % str(self)
@ -992,7 +1015,13 @@ cdef class _DynamicStructReader:
cpdef _has_by_field(self, _StructSchemaField field):
return self.thisptr.hasByField(field.thisptr)
cpdef _which(self):
cpdef _which_str(self):
try:
return <char *>helpers.fixMaybe(self.thisptr.which()).getProto().getName().cStr()
except:
raise ValueError("Attempted to call which on a non-union type")
cpdef _DynamicEnumField _which(self):
"""Returns the enum corresponding to the union in this struct
:rtype: :class:`_DynamicEnumField`
@ -1245,7 +1274,13 @@ cdef class _DynamicStructBuilder:
"""
return _DynamicResizableListBuilder(self, field, _StructSchema()._init((<C_DynamicValue.Builder>self.thisptr.get(field)).asList().getStructElementType()))
cpdef _which(self):
cpdef _which_str(self):
try:
return <char *>helpers.fixMaybe(self.thisptr.which()).getProto().getName().cStr()
except:
raise ValueError("Attempted to call which on a non-union type")
cpdef _DynamicEnumField _which(self):
"""Returns the enum corresponding to the union in this struct
:rtype: :class:`_DynamicEnumField`
@ -1343,6 +1378,18 @@ cdef class _DynamicStructBuilder:
def to_dict(self, verbose=False):
return _to_dict(self, verbose)
def from_dict(self, dict d):
for key, val in d.iteritems():
if key != 'which':
try:
self._set(key, val)
except Exception as e:
if 'expected isSetInUnion(field)' in str(e):
self.init(key)
self._set(key, val)
else:
raise
property total_size:
def __get__(self):
size = self.thisptr.totalSize()
@ -2463,7 +2510,7 @@ cdef _new_message(self, kwargs, num_first_segment_words):
builder = _MallocMessageBuilder(num_first_segment_words)
msg = builder.init_root(self.schema)
if kwargs is not None:
_from_dict(msg, kwargs)
msg.from_dict(kwargs)
return msg
class _RestorerImpl(object):
@ -2483,18 +2530,18 @@ class _StructModule(object):
# Add enums for union fields
for field in schema.node.struct.fields:
if field.which() == 'group':
name = field.name.capitalize()
name = field.name[0].upper() + field.name[1:]
raw_schema = schema.get_dependency(field.group.typeId)
union_schema = raw_schema.node.struct
field_schema = raw_schema.node.struct
if union_schema.discriminantCount == 0:
continue
union_module = _StructModuleWhich()
setattr(union_module, 'schema', raw_schema.as_struct())
for union_field in union_schema.fields:
setattr(union_module, union_field.name, union_field.discriminantValue)
setattr(self, name, union_module)
if field_schema.discriminantCount == 0:
sub_module = _StructModule(raw_schema, name)
else:
sub_module = _StructModuleWhich()
setattr(sub_module, 'schema', raw_schema.as_struct())
for union_field in field_schema.fields:
setattr(sub_module, union_field.name, union_field.discriminantValue)
setattr(self, name, sub_module)
def read(self, file, traversal_limit_in_words = None, nesting_limit = None):
"""Returns a Reader for the unpacked object read from file.

View file

@ -7,13 +7,114 @@
# cython: c_string_type = str
# cython: c_string_encoding = default
# cython: embedsignature = True
# cython: profile=True
{% macro getter(field, type) -%}
{% if 'uint' in field['type'] -%}
uint64_t get{{field.c_name}}() except +reraise_kj_exception
{% elif 'int' in field['type'] -%}
int64_t get{{field.c_name}}() except +reraise_kj_exception
{% elif 'void' == field['type'] -%}
void get{{field.c_name}}() except +reraise_kj_exception
{% elif 'bool' == field['type'] -%}
cbool get{{field.c_name}}() except +reraise_kj_exception
{% elif 'text' == field['type'] -%}
StringPtr get{{field.c_name}}() except +reraise_kj_exception
{% elif 'data' == field['type'] -%}
Data.{{type}} get{{field.c_name}}() except +reraise_kj_exception
{% else -%}
DynamicValue.{{type}} get{{field.c_name}}() except +reraise_kj_exception
{%- endif %}
{%- endmacro %}
# TODO: add struct/enum/list types
{% macro getfield(field, type) -%}
cpdef _get_{{field.name}}(self):
{% if 'int' in field['type'] -%}
return self.thisptr_child.get{{field.c_name}}()
{% elif 'void' == field['type'] -%}
self.thisptr_child.get{{field.c_name}}()
return None
{% elif 'bool' == field['type'] -%}
return self.thisptr_child.get{{field.c_name}}()
{% elif 'text' == field['type'] -%}
temp = self.thisptr_child.get{{field.c_name}}()
return (<char*>temp.begin())[:temp.size()]
{% elif 'data' == field['type'] -%}
temp = self.thisptr_child.get{{field.c_name}}()
return <bytes>((<char*>temp.begin())[:temp.size()])
{% else -%}
cdef DynamicValue.{{type}} temp = self.thisptr_child.get{{field.c_name}}()
return to_python_{{type | lower}}(temp, self._parent)
{% endif -%}
{%- endmacro %}
{% macro setter(field) -%}
{% if 'int' in field['type'] -%}
void set{{field.c_name}}({{field.type}}_t) except +reraise_kj_exception
{% elif 'bool' == field['type'] -%}
void set{{field.c_name}}(cbool) except +reraise_kj_exception
{% elif 'text' == field['type'] -%}
void set{{field.c_name}}(StringPtr) except +reraise_kj_exception
{% elif 'data' == field['type'] -%}
void set{{field.c_name}}(ArrayPtr[byte]) except +reraise_kj_exception
{% else -%}
void set{{field.c_name}}(DynamicValue.Reader) except +reraise_kj_exception
{%- endif %}
{%- endmacro %}
{% macro setfield(field) -%}
{% if 'int' in field['type'] -%}
cpdef _set_{{field.name}}(self, {{field.type}}_t value):
self.thisptr_child.set{{field.c_name}}(value)
{% elif 'void' == field['type'] -%}
cpdef _set_{{field.name}}(self, value=None):
pass
{% elif 'bool' == field['type'] -%}
cpdef _set_{{field.name}}(self, bool value):
self.thisptr_child.set{{field.c_name}}(value)
{% elif 'list' == field['type'] -%}
cpdef _set_{{field.name}}(self, list value):
cdef uint i = 0
self.init("{{field.name}}", len(value))
cdef _DynamicListBuilder temp = self._get_{{field.name}}()
for elem in value:
{% if 'struct' in field['sub_type'] -%}
temp._get(i).from_dict(elem)
{% else -%}
temp[i] = elem
{% endif -%}
i += 1
{% elif 'text' == field['type'] -%}
cpdef _set_{{field.name}}(self, value):
cdef StringPtr temp_string
if type(value) is bytes:
temp_string = StringPtr(<char*>value, len(value))
else:
encoded_value = value.encode()
temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
self.thisptr_child.set{{field.c_name}}(temp_string)
{% elif 'data' == field['type'] -%}
cpdef _set_{{field.name}}(self, value):
cdef StringPtr temp_string
if type(value) is bytes:
temp_string = StringPtr(<char*>value, len(value))
else:
encoded_value = value.encode()
temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
self.thisptr_child.set{{field.c_name}}(ArrayPtr[byte](<byte *>temp_string.begin(), temp_string.size()))
{% else -%}
cpdef _set_{{field.name}}(self, value):
_setDynamicFieldStatic(self.thisptr, "{{field.name}}", value, self._parent)
{% endif -%}
{%- endmacro %}
import capnp
import {{file.filename | replace('.', '_')}}
from libcpp cimport bool as cbool
from capnp.includes.types cimport *
from capnp cimport helpers
from capnp.includes.capnp_cpp cimport DynamicValue, Schema, VOID, StringPtr
from capnp.includes.capnp_cpp cimport DynamicValue, Schema, VOID, StringPtr, ArrayPtr, Data
from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _DynamicListBuilder, _DynamicEnum, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldStatic, _Schema, _InterfaceSchema
from capnp.helpers.non_circular cimport reraise_kj_exception
@ -27,16 +128,6 @@ cdef DynamicValue.Reader _extract_dynamic_struct_reader(_DynamicStructReader val
cdef DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value):
return DynamicValue.Reader(value.thisptr)
cdef _from_dict(_DynamicStructBuilder msg, dict d):
for key, val in d.iteritems():
if key != 'which':
try:
msg._set(key, val)
except Exception as e:
if 'expected isSetInUnion(field)' in str(e):
msg.init(key)
msg._set(key, val)
cdef _from_list(_DynamicListBuilder msg, list d):
cdef size_t count = 0
for val in d:
@ -85,12 +176,12 @@ cdef extern from "{{file.filename}}.h":
cdef cppclass {{node.module_name}}"{{node.c_module_path}}":
cppclass Reader:
{%- for field in node.struct.fields %}
DynamicValue.Reader get{{field.c_name}}()
{{ getter(field, "Reader")|indent(12)}}
{%- endfor %}
cppclass Builder:
{%- for field in node.struct.fields %}
DynamicValue.Builder get{{field.c_name}}()
set{{field.c_name}}(DynamicValue.Reader)
{{ getter(field, "Builder")|indent(12)}}
{{ setter(field)|indent(12)}}
{%- endfor %}
{%- endfor %}
@ -115,9 +206,9 @@ cdef class {{node.module_name}}_Reader(_DynamicStructReader):
self._init(struct.thisptr, struct._parent, struct.is_root, False)
self.thisptr_child = (<C_DynamicStruct_Reader>struct.thisptr).as{{node.module_name}}()
{% for field in node.struct.fields %}
cpdef _get_{{field.name}}(self) except +reraise_kj_exception:
cdef DynamicValue.Reader temp = self.thisptr_child.get{{field.c_name}}()
return to_python_reader(temp, self._parent)
{{ getfield(field, "Reader")|indent(4) }}
property {{field.name}}:
def __get__(self):
return self._get_{{field.name}}()
@ -133,7 +224,7 @@ cdef class {{node.module_name}}_Reader(_DynamicStructReader):
}
{% if node.is_union %}
which = self.which()
which = self._which_str()
ret[which] = getattr(self, which)
{% endif %}
@ -145,21 +236,8 @@ cdef class {{node.module_name}}_Builder(_DynamicStructBuilder):
self._init(struct.thisptr, struct._parent, struct.is_root, False)
self.thisptr_child = (<C_DynamicStruct_Builder>struct.thisptr).as{{node.module_name}}()
{% for field in node.struct.fields %}
cpdef _get_{{field.name}}(self) except +reraise_kj_exception:
cdef DynamicValue.Builder temp = self.thisptr_child.get{{field.c_name}}()
return to_python_builder(temp, self._parent)
cpdef _set_{{field.name}}(self, value) except +reraise_kj_exception:
_setDynamicFieldStatic(self.thisptr, "{{field.name}}", value, self._parent)
# cdef DynamicValue.Builder temp
# value_type = type(value)
# if value_type is list:
# builder = to_python_builder(self.thisptr_child.get{{field.c_name}}(), self._parent)
# _from_list(builder, value)
# elif value_type is dict:
# builder = to_python_builder(self.thisptr_child.get{{field.c_name}}(), self._parent)
# _from_dict(builder, value)
# else:
# self.thisptr_child.set{{field.c_name}}(to_dynamic_value(value))
{{ getfield(field, "Builder")|indent(4) }}
{{ setfield(field)|indent(4) }}
property {{field.name}}:
def __get__(self):
@ -178,11 +256,30 @@ cdef class {{node.module_name}}_Builder(_DynamicStructBuilder):
}
{% if node.is_union %}
which = self.which()
which = self._which_str()
ret[which] = getattr(self, which)
{% endif %}
return ret
def from_dict(self, dict d):
cdef str key
for key, val in d.iteritems():
if False: pass
{% for field in node.struct.fields %}
elif key == "{{field.name}}":
try:
self._set_{{field.name}}(val)
except Exception as e:
if 'expected isSetInUnion(field)' in str(e):
self.init(key)
self._set_{{field.name}}(val)
else:
raise
{%- endfor %}
else:
raise ValueError('Key not found in struct: ' + key)
capnp.register_type({{node.id}}, ({{node.module_name}}_Reader, {{node.module_name}}_Builder))
{% endfor %}

View file

@ -5,14 +5,21 @@ import os
import re
files = [{% for f in code.requestedFiles %}"{{f.filename}}",{% endfor %}]
files = [{% for f in code.requestedFiles %}"{{f.filename}}", {% endfor %}]
for f in files:
cpp_file = f + '.cpp'
if not os.path.exists(cpp_file):
if not os.path.exists(f + '.c++'):
cplus_file = f + '.c++'
cpp_mod = os.path.getmtime(cpp_file)
cplus_mod = 0
try:
cplus_mod = os.path.getmtime(cplus_file)
except:
pass
if not os.path.exists(cpp_file) or cpp_mod < cplus_mod:
if not os.path.exists(cplus_file):
raise RuntimeError("You need to run `capnp compile -oc++` in addition to `-ocython` first.")
os.rename(f + '.c++', cpp_file)
os.rename(cplus_file, cpp_file)
with open(f + '.h', "r") as file:
lines = file.readlines()

View file

@ -18,7 +18,6 @@ struct Person {
work @2;
}
}
employment :union {
unemployed @4 :Void;
employer @5 :Text;
@ -26,6 +25,13 @@ struct Person {
selfEmployed @7 :Void;
# We assume that a person is only one of these.
}
testGroup :group {
field1 @8 :UInt32;
field2 @9 :UInt32;
field3 @10 :UInt32;
}
extraData @11 :Data;
}
struct AddressBook {