mirrors/pycapnp
1
0
Fork 0
mirror of https://github.com/capnproto/pycapnp.git synced 2025-03-04 00:14:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Updating docs for v1.0.0b2

Browse source 
- Full cleanup of all the docs
- General sphinx housekeeping
- Updated all the old/bad links
- More reliable tests
- Updated Changelog
- Removed dead/deprecated code
- Added documentation generation test
This commit is contained in:
Jacob Alexander 2020-06-08 00:31:33 -07:00
parent 6532ca571d
commit 5e8ccba536
Failed to generate hash of commit
22 changed files with 752 additions and 328 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
.github/workflows/docs.yml vendored Normal file
View file

@ -0,0 +1,24 @@
name: Docs
on: [push, pull_request]
jobs:
docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Build pycapnp and install
run: |
python setup.py build
pip install .
- name: Build documentation
run: |
sphinx-build docs build/html

12
CHANGELOG.md
View file

@ -1,3 +1,14 @@
## v1.0.0b2 (2020-06-xx)
- Minimum capnproto version is now 0.8.0
- Added asyncio ssl calculator test
- Added poll_once to TwoPartyServer API
- More cleanup
- Fix absolute and circular imports
- Fix Promise aliasing issue (Promise to \_Promise)
- Documentation update
- Updated installation instructions
- Added RPC documentation for asyncio
## v1.0.0b1 (2019-12-26)
- Python 3.7+ required (asyncio support)
- TLS/SSL support using asyncio
@ -5,6 +16,7 @@
- General cleanup
- May be incompatible with code written for pycapnp 0.6.4 and lower
- Removing pypandoc/pandoc packaging requirement
- Minimum capnproto version is now 0.7.0
## v0.6.4 (2019-01-31)
- Fix bugs in `read_multiple_bytes` (thanks to @tsh56)

9
Pipfile
View file

@ -4,8 +4,11 @@ url = "https://pypi.org/simple"
verify_ssl = true
[packages]
pytest = "*"
tox = "*"
Jinja2 = "*"
Cython = "*"
Jinja2 = "*"
pkgconfig = "*"
pytest = "*"
sphinx = "*"
sphinx-multiversion = "*"
tox = "*"
wheel = "*"

4
README.md
View file

@ -7,7 +7,7 @@
[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/capnproto/pycapnp.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/capnproto/pycapnp/context:python)
[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/capnproto/pycapnp.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/capnproto/pycapnp/context:cpp)
[Cap'n'proto Mailing List](https://groups.google.com/forum/#!forum/capnproto)
[Cap'n'proto Mailing List](https://groups.google.com/forum/#!forum/capnproto) [Documentation](https://capnproto.github.io/pycapnp)
## Requirements
@ -103,7 +103,7 @@ python setup.py bdist_wheel
## Documentation/Example
There is some basic documentation [here](http://jparyani.github.io/pycapnp/).
There is some basic documentation [here](http://capnproto.github.io/pycapnp/).
Make sure to look at the [examples](examples). The examples are generally kept up to date with the recommended usage of the library.

1
capnp/__init__.py
View file

@ -45,6 +45,7 @@ from .lib.capnp import (
_DynamicStructReader,
_EventLoop,
_InterfaceModule,
_ListSchema,
_MallocMessageBuilder,
_PackedFdMessageReader,
_StreamFdMessageReader,

148
capnp/lib/capnp.pyx
View file

@ -12,27 +12,28 @@ cimport cython
from capnp.helpers.helpers cimport AsyncIoStreamReadHelper, init_capnp_api
from capnp.includes.capnp_cpp cimport AsyncIoStream, WaitScope, PyPromise, VoidPromise
from libc.stdlib cimport malloc, free
from libc.string cimport memcpy
from cython.operator cimport dereference as deref
from cpython.exc cimport PyErr_Clear
from cpython cimport array, Py_buffer, PyObject_CheckBuffer
from cpython.buffer cimport PyBUF_SIMPLE, PyBUF_WRITABLE
from cpython.exc cimport PyErr_Clear
from cython.operator cimport dereference as deref
from libc.stdlib cimport malloc, free
from libc.string cimport memcpy
from types import ModuleType as _ModuleType
import os as _os
import sys as _sys
import traceback as _traceback
from functools import partial as _partial
import warnings as _warnings
import inspect as _inspect
from operator import attrgetter as _attrgetter
import threading as _threading
import socket as _socket
import random as _random
import collections as _collections
import array
import asyncio
import collections as _collections
import inspect as _inspect
import os as _os
import random as _random
import socket as _socket
import sys as _sys
import threading as _threading
import traceback as _traceback
import warnings as _warnings
from types import ModuleType as _ModuleType
from operator import attrgetter as _attrgetter
from functools import partial as _partial
_CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR
_CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR
@ -1555,7 +1556,7 @@ cdef class _DynamicObjectReader:
return _DynamicCapabilityClient()._init(self.thisptr.getAsCapability(s.thisptr), self._parent)
cpdef as_list(self, schema) except +reraise_kj_exception:
cdef ListSchema s
cdef _ListSchema s
if hasattr(schema, 'schema'):
s = schema.schema
else:
@ -1597,7 +1598,7 @@ cdef class _DynamicObjectBuilder:
return _DynamicCapabilityClient()._init(self.thisptr.getAsCapability(s.thisptr), self._parent)
cpdef as_list(self, schema) except +reraise_kj_exception:
cdef ListSchema s
cdef _ListSchema s
if hasattr(schema, 'schema'):
s = schema.schema
else:
@ -1614,7 +1615,7 @@ cdef class _DynamicObjectBuilder:
self.thisptr.setAsText(text)
cpdef init_as_list(self, schema, size):
cdef ListSchema s
cdef _ListSchema s
if hasattr(schema, 'schema'):
s = schema.schema
else:
@ -1682,10 +1683,13 @@ cdef class _Timer:
return _VoidPromise()._init(self.thisptr.afterDelay(capnp.Nanoseconds(time)))
def getTimer():
"""
Get libcapnp event loop timer
"""
return _Timer()._init(helpers.getTimer(C_DEFAULT_EVENT_LOOP_GETTER().thisptr))
cpdef remove_event_loop(ignore_errors=False):
'Remove the global event loop'
'''Remove the global event loop'''
global C_DEFAULT_EVENT_LOOP
global _THREAD_LOCAL_EVENT_LOOPS
global _C_DEFAULT_EVENT_LOOP_LOCAL
@ -1721,16 +1725,22 @@ cpdef create_event_loop(threaded=True):
else:
C_DEFAULT_EVENT_LOOP = _EventLoop()
cpdef reset_event_loop():
global C_DEFAULT_EVENT_LOOP
C_DEFAULT_EVENT_LOOP._remove()
C_DEFAULT_EVENT_LOOP = _EventLoop()
cpdef reset_event_loop(ignore_errors=False, threaded=True):
'''Removes event loop, then creates a new one. See remove_event_loop and create_event_loop for more details.'''
remove_event_loop(ignore_errors)
create_event_loop(threaded)
def wait_forever():
"""
Use libcapnp event loop to poll/wait forever
"""
cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER()
helpers.waitNeverDone(deref(loop.thisptr).waitScope)
def poll_once():
"""
Poll libcapnp event loop once
"""
cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER()
helpers.pollWaitScope(deref(loop.thisptr).waitScope)
@ -1929,6 +1939,7 @@ cdef class _RemotePromise:
return _Response()._init_childptr(helpers.waitRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope), self._parent)
def wait(self):
"""Wait on the promise. This will block until the promise has completed."""
if self.is_consumed:
raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object')
@ -1937,6 +1948,12 @@ cdef class _RemotePromise:
return ret
async def a_wait(self):
"""
Asyncio version of wait().
Required when using asyncio for socket communication.
Will still work with non-asyncio socket communication, but requires async handling of the function call.
"""
if self.is_consumed:
raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object')
@ -1953,6 +1970,7 @@ cdef class _RemotePromise:
return _Promise()._init(helpers.convert_to_pypromise(deref(self.thisptr)), self)
cpdef then(self, func, error_func=None) except +reraise_kj_exception:
"""Promise pipelining, use to queue up operations on a promise before executing the promise."""
if self.is_consumed:
raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object')
@ -2236,6 +2254,16 @@ cdef class _TwoPartyVatNetwork:
return _VoidPromise()._init(deref(self.thisptr).onDisconnect(), self)
cdef class TwoPartyClient:
"""
TwoPartyClient for RPC Communication
Can be initialized using a socket wrapper (libcapnp) or using asyncio controlled sockets.
:param socket: Can be a string defining a socket (e.g. localhost:12345) or a file descriptor for a socket.
Passes socket directly to libcapnp. Do not use with asyncio.
:param traversal_limit_in_words: Pointer derefence limit (see https://capnproto.org/cxx.html).
:param nesting_limit: Recursive limit when reading types (see https://capnproto.org/cxx.html).
"""
cdef RpcSystem * thisptr
cdef public _TwoPartyVatNetwork _network
cdef public object _orig_stream
@ -2266,6 +2294,11 @@ cdef class TwoPartyClient:
Py_INCREF(self._network) # TODO:MEMORY: attach this to onDrained, also figure out what's leaking
async def read(self, bufsize):
"""
libcapnp reader (asyncio sockets only)
:param bufsize: Buffer size to read from the libcapnp library
"""
cdef AsyncIoStreamReadHelper *reader = new AsyncIoStreamReadHelper(
self._pipe._pipe.ends[1].get(),
&self._pipe._event_loop.thisptr.waitScope,
@ -2281,6 +2314,11 @@ cdef class TwoPartyClient:
return read_buffer
def write(self, data):
"""
libcapnp writer (asyncio sockets only)
:param data: Buffer to write to the libcapnp library
"""
cdef array.array write_buffer = array.array('b', data)
deref(self._pipe._pipe.ends[1]).write(
write_buffer.data.as_voidptr,
@ -2312,9 +2350,21 @@ cdef class TwoPartyClient:
return _VoidPromise()._init(deref(self._network.thisptr).onDisconnect())
cdef class TwoPartyServer:
"""
TwoPartyServer for RPC Communication
Can be initialized using a socket wrapper (libcapnp) or using asyncio controlled sockets.
:param socket: Can be a string defining a socket (e.g. localhost:12345, also supports \*:12345)
or a file descriptor for a socket.
Passes socket directly to libcapnp. Do not use with asyncio.
:param bootstrap: Class object defining the implementation of the Cap'n'proto interface.
:param traversal_limit_in_words: Pointer derefence limit (see https://capnproto.org/cxx.html).
:param nesting_limit: Recursive limit when reading types (see https://capnproto.org/cxx.html).
"""
cdef RpcSystem * thisptr
cdef public _TwoPartyVatNetwork _network
cdef public object _orig_stream, _server_socket, _disconnect_promise
cdef public object _orig_stream, _disconnect_promise
cdef public _AsyncIoStream _stream
cdef public _TwoWayPipe _pipe
cdef object _port
@ -2322,8 +2372,7 @@ cdef class TwoPartyServer:
cdef capnp.TaskSet * _task_set
cdef capnp.ErrorHandler _error_handler
def __init__(self, socket=None, server_socket=None, bootstrap=None,
traversal_limit_in_words=None, nesting_limit=None):
def __init__(self, socket=None, bootstrap=None, traversal_limit_in_words=None, nesting_limit=None):
if not bootstrap:
raise KjException("You must provide a bootstrap interface to a server constructor.")
@ -2343,7 +2392,6 @@ cdef class TwoPartyServer:
self._pipe = _TwoWayPipe()
self._network = _TwoPartyVatNetwork()._init_pipe(self._pipe, capnp.SERVER, make_reader_opts(traversal_limit_in_words, nesting_limit))
self._server_socket = server_socket
self._port = 0
if bootstrap:
@ -2359,6 +2407,11 @@ cdef class TwoPartyServer:
self._disconnect_promise = self.on_disconnect().then(self._decref)
async def read(self, bufsize):
"""
libcapnp reader (asyncio sockets only)
:param bufsize: Buffer size to read from the libcapnp library
"""
cdef AsyncIoStreamReadHelper *reader = new AsyncIoStreamReadHelper(
self._pipe._pipe.ends[1].get(),
&self._pipe._event_loop.thisptr.waitScope,
@ -2374,6 +2427,11 @@ cdef class TwoPartyServer:
return read_buffer
async def write(self, data):
"""
libcapnp writer (asyncio sockets only)
:param data: Buffer to write to the libcapnp library
"""
cdef array.array write_buffer = array.array('b', data)
deref(self._pipe._pipe.ends[1]).write(
write_buffer.data.as_voidptr,
@ -2407,9 +2465,15 @@ cdef class TwoPartyServer:
return _VoidPromise()._init(deref(self._network.thisptr).onDisconnect())
def poll_once(self):
"""
Poll libcapnp library one cycle.
"""
return poll_once()
async def poll_forever(self):
"""
Poll libcapnp library forever (asyncio)
"""
while True:
poll_once()
await asyncio.sleep(0.01)
@ -2592,7 +2656,7 @@ cdef typeAsSchema(capnp.SchemaType fieldType):
elif fieldType.isEnum():
return _EnumSchema()._init(fieldType.asEnum())
elif fieldType.isList():
return ListSchema()._init(fieldType.asList())
return _ListSchema()._init(fieldType.asList())
else:
raise KjException("Schema type is unknown")
@ -2811,14 +2875,14 @@ cdef _SchemaType _any_pointer = _SchemaType()
_any_pointer.thisptr = capnp.SchemaType(capnp.TypeWhichANY_POINTER)
types.AnyPointer = _any_pointer
cdef class ListSchema:
cdef class _ListSchema:
cdef C_ListSchema thisptr
def __init__(self, schema=None):
cdef _StructSchema ss
cdef _EnumSchema es
cdef _InterfaceSchema iis
cdef ListSchema ls
cdef _ListSchema ls
cdef _SchemaType st
if schema is not None:
@ -2837,7 +2901,7 @@ cdef class ListSchema:
elif typeSchema is _InterfaceSchema:
iis = s
self.thisptr = capnp.listSchemaOfInterface(iis.thisptr)
elif typeSchema is ListSchema:
elif typeSchema is _ListSchema:
ls = s
self.thisptr = capnp.listSchemaOfList(ls.thisptr)
elif typeSchema is _SchemaType:
@ -3060,15 +3124,6 @@ class _StructModule(object):
:rtype: :class:`_DynamicStructBuilder`
"""
return _new_message(self, kwargs, num_first_segment_words)
def from_dict(self, kwargs):
'.. warning:: This method is deprecated and will be removed in the 0.5 release. Use the :meth:`new_message` function instead with **kwargs'
_warnings.warn('This method is deprecated and will be removed in the 0.5 release. Use the :meth:`new_message` function instead with **kwargs', UserWarning)
return _new_message(self, kwargs, None)
def from_object(self, obj):
'.. warning:: This method is deprecated and will be removed in the 0.5 release. Use the :meth:`_DynamicStructReader.as_builder` or :meth:`_DynamicStructBuilder.copy` functions instead'
_warnings.warn('This method is deprecated and will be removed in the 0.5 release. Use the :meth:`_DynamicStructReader.as_builder` or :meth:`_DynamicStructBuilder.copy` functions instead', UserWarning)
builder = _MallocMessageBuilder()
return builder.set_root(obj)
class _InterfaceModule(object):
def __init__(self, schema, name):
@ -4074,12 +4129,17 @@ def add_import_hook(additional_paths=[]):
# Highest priority at position 0
extra_paths = [
_os.path.join(_os.path.dirname(__file__), '..'), # Built-in (only used if bundled)
'/usr/local/include/capnp', # Common macOS brew location
'/usr/include/capnp', # Common posix location
# Common macOS brew location
'/usr/local/include/capnp',
'/usr/local/include',
# Common posix location
'/usr/include/capnp',
'/usr/include',
]
for path in extra_paths:
if _os.path.isdir(path):
additional_paths.append(path)
if path not in additional_paths:
additional_paths.append(path)
_importer = _Importer(additional_paths)
_sys.meta_path.append(_importer)

8
docs/_templates/versioning.html vendored Normal file
View file

@ -0,0 +1,8 @@
{% if versions %}
<h3>{{ _('Versions') }}</h3>
<ul>
{%- for item in versions %}
<li><a href="{{ item.url }}">{{ item.name }}</a></li>
{%- endfor %}
</ul>
{% endif %}

250
docs/capnp.rst
View file

@ -1,103 +1,58 @@
.. _api:
API Reference
===================
=============
.. automodule:: capnp
.. currentmodule:: capnp
Internal Classes
----------------
These classes are internal to the library. You will never need to allocate
one yourself, but you may end up using some of their member methods.
Modules
~~~~~~~~~~
These are classes that are made for you when you import a Cap'n Proto file::
import capnp
import addressbook_capnp
print type(addressbook_capnp.Person) # capnp.capnp._StructModule
.. autoclass:: _StructModule
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _InterfaceModule
:members:
:undoc-members:
:inherited-members:
Readers
~~~~~~~~~~
.. autoclass:: _DynamicStructReader
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _DynamicListReader
:members:
:undoc-members:
:inherited-members:
Builders
~~~~~~~~~~
.. autoclass:: _DynamicStructBuilder
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _DynamicListBuilder
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _DynamicResizableListBuilder
:members:
:undoc-members:
:inherited-members:
RPC
~~~~~~~~~~~~~~~
.. autoclass:: _DynamicCapabilityClient
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _CapabilityClient
:members:
:undoc-members:
:inherited-members:
Miscellaneous
~~~~~~~~~~~~~
.. autoclass:: _DynamicOrphan
:members:
:undoc-members:
:inherited-members:
.. autoclass:: KjException
:members:
:undoc-members:
:inherited-members:
Functions
-------------
.. autofunction:: load
.. autofunction:: add_import_hook
.. autofunction:: remove_import_hook
.. autofunction:: join_promises
Classes
----------------
-------
RPC
~~~~~~~~~~~~~~~
~~~
Promise
#######
.. autoclass:: Promise
:members:
:undoc-members:
:inherited-members:
Promise may be one of:
* :meth:`capnp.lib.capnp._Promise`
* :meth:`capnp.lib.capnp._RemotePromise`
* :meth:`capnp.lib.capnp._VoidPromise`
* :meth:`PromiseFulfillerPair`
.. autoclass:: capnp.lib.capnp._Promise
:members:
:undoc-members:
:inherited-members:
.. autoclass:: capnp.lib.capnp._RemotePromise
:members:
:undoc-members:
:inherited-members:
.. autoclass:: capnp.lib.capnp._VoidPromise
:members:
:undoc-members:
:inherited-members:
.. autoclass:: PromiseFulfillerPair
:members:
:undoc-members:
:inherited-members:
Communication
#############
.. autoclass:: TwoPartyClient
:members:
:undoc-members:
@ -108,21 +63,136 @@ RPC
:undoc-members:
:inherited-members:
.. autoclass:: Promise
Capability
##########
.. autoclass:: capnp.lib.capnp._DynamicCapabilityClient
:members:
:undoc-members:
:inherited-members:
.. autoclass:: PromiseFulfillerPair
Response
########
.. autoclass:: capnp.lib.capnp._Response
:members:
:undoc-members:
:inherited-members:
Miscellaneous
~~~~~~~~~~~~~
.. autoclass:: KjException
:members:
:undoc-members:
:inherited-members:
.. autoclass:: SchemaParser
:members:
:undoc-members:
:inherited-members:
Functions
---------
.. autofunction:: add_import_hook
.. autofunction:: cleanup_global_schema_parser
.. autofunction:: create_event_loop
.. autofunction:: getTimer
.. autofunction:: join_promises
.. autofunction:: load
.. autofunction:: poll_once
.. autofunction:: remove_event_loop
.. autofunction:: remove_import_hook
.. autofunction:: reset_event_loop
.. autofunction:: wait_forever
Internal Classes
----------------
These classes are internal to the library. You will never need to allocate
one yourself, but you may end up using some of their member methods.
Modules
~~~~~~~
These are classes that are made for you when you import a Cap'n Proto file::
import capnp
import addressbook_capnp
print type(addressbook_capnp.Person) # capnp.capnp._StructModule
.. autoclass:: _InterfaceModule
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _StructModule
:members:
:undoc-members:
:inherited-members:
Readers
~~~~~~~
.. autoclass:: _DynamicListReader
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _DynamicStructReader
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _PackedFdMessageReader
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _StreamFdMessageReader
:members:
:undoc-members:
:inherited-members:
Builders
~~~~~~~~
.. autoclass:: _DynamicResizableListBuilder
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _DynamicListBuilder
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _DynamicStructBuilder
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _MallocMessageBuilder
:members:
:undoc-members:
:inherited-members:
RPC
~~~
.. autoclass:: _CapabilityClient
:members:
:undoc-members:
:inherited-members:
.. autoclass:: _DynamicCapabilityClient
:members:
:undoc-members:
:inherited-members:
Miscellaneous
~~~~~~~~~~~~~
.. autoclass:: _DynamicOrphan
:members:
:undoc-members:
:inherited-members:

15
docs/conf.py
View file

@ -30,7 +30,12 @@ import capnp
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx']
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.viewcode',
'sphinx.ext.intersphinx',
'sphinx_multiversion',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@ -46,7 +51,7 @@ master_doc = 'index'
# General information about the project.
project = u'capnp'
copyright = u'2013, Author'
copyright = u'2013-2019 (Jason Paryani), 2019-2020 (Jacob Alexander)'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -128,7 +133,7 @@ html_theme = 'nature'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# html_static_path = ['_static']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
@ -139,7 +144,7 @@ html_static_path = ['_static']
# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}
html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html', 'versioning.html'] }
# Additional templates that should be rendered to pages, maps page names to
# template names.
@ -293,3 +298,5 @@ epub_copyright = u'2013, Author'
# epub_tocdup = True
intersphinx_mapping = {'http://docs.python.org/': None}
smv_branch_whitelist = r'^master$'

19
docs/index.rst
View file

@ -1,13 +1,13 @@
.. capnp documentation master file
Welcome to pycapnp's documentation!
=================================
pycapnp
=======
This is a python wrapping of the C++ implementation of the `Cap'n Proto <https://capnproto.org/>`_ library. Here is a short description, quoted from its docs:
This is a python wrapping of the C++ implementation of the `Cap'n Proto <http://kentonv.github.io/capnproto/>`_ library. Here is a short description, quoted from its docs:
Cap’n Proto is an insanely fast data interchange format and capability-based RPC system. Think JSON, except binary. Or think Protocol Buffers, except faster. In fact, in benchmarks, Cap’n Proto is INFINITY TIMES faster than Protocol Buffers.
Since the python library is just a thin wrapping of the C++ library, we inherit a lot of what makes Cap'n Proto fast. In some simplistic benchmarks (available in the `benchmark directory of the repo <https://github.com/jparyani/pycapnp/tree/master/benchmark>`_), pycapnp has proven to be decently faster than Protocol Buffers (both pure python and C++ implementations). Also, the python capnp library can load Cap'n Proto schema files directly, without the need for a seperate compile step like with Protocol Buffers or Thrift. pycapnp is available on `github <https://github.com/jparyani/pycapnp.git>`_ and `pypi <https://pypi.python.org/pypi/pycapnp>`_.
Since the python library is just a thin wrapping of the C++ library, we inherit a lot of what makes Cap'n Proto fast. In some simplistic benchmarks (available in the `benchmark directory of the repo <https://github.com/capnproto/pycapnp/tree/master/benchmark>`_), pycapnp has proven to be decently faster than Protocol Buffers (both pure python and C++ implementations). Also, the python capnp library can load Cap'n Proto schema files directly, without the need for a seperate compile step like with Protocol Buffers or Thrift. pycapnp is available on `github <https://github.com/capnproto/pycapnp.git>`_ and `pypi <https://pypi.python.org/pypi/pycapnp>`_.
Contents:
@ -17,12 +17,3 @@ Contents:
install
quickstart
capnp
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

94
docs/install.rst
View file

@ -1,61 +1,97 @@
.. _install:
Installation
===================
============
Pip
---------------------
---
The pip installation will using the binary versions of the package (if possible). These contain a bundled version of capnproto (on Linux compiled with `manylinux <https://github.com/pypa/manylinux>`_). Starting from v1.0.0b1 binary releases are available for Windows, macOS and Linux from `pypi <https://pypi.org/project/pycapnp/#history>`_::
Using pip is by far the easiest way to install the library. After you've installed the C++ library, all you need to run is::
[sudo] pip install -U cython
[sudo] pip install -U setuptools
[sudo] pip install pycapnp
On some systems you will have to install Python's headers before doing any of this. For Debian/Ubuntu, this is::
To force rebuilding the pip package from source (you'll need requirments.txt or pipenv)::
sudo apt-get install python-dev
pip install --no-binary :all: pycapnp
You can control what compiler is used with the environment variable CC, ie. `CC=gcc-4.8 pip install pycapnp`, and flags with CFLAGS. You only need to run the setuptools line if you have a setuptools older than v0.8.0, and the cython line if you have a version older than v0.19.1.
To force bundling libcapnp (or force system libcapnp), just in case setup.py isn't doing the right thing::
pip install --no-binary :all: --install-option "--force-bundled-libcapnp"
pip install --no-binary :all: --install-option "--force-system-libcapnp"
If you're using an older Linux distro (e.g. CentOS 6) you many need to set `LDFLAGS="-Wl,--no-as-needed -lrt"`::
LDFLAGS="-Wl,--no-as-needed -lrt" pip install --no-binary :all: pycapnp
It's also possible to specify the libcapnp url when bundling (this may not work, there be dragons)::
pip install --no-binary :all: --install-option "--force-bundled-libcapnp" --install-option "--libcapnp-url" --install-option "https://github.com/capnproto/capnproto/archive/master.tar.gz"
From Source
---------------------
-----------
Source installation is generally not needed unless you're looking into an issue with capnproto or pycapnp itself.
C++ Cap'n Proto Library
~~~~~~~~~~~~~~~~~~~~~~~
You need to install the C++ Cap'n Proto library first. It requires a C++ compiler with C++11 support, such as GCC 4.7+ or Clang 3.2+. Follow installation docs at `http://kentonv.github.io/capnproto/install.html <http://kentonv.github.io/capnproto/install.html>`_, or if you're feeling lazy, you can run the commands below::
curl -O http://capnproto.org/capnproto-c++-0.5.0.tar.gz
tar zxf capnproto-c++-0.5.0.tar.gz
cd capnproto-c++-0.5.0
./configure
make -j6 check
sudo make install
You need to install the C++ Cap'n Proto library first. It requires a C++ compiler with C++14 support, such as GCC 5+ or Clang 5+. Follow installation docs at `https://capnproto.org/install.html <https://capnproto.org/install.html>`_.
pycapnp from git
~~~~~~~~~~~~~~~~
If you want the latest development version, you can clone the github repo and install like so::
If you want the latest development version, you can clone the github repo::
git clone https://github.com/jparyani/pycapnp.git
pip install ./pycapnp
git clone https://github.com/capnproto/pycapnp.git
For development packages use one of the following to install the python dependencies::
pipenv install
pip install -r requirements.txt
And install pycapnp with::
cd pycapnp
pip install .
or::
cd pycapnp
python setup.py install
Development
-------------------
-----------
Clone the repo from https://github.com/capnproto/pycapnp.git::
Clone the repo from https://github.com/jparyani/pycapnp.git and use the `develop` branch. I'll probably ask you to redo pull requests that target `master` and aren't easily mergable to `develop`::
git clone https://github.com/capnproto/pycapnp.git
git clone https://github.com/jparyani/pycapnp.git
git checkout develop
For development packages use one of the following to install the python dependencies::
Testing is done through pytest, like so::
pipenv install
pip install -r requirements.txt
pip install pytest
py.test
Building::
cd pycapnp
pip install .
or::
cd pycapnp
python setup.py install
Useful targets for setup.py::
python setup.py build
python setup.py clean
Useful command-line arguments are available for setup.py::
--force-bundled-libcapnp
--force-system-libcapnp
--libcapnp-url
Testing is done through pytest::
cd pycapnp
pytest
pytest test/test_rpc_calculator.py
Once you're done installing, take a look at the :ref:`quickstart`

426
docs/quickstart.rst
View file

@ -1,17 +1,17 @@
.. _quickstart:
Quickstart
===================
==========
This assumes you already have the capnp library installed. If you don't, please follow the instructions at :ref:`Installation <install>` first.
In general, this library is a very light wrapping of the `Cap'n Proto C++ library <http://kentonv.github.io/capnproto/cxx.html>`_. You can refer to its docs for more advanced concepts, or just to get a basic idea of how the python library is structured.
In general, this library is a very light wrapping of the `Cap'n Proto C++ library <https://capnproto.org/cxx.html>`_. You can refer to its docs for more advanced concepts, or just to get a basic idea of how the python library is structured.
Load a Cap'n Proto Schema
-------------------------
First you need to import the library::
import capnp
Then you can load the Cap'n Proto schema with::
@ -23,7 +23,7 @@ This will look all through all the directories in your sys.path/PYTHONPATH, and
capnp.remove_import_hook()
addressbook_capnp = capnp.load('addressbook.capnp')
For future reference, here is the Cap'n Proto schema. Also available in the github repository under `examples/addressbook.capnp <https://github.com/jparyani/pycapnp/tree/master/examples>`_::
For future reference, here is the Cap'n Proto schema. Also available in the github repository under `examples/addressbook.capnp <https://github.com/capnproto/pycapnp/tree/master/examples>`_::
# addressbook.capnp
@0x934efea7f017fff0;
@ -60,16 +60,17 @@ For future reference, here is the Cap'n Proto schema. Also available in the gith
people @0 :List(Person);
}
Const values
~~~~~~~~~~~~~~
Const values
~~~~~~~~~~~~
Const values show up just as you'd expect under the loaded schema. For example::
print addressbook_capnp.qux
# 123
Build a message
------------------
---------------
Initialize a New Cap'n Proto Object
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -87,11 +88,11 @@ Also as a shortcut, you can pass keyword arguments to the `new_message` function
person = addressbook_capnp.Person.new_message()
person.name = 'alice'
List
~~~~~~~~~~~~~~~~~~~~~~~~~~~
List
~~~~
Allocating a list inside of an object requires use of the `init` function::
people = addresses.init('people', 2)
For now, let's grab the first element out of this list and assign it to a variable named `alice`::
@ -100,9 +101,9 @@ For now, let's grab the first element out of this list and assign it to a variab
.. note:: It is a very bad idea to call `init` more than once on a single field. Every call to `init` allocates new memory inside your Cap'n Proto message, and if you call it more than once, the previous memory is left as dead space in the message. See `Tips and Best Practices <https://capnproto.org/cxx.html#tips-and-best-practices>`_ for more details.
Primitive Types
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Primitive Types
~~~~~~~~~~~~~~~
For all primitive types, from the Cap'n Proto docs:
- Boolean: Bool
@ -119,9 +120,9 @@ You can assign straight to the variable with the corresponding Python type. For
.. note:: Text fields will behave differently depending on your version of Python. In Python 2.x, Text fields will expect and return a `bytes` string, while in Python 3.x, they will expect and return a `unicode` string. Data fields will always a return `bytes` string.
Enums
~~~~~~~~~~~~~~
Enums
~~~~~
First we'll allocate a length one list of phonenumbers for `alice`::
alicePhone = alice.init('phones', 1)[0]
@ -129,7 +130,7 @@ First we'll allocate a length one list of phonenumbers for `alice`::
Note that even though it was a length 1 list, it was still a list that was returned, and we extracted the first (and only) element with `[0]`.
Enums are treated like strings, and you assign to them like they were a Text field::
alicePhone.type = 'mobile'
If you assign an invalid value to one, you will get a ValueError::
@ -140,8 +141,9 @@ If you assign an invalid value to one, you will get a ValueError::
...
ValueError: src/capnp/schema.c++:326: requirement not met: enum has no such enumerant; name = foo
Unions
~~~~~~~~~~~~~~~~~~
~~~~~~
For the most part, you just treat them like structs::
alice.employment.school = "MIT"
@ -150,10 +152,10 @@ Now the `school` field is the active part of the union, and we've assigned `'MIT
Also, one weird case is for Void types in Unions (and in general, but Void is really only used in Unions). For these, you will have to assign `None` to them::
bob.employment.unemployed = None
bob.employment.unemployed = None
.. note:: One caveat for unions is having structs as union members. Let us assume `employment.school` was actually a struct with a field of type `Text` called `name`::
alice.employment.school.name = "MIT"
# Raises a ValueError
@ -164,9 +166,9 @@ Also, one weird case is for Void types in Unions (and in general, but Void is re
Note that this is similar to `init` for lists, but you don't pass a size. Requiring the `init` makes it more clear that a memory allocation is occurring, and will hopefully make you mindful that you shouldn't set more than 1 field inside of a union, else you risk a memory leak
Writing to a File
~~~~~~~~~~~~~~~~~~~
Writing to a File
~~~~~~~~~~~~~~~~~
Once you're done assigning to all the fields in a message, you can write it to a file like so::
f = open('example.bin', 'w+b')
@ -174,12 +176,12 @@ Once you're done assigning to all the fields in a message, you can write it to a
There is also a `write_packed` function, that writes out the message more space-efficientally. If you use write_packed, make sure to use read_packed when reading the message.
Read a message
-----------------
--------------
Reading from a file
~~~~~~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~~~~~~~~
Much like before, you will have to de-serialize the message from a file descriptor::
f = open('example.bin', 'rb')
@ -194,9 +196,9 @@ Note that this very much needs to match the type you wrote out. In general, you
}
}
Reading Fields
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Reading Fields
~~~~~~~~~~~~~~
Fields are very easy to read. You just use the `.` syntax as before. Lists behave just like normal Python lists::
for person in addresses.people:
@ -204,9 +206,9 @@ Fields are very easy to read. You just use the `.` syntax as before. Lists behav
for phone in person.phones:
print(phone.type, ':', phone.number)
Reading Unions
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Reading Unions
~~~~~~~~~~~~~~
The only tricky one is unions, where you need to call `.which()` to determine the union type. The `.which()` call returns an enum, ie. a string, corresponding to the field name::
which = person.employment.which()
@ -222,14 +224,14 @@ The only tricky one is unions, where you need to call `.which()` to determine th
print('self employed')
print()
Serializing/Deserializing
--------------
-------------------------
Files
~~~~~~~~~~~~~~~~~~~~~~~~~~
~~~~~
As shown in the examples above, there is file serialization with `write()`::
addresses = addressbook_capnp.AddressBook.new_message()
...
f = open('example.bin', 'w+b')
@ -246,9 +248,9 @@ There are packed versions as well::
f.seek(0)
addresses = addressbook_capnp.AddressBook.read_packed(f)
Multi-message files
~~~~~~~~~~~~~~~~~~~~~~~~~~
Multi-message files
~~~~~~~~~~~~~~~~~~~
The above methods only guaranteed to work if your file contains a single message. If you have more than 1 message serialized sequentially in your file, then you need to use these convenience functions::
addresses = addressbook_capnp.AddressBook.new_message()
@ -268,8 +270,7 @@ There is also a packed version::
print addresses
Dictionaries
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~
There is a convenience method for converting Cap'n Proto messages to a dictionary. This works for both Builder and Reader type messages::
alice.to_dict()
@ -290,8 +291,7 @@ It's also worth noting, you can use python lists/dictionaries interchangably wit
Byte Strings/Buffers
~~~~~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~~~~~~~~~
There is serialization to a byte string available::
encoded_message = alice.to_bytes()
@ -307,7 +307,6 @@ There are also packed versions::
Byte Segments
~~~~~~~~~~~~~
.. note:: This feature is not supported in PyPy at the moment, pending investigation.
Cap'n Proto supports a serialization mode which minimizes object copies. In the C++ interface, ``capnp::MessageBuilder::getSegmentsForOutput()`` returns an array of pointers to segments of the message's content without copying. ``capnp::SegmentArrayMessageReader`` performs the reverse operation, i.e., takes an array of pointers to segments and uses the underlying data, again without copying. This produces a different wire serialization format from ``to_bytes()`` serialization, which uses ``capnp::messageToFlatArray()`` and ``capnp::FlatArrayMessageReader`` (both of which use segments internally, but write them in an incompatible way).
@ -328,18 +327,25 @@ For more information, please refer to the following links:
RPC
----------
---
Cap'n Proto has a rich RPC protocol. You should read the `RPC specification <http://kentonv.github.io/capnproto/rpc.html>`_ as well as the `C++ RPC documentation <http://kentonv.github.io/capnproto/cxxrpc.html>`_ before using pycapnp's RPC features. As with the serialization part of this library, the RPC component tries to be a very thin wrapper on top of the C++ API.
Cap'n Proto has a rich RPC protocol. You should read the `RPC specification <https://capnproto.org/rpc.html>`_ as well as the `C++ RPC documentation <http://kentonv.github.io/capnproto/cxxrpc.html>`_ before using pycapnp's RPC features. As with the serialization part of this library, the RPC component tries to be a very thin wrapper on top of the C++ API.
The examples below will be using `calculator.capnp <https://github.com/capnproto/pycapnp/blob/master/examples/calculator.capnp>`_. Please refer to it to understand the interfaces that will be used.
Asyncio support was added to pycapnp in v1.0.0 utilizing the TwoWayPipe interface to libcapnp (instead of having libcapnp control the socket communication). The main advantage here is that standard Python socket libraries can be used with pycapnp (more importantly, TLS/SSL). Asyncio requires a bit more boiler plate to get started but it does allow for a lot more control than using the pycapnp socket wrapper.
The examples below will be using `calculator.capnp <https://github.com/jparyani/pycapnp/blob/master/examples/calculator.capnp>`_. Please refer to it to understand the interfaces that will be used.
Client
~~~~~~~~~~~~~~
~~~~~~
There are two ways to start a client: libcapnp socket wrapper and asyncio.
The wrapper is easier to implement but is very limited (doesn't support SSL/TLS with Python).
asyncio requires more setup and can be harder to debug; however, it does support SSL/TLS and has more control over the socket error conditions. asyncio also helps get around the threading limitations around the current pycapnp implementation has with libcapnp (pycapnp objects and functions must all be in the same thread).
Starting a Client
################
#################
Starting a client is very easy::
import capnp
@ -348,23 +354,128 @@ Starting a client is very easy::
client = capnp.TwoPartyClient('localhost:60000')
.. note:: You can also pass a raw socket with a `fileno()` method to TwoPartyClient
.. note:: This will not work with SSL/TLS, please see :ref:`rpc-asyncio-client`
Restoring
###################
Before you do anything else, you will need to restore a capability from the server. Refer to the `Cap'n Proto docs <http://kentonv.github.io/capnproto/rpc.html>`_ if you don't know what this means::
.. _rpc-asyncio-client:
calculator = client.ez_restore('calculator').cast_as(calculator_capnp.Calculator)
Starting a Client (asyncio)
###########################
Asyncio takes a bit more boilerplate than using the socket wrapper, but it gives you a lot more control. The example here is very simplistic. Here's an example of full error handling (with reconnection on server failure): `hidio client <https://github.com/hid-io/hid-io-core/blob/master/python/hidiocore/client/__init__.py>`_.
There's two things worth noting here. First, we used the simpler `ez_restore` function. For servers that use a struct type as their Restorer, you will have to do the following instead::
At a basic level, asyncio splits the input and output streams of the tcp socket and sends it to the libcapnp TwoWayPipe interface. An async reader Python function/method is used to consume the incoming byte stream and an async writer Python function/method is used to write outgoing bytes to the socket.
calculator = client.restore(calculator_capnp.MyStructType.new_message(foo='bar')).cast_as(calculator_capnp.Calculator)
.. note:: You'll need to be using the async keyword on some of the Python function/methods. If you're unsure, look at the full `example code <https://github.com/capnproto/pycapnp/blob/master/examples/async_calculator_client.py>`_. Also, read up on recent Python asyncio tutorials if you're new to the concept. Make sure the tutorial is 3.7+, asyncio changed a lot from when it was first introduced in 3.4.
First you'll need two basic async functions::
async def myreader(client, reader):
while True:
data = await reader.read(4096)
client.write(data)
async def mywriter(client, writer):
while True:
data = await client.read(4096)
writer.write(data.tobytes())
await writer.drain()
.. note:: There's no socket error handling here, so this won't be sufficient for anything beyond a simple example.
Next you'll need to define an async function that sets up the socket connection. This is equivalent to `client = capnp.TwoPartyClient('localhost:60000')` in the earlier example::
async def main(host):
addr = 'localhost'
port = '6000'
# Handle both IPv4 and IPv6 cases
try:
print("Try IPv4")
reader, writer = await asyncio.open_connection(
addr, port,
family=socket.AF_INET
)
except Exception:
print("Try IPv6")
reader, writer = await asyncio.open_connection(
addr, port,
family=socket.AF_INET6
)
# Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
client = capnp.TwoPartyClient()
# Assemble reader and writer tasks, run in the background
coroutines = [myreader(client, reader), mywriter(client, writer)]
asyncio.gather(*coroutines, return_exceptions=True)
## Bootstrap Here ##
.. note:: On systems that have both IPv4 and IPv6 addresses, IPv6 is often resolved first and needs to be handled separately. If you're certain IPv6 won't be used, you can remove it (you should also avoid localhost, and stick to something like 127.0.0.1).
Finally, you'll need to start the asyncio function::
if __name__ == '__main__':
asyncio.run(main(parse_args().host))
.. note:: This is the simplest way to start asyncio and usually not sufficient for most applications.
SSL/TLS Client
^^^^^^^^^^^^^^
SSL/TLS setup effectively wraps the socket transport. You'll need an SSL certificate, for this example we'll be using a self-signed certificate. Most of the asyncio setup is the same as above::
async def main(host):
addr = 'localhost'
port = '6000'
# Setup SSL context
ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, 'selfsigned.cert'))
# Handle both IPv4 and IPv6 cases
try:
print("Try IPv4")
reader, writer = await asyncio.open_connection(
addr, port,
ssl=ctx,
family=socket.AF_INET
)
except Exception:
print("Try IPv6")
reader, writer = await asyncio.open_connection(
addr, port,
ssl=ctx,
family=socket.AF_INET6
)
# Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
client = capnp.TwoPartyClient()
# Assemble reader and writer tasks, run in the background
coroutines = [myreader(client, reader), mywriter(client, writer)]
asyncio.gather(*coroutines, return_exceptions=True)
## Bootstrap Here ##
Due to a `bug <https://bugs.python.org/issue36709>`_ in Python 3.7 and 3.8 asyncio client needs to be initialized in a slightly different way::
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main(parse_args().host))
Bootstrap
#########
Before calling any methods you'll need to bootstrap the Calculator interface::
calculator = client.bootstrap().cast_as(calculator_capnp.Calculator)
There's two things worth noting here. First, we are asking for the server capability. Secondly, you see that we are casting the capability that we receive. This is because capabilities are intrinsically dynamic, and they hold no run time type information, so we need to pick what interface to interpret them as.
Secondly, you see that we are casting the capability that we receive. This is because capabilities are intrinsically dynamic, and they hold no run time type information, so we need to pick what interface to interpret them as.
Calling methods
################
###############
There are 2 ways to call RPC methods. First the more verbose `request` syntax::
request = calculator.evaluate_request()
@ -390,37 +501,184 @@ The second is to build a promise chain by calling `then`::
eval_promise.then(do_stuff).wait()
Pipelining
#################
Pipelining
##########
If a method returns values that are themselves capabilites, then you can access these fields before having to call `wait`. Doing this is called pipelining, and it allows Cap'n Proto to chain the calls without a round-trip occurring to the server::
# evaluate returns `value` which is itself an interface.
# You can call a new method on `value` without having to call wait first
read_promise = eval_promise.value.read()
read_result = read_promise.wait() # only 1 wait call
read_result = read_promise.wait() # only 1 wait call
You can also chain promises with `then` and the same pipelining will occur::
read_result = eval_promise.then(lambda ret: ret.value.read()).wait()
Server
~~~~~~~~~~~~~~
~~~~~~
There are two ways to start a server: libcapnp socket wrapper and asyncio.
The wrapper is easier to implement but is very limited (doesn't support SSL/TLS with Python).
asyncio requires more setup and can be harder to debug; however, it does support SSL/TLS and has more control over the socket error conditions. asyncio also helps get around the threading limitations around the current pycapnp implementation has with libcapnp (pycapnp objects and functions must all be in the same thread). The asyncio Server is a bit more work to implement than an asyncio client as more error handling is required to deal with client connection/disconnection/timeout events.
Starting a Server
##################
#################
server = capnp.TwoPartyServer('*:60000', restore)
To start a server::
server = capnp.TwoPartyServer('*:60000', bootstrap=CalculatorImpl())
server.run_forever()
See the `Restore`_ section for an explanation of what the `restore` object needs to looks like.
.. note:: You can also pass a raw socket with a `fileno()` method to TwoPartyServer. In that case, `run_forever` will not work, and you will have to use `on_disconnect.wait()`.
.. note:: This will not work with SSL/TLS, please see :ref:`rpc-asyncio-server`
.. _rpc-asyncio-server:
Starting a Server (asyncio)
###########################
Like the asyncio client, an asyncio server takes a bunch of boilerplate as opposed to using the socket wrapper. Servers generally have to handle a lot more error conditions than clients so they are generally more complicated to implement with asyncio.
Just like the asyncio client, both the input and output socket streams are handled by reader/writer callback functions/methods.
.. note:: You'll need to be using the async keyword on some of the Python function/methods. If you're unsure, look at the full `example code <https://github.com/capnproto/pycapnp/blob/master/examples/async_calculator_client.py>`_. Also, read up on recent Python asyncio tutorials if you're new to the concept. Make sure the tutorial is 3.7+, asyncio changed a lot from when it was first introduced in 3.4.
To simplify the callbacks use a server class to define the reader/writer callbacks.::
class Server:
async def myreader(self):
while self.retry:
try:
# Must be a wait_for so we don't block on read()
data = await asyncio.wait_for(
self.reader.read(4096),
timeout=0.1
)
except asyncio.TimeoutError:
print("myreader timeout.")
continue
except Exception as err:
print("Unknown myreader err: %s", err)
return False
await self.server.write(data)
print("myreader done.")
return True
async def mywriter(self):
while self.retry:
try:
# Must be a wait_for so we don't block on read()
data = await asyncio.wait_for(
self.server.read(4096),
timeout=0.1
)
self.writer.write(data.tobytes())
except asyncio.TimeoutError:
print("mywriter timeout.")
continue
except Exception as err:
print("Unknown mywriter err: %s", err)
return False
print("mywriter done.")
return True
We need an additional `myserver()` method in the `Server` class to handle each of the incoming socket connections::
async def myserver(self, reader, writer):
# Start TwoPartyServer using TwoWayPipe (only requires bootstrap)
self.server = capnp.TwoPartyServer(bootstrap=CalculatorImpl())
self.reader = reader
self.writer = writer
self.retry = True
# Assemble reader and writer tasks, run in the background
coroutines = [self.myreader(), self.mywriter()]
tasks = asyncio.gather(*coroutines, return_exceptions=True)
while True:
self.server.poll_once()
# Check to see if reader has been sent an eof (disconnect)
if self.reader.at_eof():
self.retry = False
break
await asyncio.sleep(0.01)
# Make wait for reader/writer to finish (prevent possible resource leaks)
await tasks
Finally, we'll need to start an asyncio server to spawn a new async `myserver()` with it's own `Server()` object for each new connection::
async def new_connection(reader, writer):
server = Server()
await server.myserver(reader, writer)
async def main():
addr = 'localhost'
port = '60000'
# Handle both IPv4 and IPv6 cases
try:
print("Try IPv4")
server = await asyncio.start_server(
new_connection,
addr, port,
family=socket.AF_INET
)
except Exception:
print("Try IPv6")
server = await asyncio.start_server(
new_connection,
addr, port,
family=socket.AF_INET6
)
async with server:
await server.serve_forever()
if __name__ == '__main__':
asyncio.run(main())
.. note:: On systems that have both IPv4 and IPv6 addresses, IPv6 is often resolved first and needs to be handled separately. If you're certain IPv6 won't be used, you can remove it (you should also avoid localhost, and stick to something like 127.0.0.1). If you're broadcasting in general, you'll probably want to use `0.0.0.0` (IPv4) or `::/0` (IPv6).
SSL/TLS Server
^^^^^^^^^^^^^^
Adding SSL/TLS support for a pycapnp asyncio server is fairly straight-forward. Just create an SSL context before starting the asyncio server::
async def main():
addr = 'localhost'
port = '60000'
# Setup SSL context
ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ctx.load_cert_chain(os.path.join(this_dir, 'selfsigned.cert'), os.path.join(this_dir, 'selfsigned.key'))
# Handle both IPv4 and IPv6 cases
try:
print("Try IPv4")
server = await asyncio.start_server(
new_connection,
addr, port,
ssl=ctx,
family=socket.AF_INET
)
except Exception:
print("Try IPv6")
server = await asyncio.start_server(
new_connection,
addr, port,
ssl=ctx,
family=socket.AF_INET6
)
async with server:
await server.serve_forever()
Implementing a Server
#######################
#####################
Here's a part of how you would implement a Calculator server::
class CalculatorImpl(calculator_capnp.Calculator.Server):
@ -441,7 +699,7 @@ Some major things worth noting.
- You must inherit from `your_module_capnp.YourInterface.Server`, but don't worry about calling __super__ in your __init__
- Method names of your class must either match the interface exactly, or have '_context' appended to it
- If your method name is exactly the same as the interface, then you will be passed all the arguments from the interface as keyword arguments, so your argument names must match the interface spec exactly. You will also receive a `_context` parameter which is equivalent to the C++ API's Context. I highly recommend having **kwargs as well, so that even if your interface spec is upgraded and arguments were added, your server will still operate fine.
- If your method name is exactly the same as the interface, then you will be passed all the arguments from the interface as keyword arguments, so your argument names must match the interface spec exactly. You will also receive a `_context` parameter which is equivalent to the C++ API's Context. I highly recommend having `**kwargs` as well, so that even if your interface spec is upgraded and arguments were added, your server will still operate fine.
- Returns work with a bit of magic as well. If you return a promise, then it will be handled the same as if you returned a promise from a server method in the C++ API. Otherwise, your return statement will be filled into the results struct following the ordering in your spec, for example::
# capability.capnp file
@ -456,39 +714,7 @@ Some major things worth noting.
- If your method ends in _context, then you will only be passed a context parameter. You will have to access params and set results yourself manually. Returning promises still works as above, but you can't return anything else from a method.
Restore
###########
Restoring can occur in either a class::
class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer):
def restore(self, ref_id):
if ref_id.tag == 'testInterface':
return Server(100)
else:
raise Exception('Unrecognized ref passed to restore')
...
server = capnp.TwoPartyServer(sock, SimpleRestorer())
Where you inherit from StructType.Restorer, and the argument passed to restore will be cast to the proper type.
Otherwise, restore can be a function::
def restore(ref_id):
if ref_id.as_struct(test_capability_capnp.TestSturdyRefObjectId).tag == 'testInterface':
return Server(100)
else:
raise Exception('Unrecognized ref passed to restore')
...
server = capnp.TwoPartyServer(sock, restore)
Full Examples
------------------
`Full examples <https://github.com/jparyani/pycapnp/blob/master/examples>`_ are available on github. There is also an example of a very simplistic RPC available in `test_rpc.py <https://github.com/jparyani/pycapnp/blob/master/test/test_rpc.py>`_.
-------------
`Full examples <https://github.com/capnproto/pycapnp/blob/master/examples>`_ are available on github. There is also an example of a very simplistic RPC available in `test_rpc.py <https://github.com/capnproto/pycapnp/blob/master/test/test_rpc.py>`_.

5
examples/async_calculator_client.py
View file

@ -70,10 +70,7 @@ async def main(host):
coroutines = [myreader(client, reader), mywriter(client, writer)]
asyncio.gather(*coroutines, return_exceptions=True)
# Pass "calculator" to ez_restore (there's also a `restore` function that
# takes a struct or AnyPointer as an argument), and then cast the returned
# capability to it's proper type. This casting is due to capabilities not
# having a reference to their schema
# Bootstrap the Calculator interface
calculator = client.bootstrap().cast_as(calculator_capnp.Calculator)
'''Make a request that just evaluates the literal value 123.

2
examples/async_ssl_client.py
View file

@ -11,8 +11,6 @@ import capnp
import thread_capnp
this_dir = os.path.dirname(os.path.abspath(__file__))
capnp.remove_event_loop()
capnp.create_event_loop(threaded=True)
def parse_args():

5
examples/calculator_client.py
View file

@ -33,10 +33,7 @@ at the given address and does some RPCs')
def main(host):
client = capnp.TwoPartyClient(host)
# Pass "calculator" to ez_restore (there's also a `restore` function that
# takes a struct or AnyPointer as an argument), and then cast the returned
# capability to it's proper type. This casting is due to capabilities not
# having a reference to their schema
# Bootstrap the server capability and cast it to the Calculator interface
calculator = client.bootstrap().cast_as(calculator_capnp.Calculator)
'''Make a request that just evaluates the literal value 123.

2
requirements.txt
View file

@ -3,5 +3,7 @@ cython
setuptools
pkgconfig
pytest
sphinx
sphinx-multiversion
tox
wheel

2
setup.py
View file

@ -23,7 +23,7 @@ _this_dir = os.path.dirname(__file__)
MAJOR = 1
MINOR = 0
MICRO = 0
TAG = 'b1'
TAG = 'b2'
VERSION = '%d.%d.%d%s' % (MAJOR, MINOR, MICRO, TAG)

30
test/test_examples.py
View file

@ -18,17 +18,28 @@ def cleanup():
p.kill()
def run_subprocesses(address, server, client):
def run_subprocesses(address, server, client, wildcard_server=False, ipv4_force=True):
server_attempt = 0
server_attempts = 2
done = False
addr, port = address.split(':')
c_address = address
s_address = address
while not done:
assert server_attempt < server_attempts, "Failed {} server attempts".format(server_attempts)
server_attempt += 1
# Force ipv4 for tests (known issues on GitHub Actions with IPv6 for some targets)
if 'unix' not in addr and ipv4_force:
addr = socket.gethostbyname(addr)
c_address = '{}:{}'.format(addr, port)
s_address = c_address
if wildcard_server:
s_address = '*:{}'.format(port) # Use wildcard address for server
print("Forcing ipv4 -> {} => {} {}".format(address, c_address, s_address))
# Start server
cmd = [sys.executable, os.path.join(examples_dir, server), address]
cmd = [sys.executable, os.path.join(examples_dir, server), s_address]
serverp = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
print("Server started (Attempt #{})".format(server_attempt))
processes.append(serverp)
@ -74,7 +85,7 @@ def run_subprocesses(address, server, client):
client_attempt += 1
# Start client
cmd = [sys.executable, os.path.join(examples_dir, client), address]
cmd = [sys.executable, os.path.join(examples_dir, client), c_address]
clientp = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
print("Client started (Attempt #{})".format(client_attempt))
processes.append(clientp)
@ -96,11 +107,6 @@ def run_subprocesses(address, server, client):
clientp.kill()
break
# Retrying with different address (ipv4)
if 'unix' not in addr:
addr = socket.gethostbyname(addr)
address = '{}:{}'.format(addr, port)
print("Forcing ipv4 -> {}".format(address))
serverp.kill()
serverp.kill()
@ -117,7 +123,7 @@ def test_thread_example(cleanup):
address = '{}:36433'.format(hostname)
server = 'thread_server.py'
client = 'thread_client.py'
run_subprocesses(address, server, client)
run_subprocesses(address, server, client, wildcard_server=True)
def test_addressbook_example(cleanup):
@ -139,7 +145,7 @@ def test_ssl_async_example(cleanup):
address = '{}:36435'.format(hostname)
server = 'async_ssl_server.py'
client = 'async_ssl_client.py'
run_subprocesses(address, server, client)
run_subprocesses(address, server, client, ipv4_force=False)
@pytest.mark.skipif(sys.platform == "win32", reason="Asyncio bug with libcapnp timer, likely due to asyncio starving some event loop. See https://github.com/capnproto/pycapnp/issues/196")
@ -147,11 +153,11 @@ def test_ssl_reconnecting_async_example(cleanup):
address = '{}:36436'.format(hostname)
server = 'async_ssl_server.py'
client = 'async_reconnecting_ssl_client.py'
run_subprocesses(address, server, client)
run_subprocesses(address, server, client, ipv4_force=False)
def test_async_ssl_calculator_example(cleanup):
address = '{}:36437'.format(hostname)
server = 'async_ssl_calculator_server.py'
client = 'async_ssl_calculator_client.py'
run_subprocesses(address, server, client)
run_subprocesses(address, server, client, ipv4_force=False)

2
test/test_object.py
View file

@ -28,7 +28,7 @@ def test_object_basic(addressbook):
def test_object_list(addressbook):
obj = capnp._MallocMessageBuilder().get_root_as_any()
listSchema = capnp.ListSchema(addressbook.Person)
listSchema = capnp._ListSchema(addressbook.Person)
people = obj.init_as_list(listSchema, 2)
person = people[0]
person.name = 'test'

2
test/test_rpc_calculator.py
View file

@ -35,7 +35,7 @@ def test_calculator():
def test_calculator_tcp(cleanup):
address = 'localhost:36431'
test_examples.run_subprocesses(address, 'calculator_server.py', 'calculator_client.py')
test_examples.run_subprocesses(address, 'calculator_server.py', 'calculator_client.py', wildcard_server=True)
@pytest.mark.skipif(os.name == 'nt', reason="socket.AF_UNIX not supported on Windows")

6
test/test_schema.py
View file

@ -25,7 +25,7 @@ def test_list_schema(addressbook):
assert personType.node.id == addressbook.Person.schema.node.id
personListSchema = capnp.ListSchema(addressbook.Person)
personListSchema = capnp._ListSchema(addressbook.Person)
assert personListSchema.elementType.node.id == addressbook.Person.schema.node.id
@ -40,11 +40,11 @@ def test_annotations(annotations):
assert annotation.value.struct.as_struct(annotations.AnnotationStruct).test == 100
annotation = annotations.TestAnnotationThree.schema.node.annotations[0]
annotation_list = annotation.value.list.as_list(capnp.ListSchema(annotations.AnnotationStruct))
annotation_list = annotation.value.list.as_list(capnp._ListSchema(annotations.AnnotationStruct))
assert annotation_list[0].test == 100
assert annotation_list[1].test == 101
annotation = annotations.TestAnnotationFour.schema.node.annotations[0]
annotation_list = annotation.value.list.as_list(capnp.ListSchema(capnp.types.UInt16))
annotation_list = annotation.value.list.as_list(capnp._ListSchema(capnp.types.UInt16))
assert annotation_list[0] == 200
assert annotation_list[1] == 201

14
test/test_serialization.py
View file

@ -181,20 +181,6 @@ def test_roundtrip_bytes_multiple_packed(all_types):
assert i == 3
@pytest.mark.skipif(
platform.python_implementation() == 'PyPy',
reason="This works on my local PyPy v2.5.0, but is for some reason broken on TravisCI. Skip for now."
)
def test_roundtrip_dict(all_types):
msg = all_types.TestAllTypes.new_message()
test_regression.init_all_types(msg)
d = msg.to_dict()
with _warnings(1, expected_text="This method is deprecated and will be removed"):
msg = all_types.TestAllTypes.from_dict(d)
test_regression.check_all_types(msg)
def test_file_and_bytes(all_types):
f = tempfile.TemporaryFile()
msg = all_types.TestAllTypes.new_message()