From 255119b838a13e75ad3610ea72c8bc357441fabe Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Wed, 3 Jun 2020 22:49:52 -0700 Subject: [PATCH] Updating to capnproto v0.8.0 - Includes some test stabilization - Fixes manylinux2010 build issues (linker flag order due to old gcc) - More rigorous python setup.py clean - Requires capnproto v0.8.0 or greater - Including system libcapnp include path for import (e.g. import stream_capnp) - Bundle libcapnp .capnp files when not using system libcapnp - Removing more distutils usage. Now using pkg-config to determine the system version of libcapnp (mainly for Linux, but should work on macOS with brew) - Removed dead code Resolves issues #215 #216 #217 Lots of fixes for Issue #218 (all sorts of retry methods needed for GitHub Actions) --- .github/workflows/manylinux2010.yml | 2 + .gitignore | 7 +- README.md | 14 +- buildutils/__init__.py | 4 - buildutils/build.py | 6 + buildutils/bundle.py | 13 +- buildutils/config.py | 21 -- buildutils/detect.py | 154 --------- buildutils/misc.py | 64 ---- buildutils/patch.py | 67 ---- buildutils/vers.cpp | 9 - capnp/c++.capnp | 26 -- capnp/helpers/checkCompiler.h | 2 +- capnp/lib/capnp.pyx | 11 + capnp/schema.capnp | 383 ---------------------- examples/async_calculator_client.py | 1 + examples/async_calculator_server.py | 1 + examples/async_client.py | 1 + examples/async_reconnecting_ssl_client.py | 1 + examples/async_server.py | 1 + examples/async_ssl_client.py | 1 + examples/async_ssl_server.py | 1 + examples/thread_client.py | 4 +- requirements.txt | 1 + setup.py | 42 ++- test/test_examples.py | 138 ++++++-- test/test_load.py | 6 + test/test_rpc_calculator.py | 59 +--- 28 files changed, 206 insertions(+), 834 deletions(-) delete mode 100644 buildutils/config.py delete mode 100644 buildutils/detect.py delete mode 100644 buildutils/misc.py delete mode 100644 buildutils/patch.py delete mode 100644 buildutils/vers.cpp delete mode 100644 capnp/c++.capnp delete mode 100644 capnp/schema.capnp diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 2cf5db9..c2811ba 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -24,6 +24,8 @@ jobs: /opt/python/${{ matrix.python-version }}/bin/python -m pip install -r requirements.txt /opt/python/${{ matrix.python-version }}/bin/python -m pip install auditwheel - name: Build pycapnp and install + env: + LDFLAGS: -Wl,--no-as-needed -lrt run: | /opt/python/${{ matrix.python-version }}/bin/python setup.py build - name: Packaging diff --git a/.gitignore b/.gitignore index ff5b24b..0d6b022 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ *.egg-info dist build +build32 +build64 eggs parts bin @@ -32,7 +34,7 @@ nosetests.xml .project .pydevproject -# IntelliJ +# IntelliJ .idea/ # Cpp files @@ -47,3 +49,6 @@ capnp/lib/capnp.h bundled/ example *.iml + +# capnp files +*.capnp diff --git a/README.md b/README.md index 7c19e5a..0be9ffd 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ - ninja (macOS + Linux) - Visual Studio 2017+ -* capnproto-0.7.0 +* capnproto-0.8.0 - Not necessary if using bundled capnproto 32-bit Linux requires that capnproto be compiled with `-fPIC`. This is usually set correctly unless you are compiling canproto yourself. This is also called `-DCMAKE_POSITION_INDEPENDENT_CODE=1` for cmake. @@ -55,6 +55,18 @@ To force bundled python: pip install --install-option "--force-bundled-libcapnp" . ``` +Slightly more prompt error messages using distutils rather than pip. + +```bash +python setup.py install --force-bundled-libcapnp +``` + +The bundling system isn't that smart so it might be necessary to clean up the bundled build when changing versions: + +```bash +python setup.py clean +``` + ## Python Versions diff --git a/buildutils/__init__.py b/buildutils/__init__.py index 65bb10d..33130e5 100644 --- a/buildutils/__init__.py +++ b/buildutils/__init__.py @@ -5,9 +5,5 @@ Largely adapted from h5py # flake8: noqa F401 F403 from .msg import * -from .config import * -from .detect import * from .bundle import * -from .misc import * -from .patch import * from .build import * diff --git a/buildutils/build.py b/buildutils/build.py index 49f580f..f5d421b 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -22,7 +22,9 @@ def build_libcapnp(bundle_dir, build_dir): os.mkdir(tmp_dir) cxxflags = os.environ.get('CXXFLAGS', None) + ldflags = os.environ.get('LDFLAGS', None) os.environ['CXXFLAGS'] = (cxxflags or '') + ' -O2 -DNDEBUG' + os.environ['LDFLAGS'] = (ldflags or '') # Enable ninja for compilation if available build_type = [] @@ -77,5 +79,9 @@ def build_libcapnp(bundle_dir, build_dir): del os.environ['CXXFLAGS'] else: os.environ['CXXFLAGS'] = cxxflags + if ldflags is None: + del os.environ['LDFLAGS'] + else: + os.environ['LDFLAGS'] = ldflags if returncode != 0: raise RuntimeError('capnproto compilation failed: {}'.format(returncode)) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 5b4d9f5..ff628b2 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -27,7 +27,7 @@ pjoin = os.path.join # -bundled_version = (0, 7, 0) +bundled_version = (0, 8, 0) libcapnp_name = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp_name @@ -92,14 +92,3 @@ def fetch_libcapnp(savedir, url=None): else: cpp_dir = os.path.join(with_version, 'c++') shutil.move(cpp_dir, dest) - - # XXX (HaaTa): There's a bug in capnproto-0.7.0 on CentOS 6 (or any glibc older than 2.16) - # This patches the issue - with fileinput.FileInput(os.path.join(dest, 'src', 'kj', 'table.c++'), inplace=True, backup='.bak') as f: - for line in f: - print( - line.replace( - '__APPLE__ || __BIONIC__', '__APPLE__ || __BIONIC__ || (defined(__GLIBC__) && (__GLIBC__ == 2) && (__GLIBC_MINOR__ < 16))' - ), - end='' - ) diff --git a/buildutils/config.py b/buildutils/config.py deleted file mode 100644 index b9e05c7..0000000 --- a/buildutils/config.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Config functions""" -# -# Copyright (C) PyZMQ Developers -# -# This file is part of pyzmq, copied and adapted from h5py. -# h5py source used under the New BSD license -# -# h5py: -# -# Distributed under the terms of the New BSD License. The full license is in -# the file COPYING.BSD, distributed as part of this software. -# - -# -# Utility functions (adapted from h5py: http://h5py.googlecode.com) -# - - -def v_str(v_tuple): - """turn (2,0,1) into '2.0.1'.""" - return ".".join(str(x) for x in v_tuple) diff --git a/buildutils/detect.py b/buildutils/detect.py deleted file mode 100644 index 59209c0..0000000 --- a/buildutils/detect.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Detect zmq version""" -# -# Copyright (C) PyZMQ Developers -# -# This file is part of pyzmq, copied and adapted from h5py. -# h5py source used under the New BSD license -# -# h5py: -# -# Distributed under the terms of the New BSD License. The full license is in -# the file COPYING.BSD, distributed as part of this software. -# -# -# Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq -# for original project. - -import shutil -import sys -import os -import logging -import platform -from distutils import ccompiler -from distutils.ccompiler import get_default_compiler -import tempfile - -from .misc import get_compiler, get_output_error -from .patch import patch_lib_paths - -pjoin = os.path.join - -# -# Utility functions (adapted from h5py: http://h5py.googlecode.com) -# - - -def test_compilation(cfile, compiler=None, **compiler_attrs): - """Test simple compilation with given settings""" - cc = get_compiler(compiler, **compiler_attrs) - - efile, _ = os.path.splitext(cfile) - - cpreargs = lpreargs = [] - if sys.platform == 'darwin': - # use appropriate arch for compiler - if platform.architecture()[0] == '32bit': - if platform.processor() == 'powerpc': - cpu = 'ppc' - else: - cpu = 'i386' - cpreargs = ['-arch', cpu] - lpreargs = ['-arch', cpu, '-undefined', 'dynamic_lookup'] - else: - # allow for missing UB arch, since it will still work: - lpreargs = ['-undefined', 'dynamic_lookup'] - if sys.platform == 'sunos5': - if platform.architecture()[0] == '32bit': - lpreargs = ['-m32'] - else: - lpreargs = ['-m64'] - extra_compile_args = compiler_attrs.get('extra_compile_args', []) - if os.name != 'nt': - extra_compile_args += ['--std=c++14'] - extra_link_args = compiler_attrs.get('extra_link_args', []) - if cc.compiler_type == 'msvc': - extra_link_args += ['/MANIFEST'] - - objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra_compile_args) - cc.link_executable(objs, efile, extra_preargs=lpreargs, extra_postargs=extra_link_args) - return efile - - -def detect_version(basedir, compiler=None, **compiler_attrs): - """Compile, link & execute a test program, in empty directory `basedir`. - - The C compiler will be updated with any keywords given via setattr. - - Parameters - ---------- - - basedir : path - The location where the test program will be compiled and run - compiler : str - The distutils compiler key (e.g. 'unix', 'msvc', or 'mingw32') - **compiler_attrs : dict - Any extra compiler attributes, which will be set via ``setattr(cc)``. - - Returns - ------- - - A dict of properties for zmq compilation, with the following two keys: - - vers : tuple - The ZMQ version as a tuple of ints, e.g. (2,2,0) - settings : dict - The compiler options used to compile the test function, e.g. `include_dirs`, - `library_dirs`, `libs`, etc. - """ - if compiler is None: - compiler = get_default_compiler() - cfile = pjoin(basedir, 'vers.cpp') - shutil.copy(pjoin(os.path.dirname(__file__), 'vers.cpp'), cfile) - - # check if we need to link against Realtime Extensions library - if sys.platform.startswith('linux'): - cc = ccompiler.new_compiler(compiler=compiler) - cc.output_dir = basedir - if not cc.has_function('timer_create'): - if 'libraries' not in compiler_attrs: - compiler_attrs['libraries'] = [] - compiler_attrs['libraries'].append('rt') - - cc = get_compiler(compiler=compiler, **compiler_attrs) - efile = test_compilation(cfile, compiler=cc) - patch_lib_paths(efile, cc.library_dirs) - - rc, so, se = get_output_error([efile]) - if rc: - msg = "Error running version detection script:\n%s\n%s" % (so, se) - logging.error(msg) - raise IOError(msg) - - handlers = {'vers': lambda val: tuple(int(v) for v in val.split('.'))} - - props = {} - for line in (x for x in so.split('\n') if x): - key, val = line.split(':') - props[key] = handlers[key](val) - - return props - - -def test_build(**compiler_attrs): - """do a test build of libcapnp""" - tmp_dir = tempfile.mkdtemp() - - # line() - # info("Configure: Autodetecting Cap'n Proto settings...") - # info(" Custom Cap'n Proto dir: %s" % prefix) - try: - detected = detect_version(tmp_dir, None, **compiler_attrs) - finally: - erase_dir(tmp_dir) - - # info(" Cap'n Proto version detected: %s" % v_str(detected['vers'])) - - return detected - - -def erase_dir(path): - """Erase directory""" - try: - shutil.rmtree(path) - except Exception: - pass diff --git a/buildutils/misc.py b/buildutils/misc.py deleted file mode 100644 index 473dc00..0000000 --- a/buildutils/misc.py +++ /dev/null @@ -1,64 +0,0 @@ -"""misc build utility functions""" - -# Copyright (c) PyZMQ Developers -# Distributed under the terms of the Modified BSD License. - -import os -import logging -from distutils import ccompiler -from distutils.sysconfig import customize_compiler -from pipes import quote -from subprocess import Popen, PIPE - -pjoin = os.path.join - - -def customize_mingw(cc): - """customize mingw""" - # strip -mno-cygwin from mingw32 (Python Issue #12641) - for cmd in [cc.compiler, cc.compiler_cxx, cc.compiler_so, cc.linker_exe, cc.linker_so]: - if '-mno-cygwin' in cmd: - cmd.remove('-mno-cygwin') - - # remove problematic msvcr90 - if 'msvcr90' in cc.dll_libraries: - cc.dll_libraries.remove('msvcr90') - - -def customize_msvc(cc): - pass - - -def get_compiler(compiler, **compiler_attrs): - """get and customize a compiler""" - if compiler is None or isinstance(compiler, str): - cc = ccompiler.new_compiler(compiler=compiler) - customize_compiler(cc) - if cc.compiler_type == 'mingw32': - customize_mingw(cc) - elif cc.compiler_type == 'msvc': - customize_msvc(cc) - else: - cc = compiler - - for name, val in compiler_attrs.items(): - setattr(cc, name, val) - - return cc - - -def get_output_error(cmd): - """Return the exit status, stdout, stderr of a command""" - if not isinstance(cmd, list): - cmd = [cmd] - logging.debug("Running: %s", ' '.join(map(quote, cmd))) - try: - result = Popen(cmd, stdout=PIPE, stderr=PIPE) - except IOError as e: - return -1, '', 'Failed to run %r: %r' % (cmd, e) - so, se = result.communicate() - # unicode: - so = so.decode('utf8', 'replace') - se = se.decode('utf8', 'replace') - - return result.returncode, so, se diff --git a/buildutils/patch.py b/buildutils/patch.py deleted file mode 100644 index 9676a92..0000000 --- a/buildutils/patch.py +++ /dev/null @@ -1,67 +0,0 @@ -"""utils for patching libraries""" - -# Copyright (c) PyZMQ Developers. -# Distributed under the terms of the Modified BSD License. - - -import re -import sys -import os -import logging - -from .misc import get_output_error - -pjoin = os.path.join - -# LIB_PAT from delocate -LIB_PAT = re.compile( - r"\s*(.*) \(compatibility version (\d+\.\d+\.\d+), current version (\d+\.\d+\.\d+)\)" -) - - -def _get_libs(fname): - rc, so, se = get_output_error(['otool', '-L', fname]) - if rc: - logging.error("otool -L %s failed: %r", fname, se) - return - for line in so.splitlines()[1:]: - m = LIB_PAT.match(line) - if m: - yield m.group(1) - - -def _find_library(lib, path): - """Find a library""" - for d in path[::-1]: - real_lib = os.path.join(d, lib) - if os.path.exists(real_lib): - return real_lib - return None - - -def _install_name_change(fname, lib, real_lib): - rc, so, se = get_output_error(['install_name_tool', '-change', lib, real_lib, fname]) - if rc: - logging.error("Couldn't update load path: %s", se) - - -def patch_lib_paths(fname, library_dirs): - """Load any weakly-defined libraries from their real location - - (only on OS X) - - - Find libraries with `otool -L` - - Update with `install_name_tool -change` - """ - if sys.platform != 'darwin': - return - - libs = _get_libs(fname) - for lib in libs: - if not lib.startswith(('@', '/')): - real_lib = _find_library(lib, library_dirs) - if real_lib: - _install_name_change(fname, lib, real_lib) - - -__all__ = ['patch_lib_paths'] diff --git a/buildutils/vers.cpp b/buildutils/vers.cpp deleted file mode 100644 index 00631a4..0000000 --- a/buildutils/vers.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// check libcapnp version - -#include -#include "capnp/common.h" - -int main(int argc, char **argv){ - fprintf(stdout, "vers: %d.%d.%d\n", CAPNP_VERSION_MAJOR, CAPNP_VERSION_MINOR, CAPNP_VERSION_MICRO); - return 0; -} diff --git a/capnp/c++.capnp b/capnp/c++.capnp deleted file mode 100644 index 2bda547..0000000 --- a/capnp/c++.capnp +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors -# Licensed under the MIT License: -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -@0xbdf87d7bb8304e81; -$namespace("capnp::annotations"); - -annotation namespace(file): Text; -annotation name(field, enumerant, struct, enum, interface, method, param, group, union): Text; diff --git a/capnp/helpers/checkCompiler.h b/capnp/helpers/checkCompiler.h index c5e34c1..e0e03c4 100644 --- a/capnp/helpers/checkCompiler.h +++ b/capnp/helpers/checkCompiler.h @@ -5,4 +5,4 @@ #include "capnp/dynamic.h" -static_assert(CAPNP_VERSION >= 7000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.7 and then re-install this python library"); \ No newline at end of file +static_assert(CAPNP_VERSION >= 8000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.8 and then re-install this python library"); diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index d7b0dfb..d5dc798 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -4070,6 +4070,17 @@ def add_import_hook(additional_paths=[]): if _importer is not None: remove_import_hook() + # Automatically include the system and built-in capnp paths + # Highest priority at position 0 + extra_paths = [ + _os.path.join(_os.path.dirname(__file__), '..'), # Built-in (only used if bundled) + '/usr/local/include/capnp', # Common macOS brew location + '/usr/include/capnp', # Common posix location + ] + for path in extra_paths: + if _os.path.isdir(path): + additional_paths.append(path) + _importer = _Importer(additional_paths) _sys.meta_path.append(_importer) diff --git a/capnp/schema.capnp b/capnp/schema.capnp deleted file mode 100644 index bb3532e..0000000 --- a/capnp/schema.capnp +++ /dev/null @@ -1,383 +0,0 @@ -# Copyright (c) 2013, Kenton Varda -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -using Cxx = import "c++.capnp"; - -@0xa93fc509624c72d9; -$Cxx.namespace("capnp::schema"); - -using Id = UInt64; -# The globally-unique ID of a file, type, or annotation. - -struct Node { - id @0 :Id; - - displayName @1 :Text; - # Name to present to humans to identify this Node. You should not attempt to parse this. Its - # format could change. It is not guaranteed to be unique. - # - # (On Zooko's triangle, this is the node's nickname.) - - displayNamePrefixLength @2 :UInt32; - # If you want a shorter version of `displayName` (just naming this node, without its surrounding - # scope), chop off this many characters from the beginning of `displayName`. - - scopeId @3 :Id; - # ID of the lexical parent node. Typically, the scope node will have a NestedNode pointing back - # at this node, but robust code should avoid relying on this (and, in fact, group nodes are not - # listed in the outer struct's nestedNodes, since they are listed in the fields). `scopeId` is - # zero if the node has no parent, which is normally only the case with files, but should be - # allowed for any kind of node (in order to make runtime type generation easier). - - nestedNodes @4 :List(NestedNode); - # List of nodes nested within this node, along with the names under which they were declared. - - struct NestedNode { - name @0 :Text; - # Unqualified symbol name. Unlike Node.name, this *can* be used programmatically. - # - # (On Zooko's triangle, this is the node's petname according to its parent scope.) - - id @1 :Id; - # ID of the nested node. Typically, the target node's scopeId points back to this node, but - # robust code should avoid relying on this. - } - - annotations @5 :List(Annotation); - # Annotations applied to this node. - - union { - # Info specific to each kind of node. - - file @6 :Void; - - struct :group { - dataWordCount @7 :UInt16; - # Size of the data section, in words. - - pointerCount @8 :UInt16; - # Size of the pointer section, in pointers (which are one word each). - - preferredListEncoding @9 :ElementSize; - # The preferred element size to use when encoding a list of this struct. If this is anything - # other than `inlineComposite` then the struct is one word or less in size and is a candidate - # for list packing optimization. - - isGroup @10 :Bool; - # If true, then this "struct" node is actually not an independent node, but merely represents - # some named union or group within a particular parent struct. This node's scopeId refers - # to the parent struct, which may itself be a union/group in yet another struct. - # - # All group nodes share the same dataWordCount and pointerCount as the top-level - # struct, and their fields live in the same ordinal and offset spaces as all other fields in - # the struct. - # - # Note that a named union is considered a special kind of group -- in fact, a named union - # is exactly equivalent to a group that contains nothing but an unnamed union. - - discriminantCount @11 :UInt16; - # Number of fields in this struct which are members of an anonymous union, and thus may - # overlap. If this is non-zero, then a 16-bit discriminant is present indicating which - # of the overlapping fields is active. This can never be 1 -- if it is non-zero, it must be - # two or more. - # - # Note that the fields of an unnamed union are considered fields of the scope containing the - # union -- an unnamed union is not its own group. So, a top-level struct may contain a - # non-zero discriminant count. Named unions, on the other hand, are equivalent to groups - # containing unnamed unions. So, a named union has its own independent schema node, with - # `isGroup` = true. - - discriminantOffset @12 :UInt32; - # If `discriminantCount` is non-zero, this is the offset of the union discriminant, in - # multiples of 16 bits. - - fields @13 :List(Field); - # Fields defined within this scope (either the struct's top-level fields, or the fields of - # a particular group; see `isGroup`). - # - # The fields are sorted by ordinal number, but note that because groups share the same - # ordinal space, the field's index in this list is not necessarily exactly its ordinal. - # On the other hand, the field's position in this list does remain the same even as the - # protocol evolves, since it is not possible to insert or remove an earlier ordinal. - # Therefore, for most use cases, if you want to identify a field by number, it may make the - # most sense to use the field's index in this list rather than its ordinal. - } - - enum :group { - enumerants@14 :List(Enumerant); - # Enumerants ordered by numeric value (ordinal). - } - - interface :group { - methods @15 :List(Method); - # Methods ordered by ordinal. - - extends @31 :List(Id); - # Superclasses of this interface. - } - - const :group { - type @16 :Type; - value @17 :Value; - } - - annotation :group { - type @18 :Type; - - targetsFile @19 :Bool; - targetsConst @20 :Bool; - targetsEnum @21 :Bool; - targetsEnumerant @22 :Bool; - targetsStruct @23 :Bool; - targetsField @24 :Bool; - targetsUnion @25 :Bool; - targetsGroup @26 :Bool; - targetsInterface @27 :Bool; - targetsMethod @28 :Bool; - targetsParam @29 :Bool; - targetsAnnotation @30 :Bool; - } - } -} - -struct Field { - # Schema for a field of a struct. - - name @0 :Text; - - codeOrder @1 :UInt16; - # Indicates where this member appeared in the code, relative to other members. - # Code ordering may have semantic relevance -- programmers tend to place related fields - # together. So, using code ordering makes sense in human-readable formats where ordering is - # otherwise irrelevant, like JSON. The values of codeOrder are tightly-packed, so the maximum - # value is count(members) - 1. Fields that are members of a union are only ordered relative to - # the other members of that union, so the maximum value there is count(union.members). - - annotations @2 :List(Annotation); - - const noDiscriminant :UInt16 = 0xffff; - - discriminantValue @3 :UInt16 = Field.noDiscriminant; - # If the field is in a union, this is the value which the union's discriminant should take when - # the field is active. If the field is not in a union, this is 0xffff. - - union { - slot :group { - # A regular, non-group, non-fixed-list field. - - offset @4 :UInt32; - # Offset, in units of the field's size, from the beginning of the section in which the field - # resides. E.g. for a UInt32 field, multiply this by 4 to get the byte offset from the - # beginning of the data section. - - type @5 :Type; - defaultValue @6 :Value; - - hadExplicitDefault @10 :Bool; - # Whether the default value was specified explicitly. Non-explicit default values are always - # zero or empty values. Usually, whether the default value was explicit shouldn't matter. - # The main use case for this flag is for structs representing method parameters: - # explicitly-defaulted parameters may be allowed to be omitted when calling the method. - } - - group :group { - # A group. - - typeId @7 :Id; - # The ID of the group's node. - } - } - - ordinal :union { - implicit @8 :Void; - explicit @9 :UInt16; - # The original ordinal number given to the field. You probably should NOT use this; if you need - # a numeric identifier for a field, use its position within the field array for its scope. - # The ordinal is given here mainly just so that the original schema text can be reproduced given - # the compiled version -- i.e. so that `capnp compile -ocapnp` can do its job. - } -} - -struct Enumerant { - # Schema for member of an enum. - - name @0 :Text; - - codeOrder @1 :UInt16; - # Specifies order in which the enumerants were declared in the code. - # Like Struct.Field.codeOrder. - - annotations @2 :List(Annotation); -} - -struct Method { - # Schema for method of an interface. - - name @0 :Text; - - codeOrder @1 :UInt16; - # Specifies order in which the methods were declared in the code. - # Like Struct.Field.codeOrder. - - paramStructType @2 :Id; - # ID of the parameter struct type. If a named parameter list was specified in the method - # declaration (rather than a single struct parameter type) then a corresponding struct type is - # auto-generated. Such an auto-generated type will not be listed in the interface's - # `nestedNodes` and its `scopeId` will be zero -- it is completely detached from the namespace. - - resultStructType @3 :Id; - # ID of the return struct type; similar to `paramStructType`. - - annotations @4 :List(Annotation); -} - -struct Type { - # Represents a type expression. - - union { - # The ordinals intentionally match those of Value. - - void @0 :Void; - bool @1 :Void; - int8 @2 :Void; - int16 @3 :Void; - int32 @4 :Void; - int64 @5 :Void; - uint8 @6 :Void; - uint16 @7 :Void; - uint32 @8 :Void; - uint64 @9 :Void; - float32 @10 :Void; - float64 @11 :Void; - text @12 :Void; - data @13 :Void; - - list :group { - elementType @14 :Type; - } - - enum :group { - typeId @15 :Id; - } - struct :group { - typeId @16 :Id; - } - interface :group { - typeId @17 :Id; - } - - anyPointer @18 :Void; - } -} - -struct Value { - # Represents a value, e.g. a field default value, constant value, or annotation value. - - union { - # The ordinals intentionally match those of Type. - - void @0 :Void; - bool @1 :Bool; - int8 @2 :Int8; - int16 @3 :Int16; - int32 @4 :Int32; - int64 @5 :Int64; - uint8 @6 :UInt8; - uint16 @7 :UInt16; - uint32 @8 :UInt32; - uint64 @9 :UInt64; - float32 @10 :Float32; - float64 @11 :Float64; - text @12 :Text; - data @13 :Data; - - list @14 :AnyPointer; - - enum @15 :UInt16; - struct @16 :AnyPointer; - - interface @17 :Void; - # The only interface value that can be represented statically is "null", whose methods always - # throw exceptions. - - anyPointer @18 :AnyPointer; - } -} - -struct Annotation { - # Describes an annotation applied to a declaration. Note AnnotationNode describes the - # annotation's declaration, while this describes a use of the annotation. - - id @0 :Id; - # ID of the annotation node. - - value @1 :Value; -} - -enum ElementSize { - # Possible element sizes for encoded lists. These correspond exactly to the possible values of - # the 3-bit element size component of a list pointer. - - empty @0; # aka "void", but that's a keyword. - bit @1; - byte @2; - twoBytes @3; - fourBytes @4; - eightBytes @5; - pointer @6; - inlineComposite @7; -} - -struct CodeGeneratorRequest { - nodes @0 :List(Node); - # All nodes parsed by the compiler, including for the files on the command line and their - # imports. - - requestedFiles @1 :List(RequestedFile); - # Files which were listed on the command line. - - struct RequestedFile { - id @0 :Id; - # ID of the file. - - filename @1 :Text; - # Name of the file as it appeared on the command-line (minus the src-prefix). You may use - # this to decide where to write the output. - - imports @2 :List(Import); - # List of all imported paths seen in this file. - - struct Import { - id @0 :Id; - # ID of the imported file. - - name @1 :Text; - # Name which *this* file used to refer to the foreign file. This may be a relative name. - # This information is provided because it might be useful for code generation, e.g. to - # generate #include directives in C++. We don't put this in Node.file because this - # information is only meaningful at compile time anyway. - # - # (On Zooko's triangle, this is the import's petname according to the importing file.) - } - } -} diff --git a/examples/async_calculator_client.py b/examples/async_calculator_client.py index 90f29f9..f7f1f85 100755 --- a/examples/async_calculator_client.py +++ b/examples/async_calculator_client.py @@ -55,6 +55,7 @@ async def main(host): print("Try IPv4") reader, writer = await asyncio.open_connection( addr, port, + family=socket.AF_INET ) except Exception: print("Try IPv6") diff --git a/examples/async_calculator_server.py b/examples/async_calculator_server.py index 0ecd944..08ef9d4 100755 --- a/examples/async_calculator_server.py +++ b/examples/async_calculator_server.py @@ -214,6 +214,7 @@ async def main(): server = await asyncio.start_server( new_connection, addr, port, + family=socket.AF_INET ) except Exception: print("Try IPv6") diff --git a/examples/async_client.py b/examples/async_client.py index 63c3e42..147c33c 100755 --- a/examples/async_client.py +++ b/examples/async_client.py @@ -56,6 +56,7 @@ async def main(host): print("Try IPv4") reader, writer = await asyncio.open_connection( addr, port, + family=socket.AF_INET ) except Exception: print("Try IPv6") diff --git a/examples/async_reconnecting_ssl_client.py b/examples/async_reconnecting_ssl_client.py index a4967bf..5c8f34b 100755 --- a/examples/async_reconnecting_ssl_client.py +++ b/examples/async_reconnecting_ssl_client.py @@ -86,6 +86,7 @@ async def main(host): reader, writer = await asyncio.open_connection( addr, port, ssl=ctx, + family=socket.AF_INET ) except OSError: print("Try IPv6") diff --git a/examples/async_server.py b/examples/async_server.py index acd4e1d..323c70e 100755 --- a/examples/async_server.py +++ b/examples/async_server.py @@ -116,6 +116,7 @@ async def main(): server = await asyncio.start_server( new_connection, addr, port, + family=socket.AF_INET ) except Exception: print("Try IPv6") diff --git a/examples/async_ssl_client.py b/examples/async_ssl_client.py index 802044e..ff5f011 100755 --- a/examples/async_ssl_client.py +++ b/examples/async_ssl_client.py @@ -65,6 +65,7 @@ async def main(host): reader, writer = await asyncio.open_connection( addr, port, ssl=ctx, + family=socket.AF_INET ) except Exception: print("Try IPv6") diff --git a/examples/async_ssl_server.py b/examples/async_ssl_server.py index 2b8a958..5292153 100755 --- a/examples/async_ssl_server.py +++ b/examples/async_ssl_server.py @@ -128,6 +128,7 @@ async def main(): new_connection, addr, port, ssl=ctx, + family=socket.AF_INET, ) except Exception: print("Try IPv6") diff --git a/examples/thread_client.py b/examples/thread_client.py index 2b23f2d..2d3d2ee 100755 --- a/examples/thread_client.py +++ b/examples/thread_client.py @@ -30,7 +30,7 @@ class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): def start_status_thread(host): - client = capnp.TwoPartyClient(host) + client = capnp.TwoPartyClient(host, nesting_limit=64) cap = client.bootstrap().cast_as(thread_capnp.Example) subscriber = StatusSubscriber() @@ -39,7 +39,7 @@ def start_status_thread(host): def main(host): - client = capnp.TwoPartyClient(host) + client = capnp.TwoPartyClient(host, nesting_limit=64) cap = client.bootstrap().cast_as(thread_capnp.Example) status_thread = threading.Thread(target=start_status_thread, args=(host,)) diff --git a/requirements.txt b/requirements.txt index 3fd46a1..593ffc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ jinja2 cython setuptools +pkgconfig pytest tox wheel diff --git a/setup.py b/setup.py index a6976de..82b0169 100644 --- a/setup.py +++ b/setup.py @@ -5,18 +5,19 @@ pycapnp distutils setup.py from __future__ import print_function +import glob import os +import shutil import struct import sys +import pkgconfig + from distutils.command.clean import clean as _clean -from distutils.errors import CompileError -from distutils.extension import Extension -from distutils.spawn import find_executable from setuptools import setup, find_packages, Extension -from buildutils import test_build, fetch_libcapnp, build_libcapnp, info +from buildutils import fetch_libcapnp, build_libcapnp, info _this_dir = os.path.dirname(__file__) @@ -69,12 +70,20 @@ class clean(_clean): ''' def run(self): _clean.run(self) - for x in [ 'capnp/lib/capnp.cpp', 'capnp/lib/capnp.h', 'capnp/version.py' ]: + for x in [ + os.path.join('capnp', 'lib', 'capnp.cpp'), + os.path.join('capnp', 'lib', 'capnp.h'), + os.path.join('capnp', 'version.py'), + 'build', + 'build32', + 'build64', + 'bundled' + ] + glob.glob(os.path.join('capnp', '*.capnp')): print('removing %s' % x) try: os.remove(x) except OSError: - pass + shutil.rmtree(x, ignore_errors=True) # hack to parse commandline arguments @@ -113,18 +122,20 @@ class build_libcapnp_ext(build_ext_c): need_build = False else: # Try to use capnp executable to find include and lib path - capnp_executable = find_executable("capnp") + capnp_executable = shutil.which("capnp") if capnp_executable: self.include_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'include')] self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib{}'.format(8 * struct.calcsize("P")))] self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib')] - # Try to autodetect presence of library. Requires compile/run - # step so only works for host (non-cross) compliation + # Look for capnproto using pkg-config (and minimum version) try: - test_build(include_dirs=self.include_dirs, library_dirs=self.library_dirs) - need_build = False - except CompileError: + if pkgconfig.installed('capnp', '>= 0.8.0'): + need_build = False + else: + need_build = True + except EnvironmentError: + # pkg-config not available in path need_build = True if need_build: @@ -157,6 +168,13 @@ class build_libcapnp_ext(build_ext_c): self.library_dirs += [os.path.join(build_dir, 'lib{}'.format(8 * struct.calcsize("P")))] self.library_dirs += [os.path.join(build_dir, 'lib')] + # Copy .capnp files from source + src_glob = glob.glob(os.path.join(build_dir, 'include', 'capnp', '*.capnp')) + dst_dir = os.path.join(self.build_lib, "capnp") + for file in src_glob: + info("copying {} -> {}".format(file, dst_dir)) + shutil.copy(file, dst_dir) + return build_ext_c.run(self) extra_compile_args = ['--std=c++14'] diff --git a/test/test_examples.py b/test/test_examples.py index 1167418..dfd711a 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -1,75 +1,147 @@ import os +import pytest import socket import subprocess import sys import time examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') +hostname = 'localhost' + + +processes = [] + +@pytest.fixture +def cleanup(): + yield + for p in processes: + p.kill() def run_subprocesses(address, server, client): - cmd = [sys.executable, os.path.join(examples_dir, server), address] - server = subprocess.Popen(cmd) - retries = 30 + server_attempt = 0 + server_attempts = 2 + done = False addr, port = address.split(':') - while True: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((addr, int(port))) - if result == 0: - break - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - result = sock.connect_ex((addr, int(port))) - if result == 0: - break - # Give the server some small amount of time to start listening - time.sleep(0.1) - retries -= 1 - if retries == 0: - assert False, "Timed out waiting for server to start" - cmd = [sys.executable, os.path.join(examples_dir, client), address] - client = subprocess.Popen(cmd) + while not done: + assert server_attempt < server_attempts, "Failed {} server attempts".format(server_attempts) + server_attempt += 1 - ret = client.wait(timeout=30) - server.kill() - assert ret == 0 + # Start server + cmd = [sys.executable, os.path.join(examples_dir, server), address] + serverp = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) + print("Server started (Attempt #{})".format(server_attempt)) + processes.append(serverp) + retries = 300 + # Loop until we have a socket connection to the server (with timeout) + while True: + try: + if 'unix' in address: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + result = sock.connect_ex(port) + if result == 0: + break + else: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + except socket.gaierror as err: + print("gaierror: {}".format(err)) + # Give the server some small amount of time to start listening + time.sleep(0.1) + retries -= 1 + if retries == 0: + serverp.kill() + print("Timed out waiting for server to start") + break + + if serverp.poll() is not None: + print("Server exited prematurely: {}".format(serverp.returncode)) + break + + # 3 tries per server try + client_attempt = 0 + client_attempts = 3 + while not done: + if client_attempt >= client_attempts: + print("Failed {} client attempts".format(client_attempts)) + break + client_attempt += 1 + + # Start client + cmd = [sys.executable, os.path.join(examples_dir, client), address] + clientp = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) + print("Client started (Attempt #{})".format(client_attempt)) + processes.append(clientp) + + retries = 30 * 10 + # Loop until the client is finished (with timeout) + while True: + if clientp.poll() == 0: + done = True + break + + if clientp.poll() is not None: + print("Client exited prematurely: {}".format(clientp.returncode)) + break + time.sleep(0.1) + retries -= 1 + if retries == 0: + print("Timed out waiting for client to finish") + clientp.kill() + break + + # Retrying with different address (ipv4) + if 'unix' not in addr: + addr = socket.gethostbyname(addr) + address = '{}:{}'.format(addr, port) + print("Forcing ipv4 -> {}".format(address)) + serverp.kill() + + serverp.kill() -def test_async_calculator_example(): - address = 'localhost:36432' +def test_async_calculator_example(cleanup): + address = '{}:36432'.format(hostname) server = 'async_calculator_server.py' client = 'async_calculator_client.py' run_subprocesses(address, server, client) -def test_thread_example(): - address = 'localhost:36433' +def test_thread_example(cleanup): + address = '{}:36433'.format(hostname) server = 'thread_server.py' client = 'thread_client.py' run_subprocesses(address, server, client) -def test_addressbook_example(): +def test_addressbook_example(cleanup): proc = subprocess.Popen([sys.executable, os.path.join(examples_dir, 'addressbook.py')]) ret = proc.wait() assert ret == 0 -def test_async_example(): - address = 'localhost:36434' +def test_async_example(cleanup): + address = '{}:36434'.format(hostname) server = 'async_server.py' client = 'async_client.py' run_subprocesses(address, server, client) -def test_ssl_async_example(): - address = 'localhost:36435' +def test_ssl_async_example(cleanup): + address = '{}:36435'.format(hostname) server = 'async_ssl_server.py' client = 'async_ssl_client.py' run_subprocesses(address, server, client) -def test_ssl_reconnecting_async_example(): - address = 'localhost:36436' +def test_ssl_reconnecting_async_example(cleanup): + address = '{}:36436'.format(hostname) server = 'async_ssl_server.py' client = 'async_reconnecting_ssl_client.py' run_subprocesses(address, server, client) diff --git a/test/test_load.py b/test/test_load.py index c2c780a..fba97fc 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -111,3 +111,9 @@ def test_remove_import_hook(): with pytest.raises(ImportError): import addressbook_capnp # noqa: F401 + + +def test_bundled_import_hook(): + # stream.capnp should be bundled, or provided by the system capnproto + capnp.add_import_hook() + import stream_capnp diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py index 89c0bf1..8f05b67 100644 --- a/test/test_rpc_calculator.py +++ b/test/test_rpc_calculator.py @@ -14,6 +14,17 @@ sys.path.append(examples_dir) import calculator_client # noqa: E402 import calculator_server # noqa: E402 +# Uses run_subprocesses function +import test_examples + +processes = [] + +@pytest.fixture +def cleanup(): + yield + for p in processes: + p.kill() + def test_calculator(): read, write = socket.socketpair() @@ -22,53 +33,13 @@ def test_calculator(): calculator_client.main(read) -def run_subprocesses(address): - cmd = [sys.executable, os.path.join(examples_dir, 'calculator_server.py'), address] - server = subprocess.Popen(cmd) - retries = 30 - if 'unix' in address: - addr = address.split(':')[1] - while True: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - result = sock.connect_ex(addr) - if result == 0: - break - # Give the server some small amount of time to start listening - time.sleep(0.1) - retries -= 1 - if retries == 0: - assert False, "Timed out waiting for server to start" - else: - addr, port = address.split(':') - while True: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((addr, int(port))) - if result == 0: - break - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - result = sock.connect_ex((addr, int(port))) - if result == 0: - break - # Give the server some small amount of time to start listening - time.sleep(0.1) - retries -= 1 - if retries == 0: - assert False, "Timed out waiting for server to start" - cmd = [sys.executable, os.path.join(examples_dir, 'calculator_client.py'), address] - client = subprocess.Popen(cmd) - - ret = client.wait() - server.kill() - assert ret == 0 - - -def test_calculator_tcp(): +def test_calculator_tcp(cleanup): address = 'localhost:36431' - run_subprocesses(address) + test_examples.run_subprocesses(address, 'calculator_server.py', 'calculator_client.py') @pytest.mark.skipif(os.name == 'nt', reason="socket.AF_UNIX not supported on Windows") -def test_calculator_unix(): +def test_calculator_unix(cleanup): path = '/tmp/pycapnp-test' try: os.unlink(path) @@ -76,7 +47,7 @@ def test_calculator_unix(): pass address = 'unix:' + path - run_subprocesses(address) + test_examples.run_subprocesses(address, 'calculator_server.py', 'calculator_client.py') def test_calculator_gc():