Replacing C++ with Rust

Expose rust merge and add intersection
Update cbindgen, uncomment more code
Scaffolding for error catching
Add rust bindings as a submodule
exception handling working
Implement abundances
Use __eq__ instead of __richcmp__
max hash should be an int
Avoid init_once too many times, remove third-party

.gitignore

@@ -17,4 +17,6 @@ _minhash.so
 .cache
 *.so
 .coverage
-sourmash_lib/_minhash.cpp
+.tox
+.eggs
+rust/target

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "rust"]
+	path = rust
+	url = https://github.com/luizirber/sourmash-rust

.travis.yml

@@ -37,6 +37,9 @@ matrix:
         - TOX_ENV=py36
 
 install:
+- curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
+- export PATH="$HOME/.cargo/bin:$PATH"
+- rustc -V
 - pip install tox
 - sudo apt-get --yes install snapd
 - sudo snap install ipfs

MANIFEST.in

@@ -1,7 +1,13 @@
 include LICENSE Makefile Dockerfile LICENSE Makefile README.md requirements.txt
 include index.ipynb
 recursive-include sourmash_lib *
-recursive-include third-party *.cc *.h
+recursive-include rust *.rs *.toml *.h
+prune rust/target/debug
+prune rust/target/release
+exclude rust/.git
+prune .eggs
+global-exclude *.rlib
 global-exclude *.orig
 global-exclude *.pyc
 global-exclude *.so
+global-exclude *.git/

setup.py

@@ -1,17 +1,25 @@
 from __future__ import print_function
-import sys
 from setuptools import setup
-from setuptools import Extension
 import os
 
+
+def build_native(spec):
+    build = spec.add_external_build(
+        cmd=['cargo', 'build', '--release'],
+        path='./rust'
+    )
+
+    spec.add_cffi_module(
+        module_path='sourmash_lib._lowlevel',
+        dylib=lambda: build.find_dylib('sourmash', in_path='target/release'),
+        header_filename=lambda: build.find_header('sourmash.h', in_path='target')
+    )
+
 # retrieve VERSION from sourmash_lib/VERSION.
 thisdir = os.path.dirname(__file__)
 version_file = open(os.path.join(thisdir, 'sourmash_lib', 'VERSION'))
 VERSION = version_file.read().strip()
 
-EXTRA_COMPILE_ARGS = ['-std=c++11', '-pedantic']
-EXTRA_LINK_ARGS=[]
-
 CLASSIFIERS = [
     "Environment :: Console",
     "Environment :: MacOS X",
@@ -29,19 +37,6 @@
 
 CLASSIFIERS.append("Development Status :: 5 - Production/Stable")
 
-if sys.platform == 'darwin':              # Mac OS X?
-    # force 64bit only builds
-    EXTRA_COMPILE_ARGS.extend(['-arch', 'x86_64', '-mmacosx-version-min=10.7',
-                               '-stdlib=libc++'])
-
-else:                                     # ...likely Linux
-   if os.environ.get('SOURMASH_COVERAGE'):
-      print('Turning on coverage analysis.')
-      EXTRA_COMPILE_ARGS.extend(['-g', '--coverage', '-lgcov'])
-      EXTRA_LINK_ARGS.extend(['--coverage', '-lgcov'])
-   else:
-      EXTRA_COMPILE_ARGS.append('-O3')
-
 SETUP_METADATA = \
                {
     "name": "sourmash",
@@ -52,30 +47,21 @@
     "author_email": "[email protected]",
     "license": "BSD 3-clause",
     "packages": ["sourmash_lib"],
+    "zip_safe": False,
+    "platforms": 'any',
     "entry_points": {'console_scripts': [
         'sourmash = sourmash_lib.__main__:main'
         ]
     },
-    "ext_modules": [Extension("sourmash_lib._minhash",
-                               sources=["sourmash_lib/_minhash.pyx",
-                                        "third-party/smhasher/MurmurHash3.cc"],
-                               depends=["sourmash_lib/kmer_min_hash.hh"],
-                               include_dirs=["./sourmash_lib",
-                                             "./third-party/smhasher/"],
-                               language="c++",
-                               extra_compile_args=EXTRA_COMPILE_ARGS,
-                               extra_link_args=EXTRA_LINK_ARGS)],
     "install_requires": ["screed>=0.9", "ijson", "khmer>=2.1<3.0"],
-    "setup_requires": ['Cython>=0.25.2', "setuptools>=18.0"],
+    "setup_requires": ["setuptools>=18.0", 'milksnake', 'cffi'],
     "extras_require": {
         'test' : ['pytest', 'pytest-cov', 'numpy', 'matplotlib', 'scipy'],
         'demo' : ['jupyter', 'jupyter_client', 'ipython'],
         'doc' : ['sphinx'],
         },
     "include_package_data": True,
-    "package_data": {
-        "sourmash_lib": ['*.pxd']
-    },
+    'milksnake_tasks': [build_native],
     "classifiers": CLASSIFIERS
     }
 

sourmash_lib/__init__.py

@@ -5,7 +5,10 @@
 from __future__ import print_function
 import os
 
-from ._minhash import (MinHash, get_minhash_default_seed, get_minhash_max_hash)
+from sourmash_lib._lowlevel import ffi, lib
+ffi.init_once(lib.sourmash_init, 'init')
+
+from .minhash import (MinHash, get_minhash_default_seed, get_minhash_max_hash)
 from .signature import (load_signatures, load_one_signature, SourmashSignature,
                         save_signatures)
 from .sbtmh import load_sbt_index, search_sbt_index, create_sbt_index

sourmash_lib/_compat.py

@@ -0,0 +1,24 @@
+import sys
+
+
+PY2 = sys.version_info[0] == 2
+
+if PY2:
+    text_type = unicode
+    int_types = (int, long)
+    string_types = (str, unicode)
+    range_type = xrange
+    itervalues = lambda x: x.itervalues()
+    NUL = '\x00'
+    def implements_to_string(cls):
+        cls.__unicode__ = cls.__str__
+        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
+        return cls
+else:
+    text_type = str
+    int_types = (int,)
+    string_types = (str,)
+    range_type = range
+    itervalues = lambda x: x.values()
+    NUL = 0
+    implements_to_string = lambda x: x

sourmash_lib/exceptions.py

@@ -0,0 +1,44 @@
+from sourmash_lib._compat import implements_to_string
+from sourmash_lib._lowlevel import lib
+
+
+__all__ = ['SourmashError']
+exceptions_by_code = {}
+
+
+@implements_to_string
+class SourmashError(Exception):
+    code = None
+
+    def __init__(self, msg):
+        Exception.__init__(self)
+        self.message = msg
+        self.rust_info = None
+
+    def __str__(self):
+        rv = self.message
+        if self.rust_info is not None:
+            return u'%s\n\n%s' % (rv, self.rust_info)
+        return rv
+
+
+def _make_exceptions():
+    for attr in dir(lib):
+        if not attr.startswith('SOURMASH_ERROR_CODE_'):
+            continue
+
+        class Exc(SourmashError):
+            pass
+
+        code = getattr(lib, attr)
+        if code < 100 or code > 10000:
+            Exc.__name__ = attr[20:].title().replace('_', '')
+            Exc.code = getattr(lib, attr)
+            globals()[Exc.__name__] = Exc
+            Exc.code = code
+            exceptions_by_code[code] = Exc
+            __all__.append(Exc.__name__)
+        else:
+            exceptions_by_code[code] = ValueError
+
+_make_exceptions()