Indexing in Rust (#773)

* working on keeping Index trait close to Index abc add coverage CI job initial index cmd impl add batch_insert method use needletail for fastx parsing fix finch feature lint clippy fix benchmarks comment out code that depends on unreleased crates... another ci check, can we publish it\? add windows, macos, beta and nightly tests tests move ocf to another repo needletail 0.3.2 update for finch 0.3.0 fix warnings initial search test oracle tests for search install sourmash for oracle tests keep nodegraph working on wasm32 use only hash_functions internally niffler instead of ocf test all features for coverage compatibility with sourmash-py SBT note about md5sum and filenames move tests into specific SBTs remove unwrap add note to nodegraph parsing update typed-builder fix nodegraph behavior to match khmer ignore roundtrip_sbt while figuring out heisenbug
sourmash-bio · Dec 16, 2019 · 93b9b90 · 93b9b90
1 parent ec8a00c
commit 93b9b90
Show file tree

Hide file tree

Showing 41 changed files with 2,118 additions and 1,222 deletions.
diff --git a/.github/actions-rs/grcov.yml b/.github/actions-rs/grcov.yml
@@ -0,0 +1,7 @@
+branch: true
+ignore-not-existing: true
+llvm: true
+filter: covered
+output-type: lcov
+output-file: ./lcov.info
+prefix-dir: /home/user/build/
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -0,0 +1,231 @@
+name: Rust checks
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v1
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Run cargo check
+        uses: actions-rs/cargo@v1
+        with:
+          command: check
+
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [beta, stable, windows, macos]
+        include:
+          - build: macos
+            os: macos-latest
+            rust: stable
+          - build: windows
+            os: windows-latest
+            rust: stable
+          - build: beta
+            os: ubuntu-latest
+            rust: beta
+          - build: stable
+            os: ubuntu-latest
+            rust: stable
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ matrix.rust }}
+          override: true
+
+      - name: Set up Python 3.8
+        if: matrix.os != 'windows-latest'
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.8"
+
+      - name: Install dependencies
+        if: matrix.os != 'windows-latest'
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+
+      - name: Run tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --no-fail-fast
+
+  test_all_features:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.8"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+
+      - name: Run tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --no-fail-fast --all --all-features
+
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.8"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+
+      - name: Run tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --no-fail-fast --all --all-features
+        env:
+          'CARGO_INCREMENTAL': '0'
+          'RUSTFLAGS': '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads'
+
+      - name: Collect coverage and generate report with grcov
+        uses: actions-rs/[email protected]
+        id: coverage
+
+      - name: Upload coverage to codecov
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ${{ steps.coverage.outputs.report }}
+
+  lints:
+    name: Lints
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v1
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: rustfmt, clippy
+
+      - name: Run cargo fmt
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+      - name: Run cargo clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: -- -D warnings
+
+  wasm-pack:
+    name: Check if wasm-pack builds a valid package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-unknown-unknown
+      - uses: actions-rs/cargo@v1
+        with:
+          command: install
+          args: --force wasm-pack
+      - name: run wasm-pack
+        run: wasm-pack build
+
+  wasm32-wasi:
+    name: Run tests under wasm32-wasi
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Install wasm32-wasi target
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-wasi
+      - name: Install wasmtime
+        run: "curl https://wasmtime.dev/install.sh -sSf | bash"
+      - name: Add wasmtime to PATH
+        run: echo "::add-path::$HOME/.wasmtime/bin"
+      - name: Install cargo-wasi command
+        uses: actions-rs/cargo@v1
+        with:
+          command: install
+          args: --force cargo-wasi
+      - name: Build code with cargo-wasi
+        uses: actions-rs/cargo@v1
+        with:
+          command: wasi
+          args: build
+      - name: Run tests under wasm32-wasi
+        uses: actions-rs/cargo@v1
+        continue-on-error: true  ## TODO: remove this when tests work...
+        with:
+          command: wasi
+          args: test
+
+  publish:
+    name: Publish (dry-run)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v1
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Make sure we can publish the crate
+        uses: actions-rs/cargo@v1
+        with:
+          command: publish
+          args: --dry-run
diff --git a/.gitignore b/.gitignore
@@ -24,4 +24,12 @@ sourmash/_minhash.cpp
 .pytest_cache
 .python-version
 sourmash/version.py
-*.DS_Store
+*.DS_Store
+.tox
+sourmash/_lowlevel*.py
+.env
+Pipfile
+Pipfile.lock
+ocf/target/
+target/
+Cargo.lock
diff --git a/.travis.yml b/.travis.yml
@@ -48,26 +48,6 @@ jobs:
       python: 3.6
     - <<: *test
       python: 3.5
-    - <<: *test
-      name: wasm-pack
-      language: rust
-      rust: stable
-      before_script: skip
-      install: skip
-      script:
-        - rustup target add wasm32-unknown-unknown
-        - cargo install --force wasm-pack
-        - wasm-pack build
-    - <<: *test
-      name: wasi target
-      language: rust
-      rust: stable
-      before_script: skip
-      install: skip
-      script:
-        - rustup target add wasm32-wasi
-        - cargo install --force cargo-wasi
-        - cargo wasi build
 
     - &wheel
       stage: build wheel and send to github releases

diff --git a/Cargo.toml b/Cargo.toml
@@ -8,6 +8,9 @@ keywords = ["minhash", "bioinformatics"]
 categories = ["science", "algorithms", "data-structures"]
 license = "BSD-3-Clause"
 edition = "2018"
+default-run = "smrs"
+autoexamples = false
+autobins = false
 
 [lib]
 name = "sourmash"
@@ -18,61 +21,67 @@ bench = false
 lto=true
 
 [features]
-from-finch = ["finch", "needletail"]
+from-finch = ["finch"]
 
 [workspace]
 
 #[build-dependencies]
 #cbindgen = "~0.6.7"
 
 [dependencies]
-byteorder = "^1.2"
-cfg-if = "0.1"
-clap = { version = "~2.32", features = ["yaml"] }
-env_logger = "0.6.0"
+byteorder = "1.3.2"
+cfg-if = "0.1.10"
+clap = { version = "2.33.0", features = ["yaml"] }
+env_logger = "0.7.1"
 exitfailure = "0.5.1"
-failure = "0.1.3"
-failure_derive = "0.1.3"
-finch = { version = "~0.1.6", optional = true }
-fixedbitset = "^0.1.9"
-human-panic = "1.0.1"
-lazy_static = "1.0.0"
+failure = "0.1.6"
+failure_derive = "0.1.6"
+finch = { version = "0.3.0", optional = true }
+fixedbitset = "0.2.0"
+lazy_static = "1.4.0"
 lazy-init = "0.3.0"
-log = "0.4.0"
-md5 = "0.6.0"
-murmurhash3 = "~0.0.5"
-needletail = { version = "~0.2.1", optional = true }
-serde = "1.0"
-serde_derive = "~1.0.58"
-serde_json = "1.0.2"
-ukhs = { git = "https://github.com/luizirber/ukhs", branch = "feature/alternative_backends", features = ["boomphf_mphf"], default-features = false}
-bio = { git = "https://github.com/luizirber/rust-bio", branch = "feature/fastx_reader" }
+log = "0.4.8"
+md5 = "0.7.0"
+murmurhash3 = "0.0.5"
+serde = "1.0.103"
+serde_derive = "1.0.103"
+serde_json = "1.0.44"
+#ukhs = { git = "https://github.com/luizirber/ukhs", branch = "feature/alternative_backends", features = ["boomphf_mphf"], default-features = false}
 primal = "0.2.3"
-pdatastructs = { git = "https://github.com/luizirber/pdatastructs.rs", branch = "succinct_wasm" }
-itertools = "0.8.0"
-typed-builder = "0.3.0"
-csv = "1.0.7"
+#pdatastructs = { git = "https://github.com/luizirber/pdatastructs.rs", branch = "succinct_wasm" }
+itertools = "0.8.2"
+typed-builder = "0.4.0"
+csv = "1.1.1"
 tempfile = "3.1.0"
 
+[dependencies.needletail]
+version = "0.3.2"
+default-features = false
+#features = ["compression"]
+
 [target.'cfg(all(target_arch = "wasm32", target_vendor="unknown"))'.dependencies.wasm-bindgen]
-version = "^0.2"
+version = "0.2.55"
 features = ["serde-serialize"]
 
-[target.'cfg(not(all(target_arch = "wasm32", target_vendor="unknown")))'.dependencies.ocf]
-version = "0.1"
-path = "ocf"
+[target.'cfg(not(all(target_arch = "wasm32", target_vendor="unknown")))'.dependencies.niffler]
+version = "1.0"
 default-features = false
 
-[target.'cfg(not(target_arch = "wasm32"))'.dependencies.mqf]
-version = "1.0.0"
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+proptest = "0.9.4"
 
 [dev-dependencies]
-proptest = "^0.8"
-criterion = "^0.2"
-rand = "^0.5"
-tempfile = "3"
-assert_matches = "1.2"
+criterion = "0.3.0"
+rand = "0.7.2"
+tempfile = "3.1.0"
+assert_matches = "1.3.0"
+assert_cmd = "0.12.0"
+predicates = "1.0.2"
 
 [[bench]]
 name = "index"
 harness = false
+
+[[bin]]
+name = "smrs"
+path = "src/bin/smrs.rs"