0.5.0

.codecov.yml

@@ -5,15 +5,9 @@ ignore:
   - "**/integration/"
   - "**/examples/"
   - "**/benches/"
-  - "src/tests.rs"
   - "src/error.rs"
-  - "src/swmr/generic/tests.rs"
-  - "src/swmr/generic/tests/"
-  - "src/swmr/wal/tests.rs"
-  - "src/swmr/wal/tests/"
-  - "src/wal/type/"
-  - "src/unsync/tests.rs"
-  - "src/unsync/tests/"
+  - "src/swmr/tests.rs"
+  - "src/swmr/tests/" 
 
 coverage:
   status:

.github/workflows/ci.yml

@@ -183,12 +183,12 @@ jobs:
         path: ~/.cargo
         key: ${{ runner.os }}-coverage-dotcargo
     - name: Run test (Unix)
-      run: RUSTFLAGS="--cfg all_tests" cargo test --all-features
+      run: RUSTFLAGS="--cfg all_orderwal_tests" cargo test --all-features
       if: matrix.os != 'windows-latest'
     - name: Run test (Windows)
       shell: pwsh
       run: |
-        $env:RUSTFLAGS="--cfg all_tests"
+        $env:RUSTFLAGS="--cfg all_orderwal_tests"
         cargo test --all-features
       if: matrix.os == 'windows-latest'
 
@@ -238,18 +238,14 @@ jobs:
           - x86_64-apple-darwin
           - aarch64-apple-darwin
         cfg:
-          - unsync_insert
-          - unsync_iters
-          - unsync_get
-          - unsync_constructor
           - swmr_insert
           - swmr_iters
           - swmr_get
           - swmr_constructor
-          - swmr_generic_insert
-          - swmr_generic_iters
-          - swmr_generic_get
-          - swmr_generic_constructor
+          - swmr_multiple_version_insert
+          - swmr_multiple_version_iters
+          - swmr_multiple_version_get
+          - swmr_multiple_version_constructor
         # Exclude invalid combinations
         exclude:
           - os: ubuntu-latest
@@ -303,10 +299,10 @@ jobs:
   #         - swmr_iters
   #         - swmr_get
   #         - swmr_constructor
-  #         - swmr_generic_insert
-  #         - swmr_generic_iters
-  #         - swmr_generic_get
-  #         - swmr_generic_constructor
+  #         - swmr_insert
+  #         - swmr_iters
+  #         - swmr_get
+  #         - swmr_constructor
   #       # Exclude invalid combinations
   #       exclude:
   #         - os: ubuntu-latest
@@ -433,7 +429,7 @@ jobs:
       - name: Run tarpaulin
         uses: actions-rs/cargo@v1
         env:
-          RUSTFLAGS: "--cfg all_tests"
+          RUSTFLAGS: "--cfg all_orderwal_tests"
         with:
           command: tarpaulin
           args: --all-features --run-types tests --run-types doctests --workspace --out xml

CHANGELOG.md

@@ -1,10 +1,16 @@
 # Rleases
 
+## 0.5.0 (Oct 27th, 2024)
+
+- Refactor the project to make all of the WALs based on the generic implementation.
+- Support different memtables based on [`crossbeam-skiplist`](https://github.com/crossbeam-rs/crossbeam) or [`skl`](https://github.com/al8n/skl)
+- More user-friendly APIs
+
 ## 0.4.0 (Sep 30th, 2024)
 
 FEATURES
 
-- Support `K: ?Sized` and `V: ?Sized` for `GenericOrderWal`.
+- Support `K: ?Sized` and `V: ?Sized` for `OrderWal`.
 - Use `flush_header_and_range` instead of `flush_range` when insertion.
 
 ## 0.1.0 (Sep 14th, 2024)

Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "orderwal"
-version = "0.4.1"
+version = "0.5.0"
 edition = "2021"
 repository = "https://github.com/al8n/orderwal"
 homepage = "https://github.com/al8n/orderwal"
 documentation = "https://docs.rs/orderwal"
 description = "A generic-purpose, atomic, ordered, zero-copy, Write-Ahead Log implementation for Rust."
 license = "MIT OR Apache-2.0"
-rust-version = "1.80"
+rust-version = "1.81"
 categories = ["filesystem", "database-implementations", "development-tools", "data-structures"]
 keywords = ["wal", "write-ahead-log", "append-only", "append-only-log", "bitcask"]
 
@@ -18,7 +18,7 @@ harness = false
 
 [features]
 default = ["std"]
-std = ["rarena-allocator/default", "crossbeam-skiplist/default", "bitflags/std", "dbutils/default", "among/default", "faststr?/default", "bytes?/default", "smol_str?/default"]
+std = ["rarena-allocator/default", "crossbeam-skiplist/default", "bitflags/std", "dbutils/default", "among/default", "faststr?/default", "bytes?/default", "smol_str?/default", "skl/memmap"]
 
 xxhash3 = ["dbutils/xxhash3", "std"]
 xxhash64 = ["dbutils/xxhash64", "std"]
@@ -28,12 +28,14 @@ tracing = ["dep:tracing", "dbutils/tracing"]
 [dependencies]
 among = { version = "0.1", default-features = false, features = ["either"] }
 bitflags = { version = "2", default-features = false }
-dbutils = { version = "0.6", default-features = false, features = ["crc32fast"] }
+dbutils = { version = "0.8", default-features = false, features = ["crc32fast"] }
+derive-where = "1"
 ref-cast = "1"
 rarena-allocator = { version = "0.4", default-features = false, features = ["memmap"] }
 crossbeam-skiplist = { version = "0.1", default-features = false, package = "crossbeam-skiplist-pr1132" }
+crossbeam-skiplist-mvcc = "0.1"
+skl = { version = "0.18", default-features = false, features = ["alloc"] }
 paste = "1"
-thiserror = "1"
 
 bytes = { version = "1", default-features = false, optional = true }
 smallvec = { version = "1", default-features = false, optional = true, features = ["const_generics"] }
@@ -68,17 +70,13 @@ rustdoc-args = ["--cfg", "docsrs"]
 rust_2018_idioms = "warn"
 single_use_lifetimes = "warn"
 unexpected_cfgs = { level = "warn", check-cfg = [
-  'cfg(all_tests)',
-  'cfg(test_unsync_constructor)',
-  'cfg(test_unsync_insert)',
-  'cfg(test_unsync_iters)',
-  'cfg(test_unsync_get)',
+  'cfg(all_orderwal_tests)',
   'cfg(test_swmr_constructor)',
   'cfg(test_swmr_insert)',
   'cfg(test_swmr_iters)',
   'cfg(test_swmr_get)',
-  'cfg(test_swmr_generic_constructor)',
-  'cfg(test_swmr_generic_insert)',
-  'cfg(test_swmr_generic_iters)',
-  'cfg(test_swmr_generic_get)',
+  'cfg(test_swmr_multiple_version_constructor)',
+  'cfg(test_swmr_multiple_version_insert)',
+  'cfg(test_swmr_multiple_version_iters)',
+  'cfg(test_swmr_multiple_version_get)',
 ] }

README-zh_CN.md

@@ -3,7 +3,7 @@
 </div>
 <div align="center">
 
-An ordered, zero-copy, Write-Ahead Log implementation for Rust.
+A generic-purpose, atomic, ordered, zero-copy, Write-Ahead Log implementation for Rust.
 
 [<img alt="github" src="https://img.shields.io/badge/github-al8n/orderwal-8da0cb?style=for-the-badge&logo=Github" height="22">][Github-url]
 <img alt="LoC" src="https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2Fal8n%2F327b2a8aef9003246e45c6e47fe63937%2Fraw%2Forderwal" height="22">
@@ -15,17 +15,34 @@ An ordered, zero-copy, Write-Ahead Log implementation for Rust.
 [<img alt="crates.io" src="https://img.shields.io/crates/d/orderwal?color=critical&logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBzdGFuZGFsb25lPSJubyI/PjwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+PHN2ZyB0PSIxNjQ1MTE3MzMyOTU5IiBjbGFzcz0iaWNvbiIgdmlld0JveD0iMCAwIDEwMjQgMTAyNCIgdmVyc2lvbj0iMS4xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHAtaWQ9IjM0MjEiIGRhdGEtc3BtLWFuY2hvci1pZD0iYTMxM3guNzc4MTA2OS4wLmkzIiB3aWR0aD0iNDgiIGhlaWdodD0iNDgiIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIj48ZGVmcz48c3R5bGUgdHlwZT0idGV4dC9jc3MiPjwvc3R5bGU+PC9kZWZzPjxwYXRoIGQ9Ik00NjkuMzEyIDU3MC4yNHYtMjU2aDg1LjM3NnYyNTZoMTI4TDUxMiA3NTYuMjg4IDM0MS4zMTIgNTcwLjI0aDEyOHpNMTAyNCA2NDAuMTI4QzEwMjQgNzgyLjkxMiA5MTkuODcyIDg5NiA3ODcuNjQ4IDg5NmgtNTEyQzEyMy45MDQgODk2IDAgNzYxLjYgMCA1OTcuNTA0IDAgNDUxLjk2OCA5NC42NTYgMzMxLjUyIDIyNi40MzIgMzAyLjk3NiAyODQuMTYgMTk1LjQ1NiAzOTEuODA4IDEyOCA1MTIgMTI4YzE1Mi4zMiAwIDI4Mi4xMTIgMTA4LjQxNiAzMjMuMzkyIDI2MS4xMkM5NDEuODg4IDQxMy40NCAxMDI0IDUxOS4wNCAxMDI0IDY0MC4xOTJ6IG0tMjU5LjItMjA1LjMxMmMtMjQuNDQ4LTEyOS4wMjQtMTI4Ljg5Ni0yMjIuNzItMjUyLjgtMjIyLjcyLTk3LjI4IDAtMTgzLjA0IDU3LjM0NC0yMjQuNjQgMTQ3LjQ1NmwtOS4yOCAyMC4yMjQtMjAuOTI4IDIuOTQ0Yy0xMDMuMzYgMTQuNC0xNzguMzY4IDEwNC4zMi0xNzguMzY4IDIxNC43MiAwIDExNy45NTIgODguODMyIDIxNC40IDE5Ni45MjggMjE0LjRoNTEyYzg4LjMyIDAgMTU3LjUwNC03NS4xMzYgMTU3LjUwNC0xNzEuNzEyIDAtODguMDY0LTY1LjkyLTE2NC45MjgtMTQ0Ljk2LTE3MS43NzZsLTI5LjUwNC0yLjU2LTUuODg4LTMwLjk3NnoiIGZpbGw9IiNmZmZmZmYiIHAtaWQ9IjM0MjIiIGRhdGEtc3BtLWFuY2hvci1pZD0iYTMxM3guNzc4MTA2OS4wLmkwIiBjbGFzcz0iIj48L3BhdGg+PC9zdmc+&style=for-the-badge" height="22">][crates-url]
 <img alt="license" src="https://img.shields.io/badge/License-Apache%202.0/MIT-blue.svg?style=for-the-badge&fontColor=white&logoColor=f5c076&logo=data:image/svg+xml;base64,PCFET0NUWVBFIHN2ZyBQVUJMSUMgIi0vL1czQy8vRFREIFNWRyAxLjEvL0VOIiAiaHR0cDovL3d3dy53My5vcmcvR3JhcGhpY3MvU1ZHLzEuMS9EVEQvc3ZnMTEuZHRkIj4KDTwhLS0gVXBsb2FkZWQgdG86IFNWRyBSZXBvLCB3d3cuc3ZncmVwby5jb20sIFRyYW5zZm9ybWVkIGJ5OiBTVkcgUmVwbyBNaXhlciBUb29scyAtLT4KPHN2ZyBmaWxsPSIjZmZmZmZmIiBoZWlnaHQ9IjgwMHB4IiB3aWR0aD0iODAwcHgiIHZlcnNpb249IjEuMSIgaWQ9IkNhcGFfMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgdmlld0JveD0iMCAwIDI3Ni43MTUgMjc2LjcxNSIgeG1sOnNwYWNlPSJwcmVzZXJ2ZSIgc3Ryb2tlPSIjZmZmZmZmIj4KDTxnIGlkPSJTVkdSZXBvX2JnQ2FycmllciIgc3Ryb2tlLXdpZHRoPSIwIi8+Cg08ZyBpZD0iU1ZHUmVwb190cmFjZXJDYXJyaWVyIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KDTxnIGlkPSJTVkdSZXBvX2ljb25DYXJyaWVyIj4gPGc+IDxwYXRoIGQ9Ik0xMzguMzU3LDBDNjIuMDY2LDAsMCw2Mi4wNjYsMCwxMzguMzU3czYyLjA2NiwxMzguMzU3LDEzOC4zNTcsMTM4LjM1N3MxMzguMzU3LTYyLjA2NiwxMzguMzU3LTEzOC4zNTcgUzIxNC42NDgsMCwxMzguMzU3LDB6IE0xMzguMzU3LDI1OC43MTVDNzEuOTkyLDI1OC43MTUsMTgsMjA0LjcyMywxOCwxMzguMzU3UzcxLjk5MiwxOCwxMzguMzU3LDE4IHMxMjAuMzU3LDUzLjk5MiwxMjAuMzU3LDEyMC4zNTdTMjA0LjcyMywyNTguNzE1LDEzOC4zNTcsMjU4LjcxNXoiLz4gPHBhdGggZD0iTTE5NC43OTgsMTYwLjkwM2MtNC4xODgtMi42NzctOS43NTMtMS40NTQtMTIuNDMyLDIuNzMyYy04LjY5NCwxMy41OTMtMjMuNTAzLDIxLjcwOC0zOS42MTQsMjEuNzA4IGMtMjUuOTA4LDAtNDYuOTg1LTIxLjA3OC00Ni45ODUtNDYuOTg2czIxLjA3Ny00Ni45ODYsNDYuOTg1LTQ2Ljk4NmMxNS42MzMsMCwzMC4yLDcuNzQ3LDM4Ljk2OCwyMC43MjMgYzIuNzgyLDQuMTE3LDguMzc1LDUuMjAxLDEyLjQ5NiwyLjQxOGM0LjExOC0yLjc4Miw1LjIwMS04LjM3NywyLjQxOC0xMi40OTZjLTEyLjExOC0xNy45MzctMzIuMjYyLTI4LjY0NS01My44ODItMjguNjQ1IGMtMzUuODMzLDAtNjQuOTg1LDI5LjE1Mi02NC45ODUsNjQuOTg2czI5LjE1Miw2NC45ODYsNjQuOTg1LDY0Ljk4NmMyMi4yODEsMCw0Mi43NTktMTEuMjE4LDU0Ljc3OC0zMC4wMDkgQzIwMC4yMDgsMTY5LjE0NywxOTguOTg1LDE2My41ODIsMTk0Ljc5OCwxNjAuOTAzeiIvPiA8L2c+IDwvZz4KDTwvc3ZnPg==" height="22">
 
-[English][en-url] | 简体中文
+English | [简体中文][zh-cn-url]
 
 </div>
 
+## Introduction
+
+`orderwal` is generic-purpose, atomic, ordered, zero-copy, concurrent-safe, pre-allocate style (memory map) write-ahead-log for developing databases.
+
+`orderwal` also supports generic structured key and value types, which is not limited to just bytes like other implementations.
+
 ## Installation
 
 ```toml
 [dependencies]
-orderwal = "0.3"
+orderwal = "0.5"
 ```
 
+## Example
+
+See [examples](./examples/) for more information.
+
+## Related projects
+
+- [`aol`](https://github.com/al8n/aol): Yet another generic purpose, append-only write-ahead log implementation based on `std::fs::File`.
+- [`skl`](https://github.com/al8n/skl): A lock-free, ARNEA based skiplist implementation, which supports in-memory and on-disk, suitable for frozen durable data file or memtable for LSM database.
+- [`valog`](https://github.com/al8n/valog): A lock-free, generic, lightweight value log for WiscKey or Bitcask architecture databases.
+- [`dtlog`](https://github.com/al8n/dtlog): A log for tracking discard stats of multi-files databases.
+
 #### License
 
 `orderwal` is under the terms of both the MIT license and the
@@ -36,12 +53,8 @@ See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for details.
 Copyright (c) 2024 Al Liu.
 
 [Github-url]: https://github.com/al8n/orderwal/
-[CI-url]: https://github.com/al8n/template/actions/workflows/template.yml
+[CI-url]: https://github.com/al8n/orderwal/actions/workflows/ci.yml
 [doc-url]: https://docs.rs/orderwal
 [crates-url]: https://crates.io/crates/orderwal
 [codecov-url]: https://app.codecov.io/gh/al8n/orderwal/
-[license-url]: https://opensource.org/licenses/Apache-2.0
-[rustc-url]: https://github.com/rust-lang/rust/blob/master/RELEASES.md
-[license-apache-url]: https://opensource.org/licenses/Apache-2.0
-[license-mit-url]: https://opensource.org/licenses/MIT
-[en-url]: https://github.com/al8n/orderwal/tree/main/README.md
+[zh-cn-url]: https://github.com/al8n/orderwal/tree/main/README-zh_CN.md

README.md

@@ -29,7 +29,7 @@ English | [简体中文][zh-cn-url]
 
 ```toml
 [dependencies]
-orderwal = "0.4"
+orderwal = "0.5"
 ```
 
 ## Example
@@ -40,6 +40,8 @@ See [examples](./examples/) for more information.
 
 - [`aol`](https://github.com/al8n/aol): Yet another generic purpose, append-only write-ahead log implementation based on `std::fs::File`.
 - [`skl`](https://github.com/al8n/skl): A lock-free, ARNEA based skiplist implementation, which supports in-memory and on-disk, suitable for frozen durable data file or memtable for LSM database.
+- [`valog`](https://github.com/al8n/valog): A lock-free, generic, lightweight value log for WiscKey or Bitcask architecture databases.
+- [`dtlog`](https://github.com/al8n/dtlog): A log for tracking discard stats of multi-files databases.
 
 #### License
 

ci/miri_tb.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
-set -e
+set -euxo pipefail
+IFS=$'\n\t'
+
+# We need 'ts' for the per-line timing
+sudo apt-get -y install moreutils
+echo
 
 # Check if TARGET and CONFIG_FLAGS are provided, otherwise panic
 if [ -z "$1" ]; then
@@ -19,9 +24,9 @@ rustup toolchain install nightly --component miri
 rustup override set nightly
 cargo miri setup
 
+# Zmiri-ignore-leaks needed because of https://github.com/crossbeam-rs/crossbeam/issues/579
 export MIRIFLAGS="-Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-tree-borrows -Zmiri-ignore-leaks"
-
 export RUSTFLAGS="--cfg test_$CONFIG_FLAGS"
 
-cargo miri test --tests --target $TARGET --lib
+cargo miri test --tests --target $TARGET --lib 2>&1 | ts -i '%.s  '
 

ci/sanitizer.sh

@@ -5,15 +5,15 @@ set -ex
 export ASAN_OPTIONS="detect_odr_violation=0 detect_leaks=0"
 
 # Run address sanitizer
-RUSTFLAGS="-Z sanitizer=address --cfg all_tests" \
+RUSTFLAGS="-Z sanitizer=address --cfg all_orderwal_tests" \
 cargo test -Z build-std --all --release --tests --target x86_64-unknown-linux-gnu --all-features --exclude benchmarks -- --test-threads=1
 
 # Run memory sanitizer
-RUSTFLAGS="-Z sanitizer=memory --cfg all_tests" \
+RUSTFLAGS="-Z sanitizer=memory --cfg all_orderwal_tests" \
 cargo test -Z build-std --all --release --tests --target x86_64-unknown-linux-gnu --all-features --exclude benchmarks -- --test-threads=1
 
 # Run thread sanitizer
 cargo clean
 TSAN_OPTIONS="suppressions=$(pwd)/ci/tsan" \
-RUSTFLAGS="${RUSTFLAGS:-} -Z sanitizer=thread --cfg all_tests" \
+RUSTFLAGS="${RUSTFLAGS:-} -Z sanitizer=thread --cfg all_orderwal_tests" \
     cargo test -Z build-std --all --release --target x86_64-unknown-linux-gnu --all-features --tests --exclude benchmarks -- --test-threads=1

examples/generic_not_sized.rs

@@ -1 +1,28 @@
-fn main() {}
+use orderwal::{
+  base::{OrderWal, Reader, Writer},
+  Builder,
+};
+
+fn main() {
+  let dir = tempfile::tempdir().unwrap();
+  let path = dir.path().join("not_sized.wal");
+
+  let mut wal = unsafe {
+    Builder::new()
+      .with_capacity(1024 * 1024)
+      .with_create_new(true)
+      .with_read(true)
+      .with_write(true)
+      .map_mut::<OrderWal<str, [u8]>, _>(&path)
+      .unwrap()
+  };
+
+  wal.insert("a", b"a1".as_slice()).unwrap();
+  wal.insert("c", b"c1".as_slice()).unwrap();
+
+  let a = wal.get("a").unwrap();
+  let c = wal.get("c").unwrap();
+
+  assert_eq!(a.value(), b"a1");
+  assert_eq!(c.value(), b"c1");
+}

examples/multiple_version.rs

@@ -0,0 +1,36 @@
+use orderwal::{
+  multiple_version::{OrderWal, Reader, Writer},
+  Builder,
+};
+
+fn main() {
+  let dir = tempfile::tempdir().unwrap();
+  let path = dir.path().join("not_sized.wal");
+
+  let mut wal = unsafe {
+    Builder::new()
+      .with_capacity(1024 * 1024)
+      .with_create_new(true)
+      .with_read(true)
+      .with_write(true)
+      .map_mut::<OrderWal<str, [u8]>, _>(&path)
+      .unwrap()
+  };
+
+  wal.insert(1, "a", b"a1".as_slice()).unwrap();
+  wal.insert(3, "a", b"a3".as_slice()).unwrap();
+  wal.insert(1, "c", b"c1".as_slice()).unwrap();
+  wal.insert(3, "c", b"c3".as_slice()).unwrap();
+
+  let a = wal.get(2, "a").unwrap();
+  let c = wal.get(2, "c").unwrap();
+
+  assert_eq!(a.value(), b"a1");
+  assert_eq!(c.value(), b"c1");
+
+  let a = wal.get(3, "a").unwrap();
+  let c = wal.get(3, "c").unwrap();
+
+  assert_eq!(a.value(), b"a3");
+  assert_eq!(c.value(), b"c3");
+}