Skip to content

Commit

Permalink
Simplify blake3 hasher and improve performance
Browse files Browse the repository at this point in the history
Improve BLAKE3 performance by reducing copies.

Rather than buffering bytes, it's faster to just update the blake3 hasher as bytes are hashed.

Performance with buffering:
```
 Benchmark                   (inputSize)  Mode  Cnt      Score     Error  Units
 Blake3Benchmark.blake3Hash            1  avgt   10   1766.697 ± 709.515  ns/op
 Blake3Benchmark.blake3Hash           10  avgt   10   1466.253 ±  19.474  ns/op
 Blake3Benchmark.blake3Hash          100  avgt   10   1522.845 ±  15.480  ns/op
 Blake3Benchmark.blake3Hash         1000  avgt   10   2254.156 ±   8.588  ns/op
 Blake3Benchmark.blake3Hash        10000  avgt   10   4660.881 ±  28.637  ns/op
 Blake3Benchmark.blake3Hash       100000  avgt   10  24283.191 ±  32.754  ns/op
 Blake3Benchmark.sha256Hash            1  avgt   10    742.091 ±   6.307  ns/op
 Blake3Benchmark.sha256Hash           10  avgt   10    757.844 ±  12.042  ns/op
 Blake3Benchmark.sha256Hash          100  avgt   10    942.902 ± 555.874  ns/op
 Blake3Benchmark.sha256Hash         1000  avgt   10   1208.336 ± 508.392  ns/op
 Blake3Benchmark.sha256Hash        10000  avgt   10   4871.231 ± 494.507  ns/op
 Blake3Benchmark.sha256Hash       100000  avgt   10  40686.231 ±  63.814  ns/op
```

Performance without buffering (after this CL):

```
 Benchmark                   (inputSize)  Mode  Cnt      Score     Error  Units
 Blake3Benchmark.blake3Hash            1  avgt   10   1021.075 ±  11.640  ns/op
 Blake3Benchmark.blake3Hash           10  avgt   10   1029.561 ±  19.850  ns/op
 Blake3Benchmark.blake3Hash          100  avgt   10   1070.509 ±  12.140  ns/op
 Blake3Benchmark.blake3Hash         1000  avgt   10   1685.043 ±  13.963  ns/op
 Blake3Benchmark.blake3Hash        10000  avgt   10   3939.516 ±  13.212  ns/op
 Blake3Benchmark.blake3Hash       100000  avgt   10  21730.550 ±  22.976  ns/op
 Blake3Benchmark.sha256Hash            1  avgt   10    745.943 ±   9.853  ns/op
 Blake3Benchmark.sha256Hash           10  avgt   10    747.649 ±  17.381  ns/op
 Blake3Benchmark.sha256Hash          100  avgt   10    962.802 ± 590.879  ns/op
 Blake3Benchmark.sha256Hash         1000  avgt   10   1189.069 ± 412.327  ns/op
 Blake3Benchmark.sha256Hash        10000  avgt   10   4594.978 ±  21.833  ns/op
 Blake3Benchmark.sha256Hash       100000  avgt   10  39224.248 ± 229.265  ns/op
```

Closes bazelbuild#19041.

PiperOrigin-RevId: 551225043
Change-Id: I57dc0700b2f44d6faf75ffbd29f1607544e54f29
  • Loading branch information
tylerwilliams authored and copybara-github committed Jul 26, 2023
1 parent 5647b26 commit d922ab3
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@
// limitations under the License.
package com.google.devtools.build.lib.vfs.bazel;

import static java.lang.Math.min;

import com.google.devtools.build.lib.jni.JniLoader;
import java.nio.ByteBuffer;
import java.security.DigestException;
Expand All @@ -38,45 +36,23 @@ public final class Blake3MessageDigest extends MessageDigest {
initialize_hasher(INITIAL_STATE);
}

// To reduce the number of calls made via JNI, buffer up to this many bytes
// before updating the hasher.
public static final int ONESHOT_THRESHOLD = 8 * 1024;

private final ByteBuffer buffer = ByteBuffer.allocate(ONESHOT_THRESHOLD);
private final byte[] hasher = new byte[STATE_SIZE];
private final byte[] oneByteArray = new byte[1];

public Blake3MessageDigest() {
super("BLAKE3");
System.arraycopy(INITIAL_STATE, 0, hasher, 0, STATE_SIZE);
}

private void flush() {
if (buffer.position() > 0) {
blake3_hasher_update(hasher, buffer.array(), buffer.position());
buffer.clear();
}
}

@Override
public void engineUpdate(byte[] data, int offset, int length) {
while (length > 0) {
int numToCopy = min(length, buffer.remaining());
buffer.put(data, offset, numToCopy);
length -= numToCopy;
offset += numToCopy;

if (buffer.remaining() == 0) {
flush();
}
}
blake3_hasher_update(hasher, data, offset, length);
}

@Override
public void engineUpdate(byte b) {
if (buffer.remaining() == 0) {
flush();
}
buffer.put(b);
oneByteArray[0] = b;
engineUpdate(oneByteArray, 0, 1);
}

@Override
Expand All @@ -85,8 +61,6 @@ public void engineUpdate(ByteBuffer input) {
}

private byte[] getOutput(int outputLength) {
flush();

byte[] retByteArray = new byte[outputLength];
blake3_hasher_finalize(hasher, retByteArray, outputLength);

Expand All @@ -101,7 +75,6 @@ public Object clone() throws CloneNotSupportedException {

@Override
public void engineReset() {
buffer.clear();
System.arraycopy(INITIAL_STATE, 0, hasher, 0, STATE_SIZE);
}

Expand Down Expand Up @@ -133,7 +106,8 @@ public int engineDigest(byte[] buf, int off, int len) throws DigestException {

public static final native void initialize_hasher(byte[] hasher);

public static final native void blake3_hasher_update(byte[] hasher, byte[] input, int inputLen);
public static final native void blake3_hasher_update(
byte[] hasher, byte[] input, int offset, int inputLen);

public static final native void blake3_hasher_finalize(byte[] hasher, byte[] out, int outLen);
}
4 changes: 2 additions & 2 deletions src/main/native/blake3_jni.cc
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,12 @@ Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_initialize_1has

extern "C" JNIEXPORT void JNICALL
Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_blake3_1hasher_1update(
JNIEnv *env, jobject obj, jbyteArray jhasher, jbyteArray input,
JNIEnv *env, jobject obj, jbyteArray jhasher, jbyteArray input, jint offset,
jint input_len) {
blake3_hasher *hasher = (blake3_hasher *)get_byte_array(env, jhasher);
if (hasher) {
jbyte *input_addr = get_byte_array(env, input);
blake3_hasher_update(hasher, input_addr, input_len);
blake3_hasher_update(hasher, input_addr + offset, input_len);
release_byte_array(env, input, input_addr);
release_byte_array(env, jhasher, (jbyte *)hasher);
}
Expand Down

0 comments on commit d922ab3

Please sign in to comment.