chore: switch to Noir Keccak implementation with variable size support (

#5508)

# Description

## Problem\*

We are still using a blackbox function for Keccak, while we have the
keccak permutation blackbox.


## Summary\*

Add variable size support for Noir Keccak, and also switch to the Noir
implementation in stdlib

## Additional Context



## Documentation\*

Check one:
- [X] No documentation needed.
- [ ] Documentation included in this PR.
- [ ] **[For Experimental Features]** Documentation to be submitted in a
separate PR.

# PR Checklist\*

- [X] I have tested the changes locally.
- [X] I have formatted the changes with [Prettier](https://prettier.io/)
and/or `cargo fmt` on default settings.

compiler/noirc_evaluator/src/ssa/acir_gen/acir_ir/generated_acir.rs

@@ -294,24 +294,7 @@ impl<F: AcirField> GeneratedAcir<F> {
                 outputs: (outputs[0], outputs[1], outputs[2]),
             },
             BlackBoxFunc::Keccak256 => {
-                let var_message_size = match inputs.to_vec().pop() {
-                    Some(var_message_size) => var_message_size[0],
-                    None => {
-                        return Err(InternalError::MissingArg {
-                            name: "".to_string(),
-                            arg: "message_size".to_string(),
-                            call_stack: self.call_stack.clone(),
-                        });
-                    }
-                };
-
-                BlackBoxFuncCall::Keccak256 {
-                    inputs: inputs[0].clone(),
-                    var_message_size,
-                    outputs: outputs
-                        .try_into()
-                        .expect("Compiler should generate correct size outputs"),
-                }
+                unreachable!("unexpected BlackBox {}", func_name.to_string())
             }
             BlackBoxFunc::Keccakf1600 => BlackBoxFuncCall::Keccakf1600 {
                 inputs: inputs[0]

compiler/noirc_evaluator/src/ssa/ir/instruction/call.rs

@@ -461,28 +461,37 @@ fn simplify_black_box_func(
         BlackBoxFunc::SHA256 => simplify_hash(dfg, arguments, acvm::blackbox_solver::sha256),
         BlackBoxFunc::Blake2s => simplify_hash(dfg, arguments, acvm::blackbox_solver::blake2s),
         BlackBoxFunc::Blake3 => simplify_hash(dfg, arguments, acvm::blackbox_solver::blake3),
-        BlackBoxFunc::PedersenCommitment
-        | BlackBoxFunc::PedersenHash
-        | BlackBoxFunc::Keccakf1600 => SimplifyResult::None, //TODO(Guillaume)
-        BlackBoxFunc::Keccak256 => {
-            match (dfg.get_array_constant(arguments[0]), dfg.get_numeric_constant(arguments[1])) {
-                (Some((input, _)), Some(num_bytes)) if array_is_constant(dfg, &input) => {
-                    let input_bytes: Vec<u8> = to_u8_vec(dfg, input);
-
-                    let num_bytes = num_bytes.to_u128() as usize;
-                    let truncated_input_bytes = &input_bytes[0..num_bytes];
-                    let hash = acvm::blackbox_solver::keccak256(truncated_input_bytes)
-                        .expect("Rust solvable black box function should not fail");
-
-                    let hash_values =
-                        vecmap(hash, |byte| FieldElement::from_be_bytes_reduce(&[byte]));
-
-                    let result_array = make_constant_array(dfg, hash_values, Type::unsigned(8));
+        BlackBoxFunc::PedersenCommitment | BlackBoxFunc::PedersenHash => SimplifyResult::None,
+        BlackBoxFunc::Keccakf1600 => {
+            if let Some((array_input, _)) = dfg.get_array_constant(arguments[0]) {
+                if array_is_constant(dfg, &array_input) {
+                    let const_input: Vec<u64> = array_input
+                        .iter()
+                        .map(|id| {
+                            let field = dfg
+                                .get_numeric_constant(*id)
+                                .expect("value id from array should point at constant");
+                            field.to_u128() as u64
+                        })
+                        .collect();
+
+                    let state = acvm::blackbox_solver::keccakf1600(
+                        const_input.try_into().expect("Keccakf1600 input should have length of 25"),
+                    )
+                    .expect("Rust solvable black box function should not fail");
+                    let state_values = vecmap(state, |x| FieldElement::from(x as u128));
+                    let result_array = make_constant_array(dfg, state_values, Type::unsigned(64));
                     SimplifyResult::SimplifiedTo(result_array)
+                } else {
+                    SimplifyResult::None
                 }
-                _ => SimplifyResult::None,
+            } else {
+                SimplifyResult::None
             }
         }
+        BlackBoxFunc::Keccak256 => {
+            unreachable!("Keccak256 should have been replaced by calls to Keccakf1600")
+        }
         BlackBoxFunc::Poseidon2Permutation => SimplifyResult::None, //TODO(Guillaume)
         BlackBoxFunc::EcdsaSecp256k1 => {
             simplify_signature(dfg, arguments, acvm::blackbox_solver::ecdsa_secp256k1_verify)

compiler/noirc_evaluator/src/ssa/opt/flatten_cfg.rs

@@ -1412,7 +1412,7 @@ mod test {
         // Tests that it does not simplify a true constraint an always-false constraint
         // acir(inline) fn main f1 {
         //     b0(v0: [u8; 2]):
-        //       v4 = call keccak256(v0, u8 2)
+        //       v4 = call sha256(v0, u8 2)
         //       v5 = array_get v4, index u8 0
         //       v6 = cast v5 as u32
         //       v8 = truncate v6 to 1 bits, max_bit_size: 32
@@ -1448,7 +1448,7 @@ mod test {
         let two = builder.numeric_constant(2_u128, Type::unsigned(8));
 
         let keccak =
-            builder.import_intrinsic_id(Intrinsic::BlackBox(acvm::acir::BlackBoxFunc::Keccak256));
+            builder.import_intrinsic_id(Intrinsic::BlackBox(acvm::acir::BlackBoxFunc::SHA256));
         let v4 =
             builder.insert_call(keccak, vec![array, two], vec![Type::Array(element_type, 32)])[0];
         let v5 = builder.insert_array_get(v4, zero, Type::unsigned(8));

noir_stdlib/src/hash/keccak.nr

@@ -3,38 +3,39 @@ global NUM_KECCAK_LANES = 25;
 global BLOCK_SIZE = 136; //(1600 - BITS * 2) / WORD_SIZE;
 global WORD_SIZE = 8;
 
-use crate::collections::bounded_vec::BoundedVec;
+use crate::collections::vec::Vec;
 
 #[foreign(keccakf1600)]
 fn keccakf1600(input: [u64; 25]) -> [u64; 25] {}
 
-fn keccak256<let N: u32>(input: [u8; N], message_size: u32) -> [u8; 32] {
-    // No var keccak for now
-    assert(N == message_size);
+#[no_predicates]
+pub(crate) fn keccak256<let N: u32>(mut input: [u8; N], message_size: u32) -> [u8; 32] {
+    assert(N >= message_size);
+    for i in 0..N {
+        if i >= message_size {
+            input[i] = 0;
+        }
+    }
 
     //1. format_input_lanes
     let max_blocks = (N + BLOCK_SIZE) / BLOCK_SIZE;
     //maximum number of bytes to hash
     let max_blocks_length = (BLOCK_SIZE * (max_blocks));
-    assert(max_blocks_length < 1000);
-    let mut block_bytes :BoundedVec<u8,1000> = BoundedVec::from_array(input);
-    for i in N..N + BLOCK_SIZE {
-        let data = if i == N {
-            1
-        } else if i == max_blocks_length - 1 {
-            0x80
-        } else {
-            0
-        };
-        block_bytes.push(data);
+    let real_max_blocks = (message_size + BLOCK_SIZE) / BLOCK_SIZE;
+    let real_blocks_bytes = real_max_blocks * BLOCK_SIZE;
+
+    let mut block_bytes = Vec::from_slice(input.as_slice());
+    for _i in N..N + BLOCK_SIZE {
+        block_bytes.push(0);
     }
+    block_bytes.set(message_size, 1);
+    block_bytes.set(real_blocks_bytes - 1, 0x80);
 
     // keccak lanes interpret memory as little-endian integers,
     // means we need to swap our byte ordering
     let num_limbs = max_blocks * LIMBS_PER_BLOCK; //max_blocks_length / WORD_SIZE;
     for i in 0..num_limbs {
         let mut temp = [0; 8];
-
         for j in 0..8 {
             temp[j] = block_bytes.get(8*i+j);
         }
@@ -43,8 +44,10 @@ fn keccak256<let N: u32>(input: [u8; N], message_size: u32) -> [u8; 32] {
         }
     }
     let  byte_size = max_blocks_length;
-    assert(num_limbs < 1000);
-    let mut sliced_buffer = [0; 1000];
+    let mut sliced_buffer = Vec::new();
+    for _i in 0..num_limbs {
+        sliced_buffer.push(0);
+    }
     // populate a vector of 64-bit limbs from our byte array
     for i in 0..num_limbs {
         let mut sliced = 0;
@@ -65,24 +68,29 @@ fn keccak256<let N: u32>(input: [u8; N], message_size: u32) -> [u8; 32] {
                 v *= 256;
             }
         }
-        sliced_buffer[i] = sliced as u64;
+        sliced_buffer.set(i, sliced as u64);
     }
 
     //2. sponge_absorb
     let num_blocks = max_blocks;
     let mut state : [u64;NUM_KECCAK_LANES]= [0; NUM_KECCAK_LANES];
-
+    let mut under_block = true;
     for i in 0..num_blocks {
-        if (i == 0) {
-            for j in 0..LIMBS_PER_BLOCK {
-                state[j] = sliced_buffer[j];
-            }
-        } else {
-            for j in 0..LIMBS_PER_BLOCK {
-                state[j] = state[j] ^ sliced_buffer[i * LIMBS_PER_BLOCK + j];
+        if i == real_max_blocks {
+            under_block = false;
+        }
+        if under_block {
+            if (i == 0) {
+                for j in 0..LIMBS_PER_BLOCK {
+                    state[j] = sliced_buffer.get(j);
+                }
+            } else {
+                for j in 0..LIMBS_PER_BLOCK {
+                    state[j] = state[j] ^ sliced_buffer.get(i * LIMBS_PER_BLOCK + j);
+                }
             }
+            state = keccakf1600(state);
         }
-        state = keccakf1600(state);
     }
 
     //3. sponge_squeeze
@@ -118,5 +126,16 @@ mod tests {
         ];
         assert_eq(keccak256(input, input.len()), result);
     }
+
+    #[test]
+    fn var_size_hash() {
+        let input = [
+            189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223
+        ];
+        let result = [
+            226, 37, 115, 94, 94, 196, 72, 116, 194, 105, 79, 233, 65, 12, 30, 94, 181, 131, 170, 219, 171, 166, 236, 88, 143, 67, 255, 160, 248, 214, 39, 129
+        ];
+        assert_eq(keccak256(input, 13), result);
+    }
 }
 

noir_stdlib/src/hash/mod.nr

@@ -101,11 +101,12 @@ pub fn hash_to_field(inputs: [Field]) -> Field {
     sum
 }
 
-#[foreign(keccak256)]
 // docs:start:keccak256
 pub fn keccak256<let N: u32>(input: [u8; N], message_size: u32) -> [u8; 32]
 // docs:end:keccak256
-{}
+{
+    crate::hash::keccak::keccak256(input, message_size)
+}
 
 #[foreign(poseidon2_permutation)]
 pub fn poseidon2_permutation<let N: u32>(_input: [Field; N], _state_length: u32) -> [Field; N] {}