feat: sync tagged logs (#9595)

Closes: #9380

Implements the logic to sync tagged note logs from the node, taking into
account the indices of "last seen" ones.

Unfortunately, the approach of modularizing it as oracles and putting
the logic in the contract itself was met with limitations in noir
(namely, nested slices) and even the code is ready and close to becoming
separate oracles, it cannot be done (yet! or at least without major
caveats and performance implications)

noir-projects/aztec-nr/aztec/src/oracle/notes.nr

@@ -231,11 +231,11 @@ pub unconstrained fn get_app_tagging_secrets_for_senders(
     let results = get_app_tagging_secrets_for_senders_oracle(recipient);
     let mut indexed_tagging_secrets = &[];
     for i in 0..results.len() {
-        if i % 2 != 0 {
+        if i % 3 != 0 {
             continue;
         }
         indexed_tagging_secrets = indexed_tagging_secrets.push_back(
-            IndexedTaggingSecret::deserialize([results[i], results[i + 1]]),
+            IndexedTaggingSecret::deserialize([results[i], results[i + 1], results[i + 2]]),
         );
     }
     indexed_tagging_secrets

noir-projects/noir-protocol-circuits/crates/types/src/indexed_tagging_secret.nr

@@ -1,10 +1,12 @@
 use crate::traits::Deserialize;
+use super::address::aztec_address::AztecAddress;
 use std::meta::derive;
 
-pub global INDEXED_TAGGING_SECRET_LENGTH: u32 = 2;
+pub global INDEXED_TAGGING_SECRET_LENGTH: u32 = 3;
 
 #[derive(Deserialize)]
 pub struct IndexedTaggingSecret {
     secret: Field,
+    recipient: AztecAddress,
     index: u32,
 }

yarn-project/aztec.js/src/utils/account.ts

@@ -1,4 +1,5 @@
-import { type CompleteAddress, type PXE } from '@aztec/circuit-types';
+import { type PXE } from '@aztec/circuit-types';
+import { type CompleteAddress } from '@aztec/circuits.js';
 import { retryUntil } from '@aztec/foundation/retry';
 
 import { DefaultWaitOpts, type WaitOpts } from '../contract/sent_tx.js';

yarn-project/circuit-types/src/logs/encrypted_l2_note_log.ts

@@ -60,10 +60,10 @@ export class EncryptedL2NoteLog {
    * Crates a random log.
    * @returns A random log.
    */
-  public static random(): EncryptedL2NoteLog {
+  public static random(tag: Fr = Fr.random()): EncryptedL2NoteLog {
     const randomEphPubKey = Point.random();
     const randomLogContent = randomBytes(144 - Point.COMPRESSED_SIZE_IN_BYTES);
-    const data = Buffer.concat([Fr.random().toBuffer(), randomLogContent, randomEphPubKey.toCompressedBuffer()]);
+    const data = Buffer.concat([tag.toBuffer(), randomLogContent, randomEphPubKey.toCompressedBuffer()]);
     return new EncryptedL2NoteLog(data);
   }
 

yarn-project/circuits.js/src/keys/derivation.ts

@@ -134,5 +134,8 @@ export function computeTaggingSecret(knownAddress: CompleteAddress, ivsk: Fq, ex
   const curve = new Grumpkin();
   // Given A (known complete address) -> B (external address) and h == preaddress
   // Compute shared secret as S = (h_A + ivsk_A) * Addr_Point_B
-  return curve.mul(externalAddressPoint, ivsk.add(new Fq(knownPreaddress.toBigInt())));
+
+  // Beware! h_a + ivsk_a (also known as the address secret) can lead to an address point with a negative y-coordinate, since there's two possible candidates
+  // computeAddressSecret takes care of selecting the one that leads to a positive y-coordinate, which is the only valid address point
+  return curve.mul(externalAddressPoint, computeAddressSecret(knownPreaddress, ivsk));
 }

yarn-project/circuits.js/src/structs/index.ts

@@ -13,7 +13,7 @@ export * from './gas_fees.js';
 export * from './gas_settings.js';
 export * from './global_variables.js';
 export * from './header.js';
-export * from './indexed_tagging_secret.js';
+export * from './tagging_secret.js';
 export * from './kernel/combined_accumulated_data.js';
 export * from './kernel/combined_constant_data.js';
 export * from './kernel/enqueued_call_data.js';

yarn-project/circuits.js/src/structs/tagging_secret.ts

@@ -0,0 +1,20 @@
+import { type AztecAddress } from '@aztec/foundation/aztec-address';
+import { Fr } from '@aztec/foundation/fields';
+
+export class TaggingSecret {
+  constructor(public secret: Fr, public recipient: AztecAddress) {}
+
+  toFields(): Fr[] {
+    return [this.secret, this.recipient];
+  }
+}
+
+export class IndexedTaggingSecret extends TaggingSecret {
+  constructor(secret: Fr, recipient: AztecAddress, public index: number) {
+    super(secret, recipient);
+  }
+
+  override toFields(): Fr[] {
+    return [this.secret, this.recipient, new Fr(this.index)];
+  }
+}

yarn-project/pxe/package.json

@@ -87,9 +87,11 @@
     "@jest/globals": "^29.5.0",
     "@types/jest": "^29.5.0",
     "@types/lodash.omit": "^4.5.7",
+    "@types/lodash.times": "^4.3.9",
     "@types/node": "^18.7.23",
     "jest": "^29.5.0",
     "jest-mock-extended": "^3.0.3",
+    "lodash.times": "^4.3.2",
     "ts-node": "^10.9.1",
     "typescript": "^5.0.4"
   },

yarn-project/pxe/src/database/kv_pxe_database.ts

@@ -1,16 +1,12 @@
-import {
-  type IncomingNotesFilter,
-  MerkleTreeId,
-  NoteStatus,
-  type OutgoingNotesFilter,
-  type PublicKey,
-} from '@aztec/circuit-types';
+import { type IncomingNotesFilter, MerkleTreeId, NoteStatus, type OutgoingNotesFilter } from '@aztec/circuit-types';
 import {
   AztecAddress,
   CompleteAddress,
   type ContractInstanceWithAddress,
   Header,
+  type PublicKey,
   SerializableContractInstance,
+  type TaggingSecret,
   computePoint,
 } from '@aztec/circuits.js';
 import { type ContractArtifact } from '@aztec/foundation/abi';
@@ -576,19 +572,23 @@ export class KVPxeDatabase implements PxeDatabase {
     return incomingNotesSize + outgoingNotesSize + treeRootsSize + authWitsSize + addressesSize;
   }
 
-  async incrementTaggingSecretsIndexes(appTaggingSecrets: Fr[]): Promise<void> {
-    const indexes = await this.getTaggingSecretsIndexes(appTaggingSecrets);
+  async incrementTaggingSecretsIndexes(appTaggingSecretsWithRecipient: TaggingSecret[]): Promise<void> {
+    const indexes = await this.getTaggingSecretsIndexes(appTaggingSecretsWithRecipient);
     await this.db.transaction(() => {
-      indexes.forEach(index => {
-        const nextIndex = index ? index + 1 : 1;
-        void this.#taggingSecretIndexes.set(appTaggingSecrets.toString(), nextIndex);
+      indexes.forEach((taggingSecretIndex, listIndex) => {
+        const nextIndex = taggingSecretIndex ? taggingSecretIndex + 1 : 1;
+        const { secret, recipient } = appTaggingSecretsWithRecipient[listIndex];
+        const key = `${secret.toString()}-${recipient.toString()}`;
+        void this.#taggingSecretIndexes.set(key, nextIndex);
       });
     });
   }
 
-  getTaggingSecretsIndexes(appTaggingSecrets: Fr[]): Promise<number[]> {
+  getTaggingSecretsIndexes(appTaggingSecretsWithRecipient: TaggingSecret[]): Promise<number[]> {
     return this.db.transaction(() =>
-      appTaggingSecrets.map(secret => this.#taggingSecretIndexes.get(secret.toString()) ?? 0),
+      appTaggingSecretsWithRecipient.map(
+        ({ secret, recipient }) => this.#taggingSecretIndexes.get(`${secret.toString()}-${recipient.toString()}`) ?? 0,
+      ),
     );
   }
 }

yarn-project/pxe/src/database/pxe_database.ts

@@ -4,6 +4,7 @@ import {
   type ContractInstanceWithAddress,
   type Header,
   type PublicKey,
+  type TaggingSecret,
 } from '@aztec/circuits.js';
 import { type ContractArtifact } from '@aztec/foundation/abi';
 import { type AztecAddress } from '@aztec/foundation/aztec-address';
@@ -186,7 +187,20 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD
    */
   estimateSize(): Promise<number>;
 
-  getTaggingSecretsIndexes(appTaggingSecrets: Fr[]): Promise<number[]>;
+  /**
+   * Returns the last seen indexes for the provided app siloed tagging secrets or 0 if they've never been seen.
+   * The recipient must also be provided to convey "directionality" of the secret and index pair, or in other words
+   * whether the index was used to tag a sent or received note.
+   * @param appTaggingSecrets - The app siloed tagging secrets.
+   * @returns The indexes for the provided secrets, 0 if they've never been seen.
+   */
+  getTaggingSecretsIndexes(appTaggingSecretsWithRecipient: TaggingSecret[]): Promise<number[]>;
 
-  incrementTaggingSecretsIndexes(appTaggingSecrets: Fr[]): Promise<void>;
+  /**
+   * Increments the index for the provided app siloed tagging secrets.
+   * The recipient must also be provided to convey "directionality" of the secret and index pair, or in other words
+   * whether the index was used to tag a sent or received note.
+   * @param appTaggingSecrets - The app siloed tagging secrets.
+   */
+  incrementTaggingSecretsIndexes(appTaggingSecretsWithRecipient: TaggingSecret[]): Promise<void>;
 }

yarn-project/pxe/src/index.ts

@@ -4,7 +4,7 @@ export * from './config/index.js';
 
 export { Tx, TxHash } from '@aztec/circuit-types';
 
-export { TxRequest, PartialAddress } from '@aztec/circuits.js';
+export { TxRequest } from '@aztec/circuits.js';
 export * from '@aztec/foundation/fields';
 export * from '@aztec/foundation/eth-address';
 export * from '@aztec/foundation/aztec-address';

yarn-project/pxe/src/simulator_oracle/index.ts

@@ -1,5 +1,6 @@
 import {
   type AztecNode,
+  type EncryptedL2NoteLog,
   type L2Block,
   MerkleTreeId,
   type NoteStatus,
@@ -17,6 +18,7 @@ import {
   IndexedTaggingSecret,
   type KeyValidationRequest,
   type L1_TO_L2_MSG_TREE_HEIGHT,
+  TaggingSecret,
   computeTaggingSecret,
 } from '@aztec/circuits.js';
 import { type FunctionArtifact, getFunctionArtifact } from '@aztec/foundation/abi';
@@ -247,9 +249,11 @@ export class SimulatorOracle implements DBOracle {
     const senderIvsk = await this.keyStore.getMasterIncomingViewingSecretKey(sender);
     const sharedSecret = computeTaggingSecret(senderCompleteAddress, senderIvsk, recipient);
     // Silo the secret to the app so it can't be used to track other app's notes
-    const secret = poseidon2Hash([sharedSecret.x, sharedSecret.y, contractAddress]);
-    const [index] = await this.db.getTaggingSecretsIndexes([secret]);
-    return new IndexedTaggingSecret(secret, index);
+    const siloedSecret = poseidon2Hash([sharedSecret.x, sharedSecret.y, contractAddress]);
+    // Get the index of the secret, ensuring the directionality (sender -> recipient)
+    const directionalSecret = new TaggingSecret(siloedSecret, recipient);
+    const [index] = await this.db.getTaggingSecretsIndexes([directionalSecret]);
+    return new IndexedTaggingSecret(siloedSecret, recipient, index);
   }
 
   /**
@@ -274,7 +278,55 @@ export class SimulatorOracle implements DBOracle {
       const sharedSecret = computeTaggingSecret(recipientCompleteAddress, recipientIvsk, sender);
       return poseidon2Hash([sharedSecret.x, sharedSecret.y, contractAddress]);
     });
-    const indexes = await this.db.getTaggingSecretsIndexes(secrets);
-    return secrets.map((secret, i) => new IndexedTaggingSecret(secret, indexes[i]));
+    // Ensure the directionality (sender -> recipient)
+    const directionalSecrets = secrets.map(secret => new TaggingSecret(secret, recipient));
+    const indexes = await this.db.getTaggingSecretsIndexes(directionalSecrets);
+    return directionalSecrets.map(
+      ({ secret, recipient }, i) => new IndexedTaggingSecret(secret, recipient, indexes[i]),
+    );
+  }
+
+  /**
+   * Synchronizes the logs tagged with the recipient's address and all the senders in the addressbook.
+   * Returns the unsynched logs and updates the indexes of the secrets used to tag them until there are no more logs to sync.
+   * @param contractAddress - The address of the contract that the logs are tagged for
+   * @param recipient - The address of the recipient
+   * @returns A list of encrypted logs tagged with the recipient's address
+   */
+  public async syncTaggedLogs(contractAddress: AztecAddress, recipient: AztecAddress): Promise<EncryptedL2NoteLog[]> {
+    // Ideally this algorithm would be implemented in noir, exposing its building blocks as oracles.
+    // However it is impossible at the moment due to the language not supporting nested slices.
+    // This nesting is necessary because for a given set of tags we don't
+    // know how many logs we will get back. Furthermore, these logs are of undetermined
+    // length, since we don't really know the note they correspond to until we decrypt them.
+
+    // 1. Get all the secrets for the recipient and sender pairs (#9365)
+    let appTaggingSecrets = await this.getAppTaggingSecretsForSenders(contractAddress, recipient);
+
+    const logs: EncryptedL2NoteLog[] = [];
+    while (appTaggingSecrets.length > 0) {
+      // 2. Compute tags using the secrets, recipient and index. Obtain logs for each tag (#9380)
+      const currentTags = appTaggingSecrets.map(({ secret, recipient, index }) =>
+        poseidon2Hash([secret, recipient, index]),
+      );
+      const logsByTags = await this.aztecNode.getLogsByTags(currentTags);
+      const newTaggingSecrets: IndexedTaggingSecret[] = [];
+      logsByTags.forEach((logsByTag, index) => {
+        // 3.1. Append logs to the list and increment the index for the tags that have logs (#9380)
+        if (logsByTag.length > 0) {
+          logs.push(...logsByTag);
+          // 3.2. Increment the index for the tags that have logs (#9380)
+          newTaggingSecrets.push(
+            new IndexedTaggingSecret(appTaggingSecrets[index].secret, recipient, appTaggingSecrets[index].index + 1),
+          );
+        }
+      });
+      // 4. Consolidate in db and replace initial appTaggingSecrets with the new ones (updated indexes)
+      await this.db.incrementTaggingSecretsIndexes(
+        newTaggingSecrets.map(secret => new TaggingSecret(secret.secret, recipient)),
+      );
+      appTaggingSecrets = newTaggingSecrets;
+    }
+    return logs;
   }
 }