HDDS-9260. Optimize OmDbInsight API to avoid redundant recursive size calculations for deleted directories during data retrieval.

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java

@@ -245,9 +245,6 @@ public void testGetDeletedDirectoryInfo()
         (KeyInsightInfoResponse) deletedDirInfo.getEntity();
     // Assert the size of deleted directory is 700.
     Assert.assertEquals(10, entity.getUnreplicatedDataSize());
-
-    // Cleanup the tables.
-    cleanupTables();
   }
 
 
@@ -322,7 +319,7 @@ public void testGetDeletedDirectoryInfoForNestedDirectories()
             mock(GlobalStatsDao.class), namespaceSummaryManager);
 
     // Delete the entire root directory dir1.
-    fs.delete(new Path("/dir1/dir2/dir3"), true);
+    fs.delete(new Path("/dir1"), true);
     impl.syncDataFromOM();
     // Verify the entries in the Recon tables after sync.
     assertTableRowCount(reconFileTable, 3, true);
@@ -335,9 +332,6 @@ public void testGetDeletedDirectoryInfoForNestedDirectories()
         (KeyInsightInfoResponse) deletedDirInfo.getEntity();
     // Assert the size of deleted directory is 1000.
     Assert.assertEquals(3, entity.getUnreplicatedDataSize());
-
-    // Cleanup the tables.
-    cleanupTables();
   }
 
   /**
@@ -399,8 +393,6 @@ public void testGetDeletedDirectoryInfoWithMultipleSubdirectories()
     // Assert the size of deleted directory is 100.
     Assert.assertEquals(100, entity.getUnreplicatedDataSize());
 
-    // Cleanup the tables.
-    cleanupTables();
   }
 
   private void createLargeDirectory(Path dir, int numSubdirs,
@@ -418,11 +410,96 @@ private void createLargeDirectory(Path dir, int numSubdirs,
     }
   }
 
+  /**
+   * This test case verifies the behavior of reusing a previously calculated
+   * size for a deleted directory in Recon. The calculation get triggered when
+   * the OMDBInsightEndpoint for deleted directory is invoked.
+   *
+   * @throws Exception if any error occurs during the test.
+   */
+  @Test
+  public void testReusingPreviouslyCalculatedSize()
+      throws Exception {
+
+    // Create a directory structure with 10 files and 3 nested directories.
+    Path path = new Path("/dir1/dir2/dir3");
+    fs.mkdirs(path);
+    // Create 3 files inside dir3.
+    for (int i = 1; i <= 3; i++) {
+      Path filePath = new Path(path, "testKey" + i);
+      try (FSDataOutputStream stream = fs.create(filePath)) {
+        stream.write(1);
+      }
+    }
+
+    // Sync data from Ozone Manager to Recon.
+    OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl)
+        cluster.getReconServer().getOzoneManagerServiceProvider();
+    impl.syncDataFromOM();
+
+    // Retrieve tables from Recon's OM-DB.
+    ReconOMMetadataManager reconOmMetadataManagerInstance =
+        (ReconOMMetadataManager) cluster.getReconServer()
+            .getOzoneManagerServiceProvider().getOMMetadataManagerInstance();
+    Table<String, OmKeyInfo> reconFileTable =
+        reconOmMetadataManagerInstance.getKeyTable(getFSOBucketLayout());
+    Table<String, OmDirectoryInfo> reconDirTable =
+        reconOmMetadataManagerInstance.getDirectoryTable();
+    Table<String, OmKeyInfo> reconDeletedDirTable =
+        reconOmMetadataManagerInstance.getDeletedDirTable();
+
+    // Create an Instance of OMDBInsightEndpoint.
+    OzoneStorageContainerManager reconSCM =
+        cluster.getReconServer().getReconStorageContainerManager();
+    ReconNamespaceSummaryManagerImpl namespaceSummaryManager =
+        (ReconNamespaceSummaryManagerImpl) cluster.getReconServer()
+            .getReconNamespaceSummaryManager();
+    OMDBInsightEndpoint omdbInsightEndpoint =
+        new OMDBInsightEndpoint(reconSCM, reconOmMetadataManagerInstance,
+            mock(GlobalStatsDao.class), namespaceSummaryManager);
+
+    // Delete the entire root directory dir1.
+    fs.delete(new Path("/dir1"), true);
+    impl.syncDataFromOM();
+
+    // Verify the entries in the Recon tables after sync.
+    assertTableRowCount(reconFileTable, 3, true);
+    assertTableRowCount(reconDirTable, 2, true);
+    assertTableRowCount(reconDeletedDirTable, 1, true);
+    Long deletedDirObjectID = null;
+
+    // Fetch the object id of the deleted directory dir1.
+    try (
+        TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+            iterator = reconDeletedDirTable.iterator()) {
+      Table.KeyValue<String, OmKeyInfo> entry = iterator.next();
+      deletedDirObjectID = entry.getValue().getObjectID();
+      Assert.assertEquals("dir1", entry.getValue().getKeyName());
+    }
+
+    // Retrieve the namespace summary for the deleted directory 'dir1'
+    // and verify that its size is 0 since it hasn't been calculated yet.
+    Assert.assertEquals(0,
+        namespaceSummaryManager.getNSSummary(deletedDirObjectID)
+            .getSizeOfFiles());
+
+    // Call the getDeletedDirInfo API to calculate the size of the deleted dir.
+    omdbInsightEndpoint.getDeletedDirInfo(-1, "");
+    omdbInsightEndpoint.getDeletedDirInfo(-1, "");
+
+    // Confirm the size as 3 when calling getDeletedDirInfo, as it's calculated
+    // and stored in the NSSummary instance.
+    Assert.assertEquals(3,
+        namespaceSummaryManager.getNSSummary(deletedDirObjectID)
+            .getSizeOfFiles());
+  }
+
   /**
    * Cleans up the tables by removing all entries from the deleted directory,
    * file, and directory tables within the Ozone metadata manager. This method
    * iterates through the tables and removes all entries from each table.
    */
+  @AfterEach
   private void cleanupTables() throws IOException {
     OMMetadataManager metadataManager =
         cluster.getOzoneManager().getMetadataManager();

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java

@@ -580,10 +580,23 @@ protected long fetchSizeForDeletedDirectory(long objectId)
     if (nsSummary == null) {
       return 0L;
     }
+    // Check if the deleted dir size is already computed and set previously.
+    if (nsSummary.getIsSizeOfDeletedDirectoryComputed()) {
+      return nsSummary.getSizeOfFiles();
+    }
+
+    // Initialize the total size.
     long totalSize = nsSummary.getSizeOfFiles();
+
+    // Compute the size recursively.
     for (long childId : nsSummary.getChildDir()) {
       totalSize += fetchSizeForDeletedDirectory(childId);
     }
+    // Update the size in the NSSummary and set sizeOfFilesSet to true to avoid
+    // re-computation the next time.
+    nsSummary.setSizeOfFiles(totalSize);
+    nsSummary.setIsSizeOfDeletedDirectoryComputed(true);
+    reconNamespaceSummaryManager.storeNSSummary(objectId, nsSummary);
     return totalSize;
   }
 

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java

@@ -36,10 +36,13 @@ public class NSSummary {
   private int[] fileSizeBucket;
   private Set<Long> childDir;
   private String dirName;
+  private boolean isSizeOfDeletedDirectoryComputed;
 
   public NSSummary() {
     this(0, 0L, new int[ReconConstants.NUM_OF_FILE_SIZE_BINS],
          new HashSet<>(), "");
+  // Avoids recalculating size for deleted directories, as it remains constant.
+    isSizeOfDeletedDirectoryComputed = false;
   }
 
   public NSSummary(int numOfFiles,
@@ -82,6 +85,15 @@ public void setSizeOfFiles(long sizeOfFiles) {
     this.sizeOfFiles = sizeOfFiles;
   }
 
+  public boolean getIsSizeOfDeletedDirectoryComputed() {
+    return isSizeOfDeletedDirectoryComputed;
+  }
+
+  public void setIsSizeOfDeletedDirectoryComputed(
+      boolean isSizeOfDeletedDirComputed) {
+    this.isSizeOfDeletedDirectoryComputed = isSizeOfDeletedDirComputed;
+  }
+
   public void setFileSizeBucket(int[] fileSizeBucket) {
     this.fileSizeBucket = Arrays.copyOf(fileSizeBucket,
             ReconConstants.NUM_OF_FILE_SIZE_BINS);

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java

@@ -67,7 +67,8 @@ public byte[] toPersistedFormat(NSSummary object) throws IOException {
     final int resSize = NUM_OF_INTS * Integer.BYTES
         + (numOfChildDirs + 1) * Long.BYTES // 1 long field + list size
         + Short.BYTES // 2 dummy shorts to track length
-        + stringLen; // directory name length
+        + stringLen // directory name length
+        + 1; // for the boolean field
 
     ByteArrayOutputStream out = new ByteArrayOutputStream(resSize);
     out.write(integerCodec.toPersistedFormat(object.getNumOfFiles()));
@@ -84,6 +85,8 @@ public byte[] toPersistedFormat(NSSummary object) throws IOException {
     }
     out.write(integerCodec.toPersistedFormat(stringLen));
     out.write(stringCodec.toPersistedFormat(dirName));
+    // Serialize the boolean field
+    out.write(object.getIsSizeOfDeletedDirectoryComputed() ? 1 : 0);
     return out.toByteArray();
   }
 
@@ -117,6 +120,11 @@ public NSSummary fromPersistedFormat(byte[] rawData) throws IOException {
     assert (bytesRead == strLen);
     String dirName = stringCodec.fromPersistedFormat(buffer);
     res.setDirName(dirName);
+
+    // Deserialize the boolean field
+    byte boolByte = in.readByte();
+    res.setIsSizeOfDeletedDirectoryComputed(boolByte != 0);
+
     return res;
   }
 
@@ -128,6 +136,8 @@ public NSSummary copyObject(NSSummary object) {
     copy.setFileSizeBucket(object.getFileSizeBucket());
     copy.setChildDir(object.getChildDir());
     copy.setDirName(object.getDirName());
+    copy.setIsSizeOfDeletedDirectoryComputed(
+        object.getIsSizeOfDeletedDirectoryComputed());
     return copy;
   }
 }