Unnest functionality for Druid

processing/src/main/java/org/apache/druid/query/UnnestDataSource.java

@@ -35,6 +35,15 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 
+/**
+ * The data source for representing an unnest operation.
+ *
+ * An unnest data source has the following:
+ * a base data source which is to be unnested
+ * the column name of the MVD which will be unnested
+ * the name of the column that will hold the unnested values
+ * and an allowlist serving as a filter of which values in the MVD will be unnested.
+ */
 public class UnnestDataSource implements DataSource
 {
   private final DataSource base;

processing/src/main/java/org/apache/druid/segment/ColumnarValueUnnestCursor.java

@@ -30,6 +30,31 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 
+/**
+ * The cursor to help unnest MVDs without dictionary encoding.
+ * Consider a segment has 2 rows
+ * ['a', 'b', 'c']
+ * ['d', 'e']
+ *
+ * The baseCursor points to the row ['a', 'b', 'c']
+ * while the unnestCursor with each call of advance() moves over individual elements.
+ *
+ * unnestCursor.advance() -> 'a'
+ * unnestCursor.advance() -> 'b'
+ * unnestCursor.advance() -> 'c'
+ * unnestCursor.advance() -> 'd' (advances base cursor first)
+ * unnestCursor.advance() -> 'e'
+ *
+ *
+ * The allowSet if available helps skip over elements which are not in the allowList by moving the cursor to
+ * the next available match.
+ *
+ * The index reference points to the index of each row that the unnest cursor is accessing through currentVal
+ * The index ranges from 0 to the size of the list in each row which is held in the unnestListForCurrentRow
+ *
+ * The needInitialization flag sets up the initial values of unnestListForCurrentRow at the beginning of the segment
+ *
+ */
 public class ColumnarValueUnnestCursor implements Cursor
 {
   private final Cursor baseCursor;
@@ -69,7 +94,11 @@ public ColumnSelectorFactory getColumnSelectorFactory()
       @Override
       public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec)
       {
-        throw new UnsupportedOperationException("Dimension selector not applicable for column value selector");
+        if (!outputName.equals(dimensionSpec.getDimension())) {
+          return baseColumSelectorFactory.makeDimensionSelector(dimensionSpec);
+        }
+        throw new UnsupportedOperationException(
+            "Dimension selector not applicable for column value selector for column " + outputName);
       }
 
       @Override
@@ -203,6 +232,11 @@ public void reset()
     baseCursor.reset();
   }
 
+  /**
+   * This method populates the objects when the base cursor moves to the next row
+   *
+   * @param firstRun flag to populate one time object references to hold values for unnest cursor
+   */
   private void getNextRow(boolean firstRun)
   {
     currentVal = this.columnValueSelector.getObject();

processing/src/main/java/org/apache/druid/segment/DimensionUnnestCursor.java

@@ -33,6 +33,46 @@
 import java.util.BitSet;
 import java.util.LinkedHashSet;
 
+/**
+ * The cursor to help unnest MVDs with dictionary encoding.
+ * Consider a segment has 2 rows
+ * ['a', 'b', 'c']
+ * ['d', 'c']
+ *
+ * Considering dictionary encoding, these are represented as
+ *
+ * 'a' -> 0
+ * 'b' -> 1
+ * 'c' -> 2
+ * 'd' -> 3
+ *
+ * The baseCursor points to the row of IndexedInts [0, 1, 2]
+ * while the unnestCursor with each call of advance() moves over individual elements.
+ *
+ * advance() -> 0 -> 'a'
+ * advance() -> 1 -> 'b'
+ * advance() -> 2 -> 'c'
+ * advance() -> 3 -> 'd' (advances base cursor first)
+ * advance() -> 2 -> 'c'
+ *
+ * Total 5 advance calls above
+ *
+ * The allowSet, if available, helps skip over elements that are not in the allowList by moving the cursor to
+ * the next available match. The hashSet is converted into a bitset (during initialization) for efficiency.
+ * If allowSet is ['c', 'd'] then the advance moves over to the next available match
+ *
+ * advance() -> 2 -> 'c'
+ * advance() -> 3 -> 'd' (advances base cursor first)
+ * advance() -> 2 -> 'c'
+ *
+ * Total 3 advance calls in this case
+ *
+ * The index reference points to the index of each row that the unnest cursor is accessing
+ * The indexedInts for each row are held in the indexedIntsForCurrentRow object
+ *
+ * The needInitialization flag sets up the initial values of indexedIntsForCurrentRow at the beginning of the segment
+ *
+ */
 public class DimensionUnnestCursor implements Cursor
 {
   private final Cursor baseCursor;
@@ -188,6 +228,9 @@ public IdLookup idLookup()
         };
       }
 
+      /*
+      This ideally should not be called. If called delegate using the makeDimensionSelector
+       */
       @Override
       public ColumnValueSelector makeColumnValueSelector(String columnName)
       {

processing/src/main/java/org/apache/druid/segment/UnnestSegmentReference.java

@@ -31,6 +31,10 @@
 import java.util.LinkedHashSet;
 import java.util.Optional;
 
+/**
+ * The segment reference for the Unnest Data Source.
+ * The input column name, output name and the allowSet follow from {@link org.apache.druid.query.UnnestDataSource}
+ */
 public class UnnestSegmentReference implements SegmentReference
 {
   private static final Logger log = new Logger(UnnestSegmentReference.class);

processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java

@@ -38,6 +38,11 @@
 import java.util.LinkedHashSet;
 import java.util.Objects;
 
+/**
+ * This class serves as the Storage Adapter for the Unnest Segment and is responsible for creating the cursors
+ * If the column is dictionary encoded it creates {@link DimensionUnnestCursor} else {@link ColumnarValueUnnestCursor}
+ * These cursors help navigate the segments for these cases
+ */
 public class UnnestStorageAdapter implements StorageAdapter
 {
   private final StorageAdapter baseAdapter;

processing/src/test/java/org/apache/druid/segment/ListCursor.java

@@ -31,6 +31,14 @@
 import javax.annotation.Nullable;
 import java.util.List;
 
+
+/**
+ * A Cursor that iterates over a user created list.
+ * This is used to test the base cursor of an UnnestCursor.
+ * Usages can be found in tests of {@link ColumnarValueUnnestCursor} in {@link ColumnarValueUnnestCursorTest}
+ * However this cannot help with {@link DimensionUnnestCursor}.
+ * Tests for {@link DimensionUnnestCursor} are done alongside tests for {@link UnnestStorageAdapterTest}
+ */
 public class ListCursor implements Cursor
 {
   List<Object> baseList;