Merge pull request #11323 from rouault/vsiaz_list_dir_no_blob_but_marker

/vsiaz/: ReadDir(): be robust to a response to list blob that returns…
OSGeo · Nov 26, 2024 · ade9d67 · ade9d67
2 parents b58e76c + 32bd86f
commit ade9d67
Show file tree

Hide file tree

Showing 2 changed files with 115 additions and 1 deletion.
diff --git a/autotest/gcore/vsiaz.py b/autotest/gcore/vsiaz.py
@@ -381,6 +381,109 @@ def test_vsiaz_fake_readdir():
     assert gdal.VSIStatL("/vsiaz/mycontainer1", gdal.VSI_STAT_CACHE_ONLY) is not None
 
 
+###############################################################################
+# Test ReadDir() when first response has no blobs but a non-empty NextMarker
+
+
+def test_vsiaz_fake_readdir_no_blobs_in_first_request():
+
+    if gdaltest.webserver_port == 0:
+        pytest.skip()
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add(
+        "GET",
+        "/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%20with_space%2F&restype=container",
+        200,
+        {"Content-type": "application/xml"},
+        """<?xml version="1.0" encoding="UTF-8"?>
+                    <EnumerationResults>
+                        <Prefix>a_dir with_space/</Prefix>
+                        <Blobs/>
+                        <NextMarker>bla</NextMarker>
+                    </EnumerationResults>
+                """,
+    )
+    handler.add(
+        "GET",
+        "/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%20with_space%2F&restype=container",
+        200,
+        {"Content-type": "application/xml"},
+        """<?xml version="1.0" encoding="UTF-8"?>
+                    <EnumerationResults>
+                        <Prefix>a_dir with_space/</Prefix>
+                        <Blobs>
+                          <Blob>
+                            <Name>a_dir with_space/resource4.bin</Name>
+                            <Properties>
+                              <Last-Modified>16 Oct 2016 12:34:56</Last-Modified>
+                              <Content-Length>456789</Content-Length>
+                            </Properties>
+                          </Blob>
+                          <BlobPrefix>
+                            <Name>a_dir with_space/subdir/</Name>
+                          </BlobPrefix>
+                        </Blobs>
+                    </EnumerationResults>
+                """,
+    )
+
+    with webserver.install_http_handler(handler):
+        dir_contents = gdal.ReadDir("/vsiaz/az_fake_bucket2/a_dir with_space")
+    assert dir_contents == ["resource4.bin", "subdir"]
+
+
+###############################################################################
+#
+
+
+@gdaltest.enable_exceptions()
+def test_vsiaz_fake_readdir_protection_again_infinite_looping():
+
+    if gdaltest.webserver_port == 0:
+        pytest.skip()
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add(
+        "GET",
+        "/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%20with_space%2F&restype=container",
+        200,
+        {"Content-type": "application/xml"},
+        """<?xml version="1.0" encoding="UTF-8"?>
+                    <EnumerationResults>
+                        <Prefix>a_dir with_space/</Prefix>
+                        <Blobs/>
+                        <NextMarker>bla0</NextMarker>
+                    </EnumerationResults>
+                """,
+    )
+    for i in range(10):
+        handler.add(
+            "GET",
+            f"/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla{i}&prefix=a_dir%20with_space%2F&restype=container",
+            200,
+            {"Content-type": "application/xml"},
+            f"""<?xml version="1.0" encoding="UTF-8"?>
+                        <EnumerationResults>
+                            <Prefix>a_dir with_space/</Prefix>
+                            <Blobs/>
+                            <NextMarker>bla{i+1}</NextMarker>
+                        </EnumerationResults>
+                    """,
+        )
+
+    with webserver.install_http_handler(handler):
+        with pytest.raises(
+            Exception,
+            match="More than 10 consecutive List Blob requests returning no blobs",
+        ):
+            gdal.ReadDir("/vsiaz/az_fake_bucket2/a_dir with_space")
+
+
 ###############################################################################
 # Test AZURE_STORAGE_SAS_TOKEN option with fake server
 

diff --git a/port/cpl_vsil_az.cpp b/port/cpl_vsil_az.cpp
@@ -358,6 +358,11 @@ bool VSIDIRAz::AnalyseAzureFileList(const std::string &osBaseURL,
         }
 
         osNextMarker = CPLGetXMLValue(psEnumerationResults, "NextMarker", "");
+        // For some containers, a list blob request can return a response
+        // with no blobs, but with a non-empty NextMarker, and the following
+        // request using that marker will return blobs...
+        if (!osNextMarker.empty())
+            bOK = true;
     }
     CPLDestroyXMLNode(psTree);
 
@@ -460,7 +465,8 @@ bool VSIDIRAz::IssueListDir()
 
 const VSIDIREntry *VSIDIRAz::NextDirEntry()
 {
-    while (true)
+    constexpr int ARBITRARY_LIMIT = 10;
+    for (int i = 0; i < ARBITRARY_LIMIT; ++i)
     {
         if (nPos < static_cast<int>(aoEntries.size()))
         {
@@ -477,6 +483,11 @@ const VSIDIREntry *VSIDIRAz::NextDirEntry()
             return nullptr;
         }
     }
+    CPLError(CE_Failure, CPLE_AppDefined,
+             "More than %d consecutive List Blob "
+             "requests returning no blobs",
+             ARBITRARY_LIMIT);
+    return nullptr;
 }
 
 /************************************************************************/