diff --git a/autotest/gcore/vsiaz.py b/autotest/gcore/vsiaz.py index 348f794b44dd..d2b094453148 100755 --- a/autotest/gcore/vsiaz.py +++ b/autotest/gcore/vsiaz.py @@ -381,6 +381,109 @@ def test_vsiaz_fake_readdir(): assert gdal.VSIStatL("/vsiaz/mycontainer1", gdal.VSI_STAT_CACHE_ONLY) is not None +############################################################################### +# Test ReadDir() when first response has no blobs but a non-empty NextMarker + + +def test_vsiaz_fake_readdir_no_blobs_in_first_request(): + + if gdaltest.webserver_port == 0: + pytest.skip() + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%20with_space%2F&restype=container", + 200, + {"Content-type": "application/xml"}, + """ + + a_dir with_space/ + + bla + + """, + ) + handler.add( + "GET", + "/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%20with_space%2F&restype=container", + 200, + {"Content-type": "application/xml"}, + """ + + a_dir with_space/ + + + a_dir with_space/resource4.bin + + 16 Oct 2016 12:34:56 + 456789 + + + + a_dir with_space/subdir/ + + + + """, + ) + + with webserver.install_http_handler(handler): + dir_contents = gdal.ReadDir("/vsiaz/az_fake_bucket2/a_dir with_space") + assert dir_contents == ["resource4.bin", "subdir"] + + +############################################################################### +# + + +@gdaltest.enable_exceptions() +def test_vsiaz_fake_readdir_protection_again_infinite_looping(): + + if gdaltest.webserver_port == 0: + pytest.skip() + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%20with_space%2F&restype=container", + 200, + {"Content-type": "application/xml"}, + """ + + a_dir with_space/ + + bla0 + + """, + ) + for i in range(10): + handler.add( + "GET", + f"/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla{i}&prefix=a_dir%20with_space%2F&restype=container", + 200, + {"Content-type": "application/xml"}, + f""" + + a_dir with_space/ + + bla{i+1} + + """, + ) + + with webserver.install_http_handler(handler): + with pytest.raises( + Exception, + match="More than 10 consecutive List Blob requests returning no blobs", + ): + gdal.ReadDir("/vsiaz/az_fake_bucket2/a_dir with_space") + + ############################################################################### # Test AZURE_STORAGE_SAS_TOKEN option with fake server diff --git a/port/cpl_vsil_az.cpp b/port/cpl_vsil_az.cpp index a4cc28b39e07..1b115a9430a3 100644 --- a/port/cpl_vsil_az.cpp +++ b/port/cpl_vsil_az.cpp @@ -358,6 +358,11 @@ bool VSIDIRAz::AnalyseAzureFileList(const std::string &osBaseURL, } osNextMarker = CPLGetXMLValue(psEnumerationResults, "NextMarker", ""); + // For some containers, a list blob request can return a response + // with no blobs, but with a non-empty NextMarker, and the following + // request using that marker will return blobs... + if (!osNextMarker.empty()) + bOK = true; } CPLDestroyXMLNode(psTree); @@ -460,7 +465,8 @@ bool VSIDIRAz::IssueListDir() const VSIDIREntry *VSIDIRAz::NextDirEntry() { - while (true) + constexpr int ARBITRARY_LIMIT = 10; + for (int i = 0; i < ARBITRARY_LIMIT; ++i) { if (nPos < static_cast(aoEntries.size())) { @@ -477,6 +483,11 @@ const VSIDIREntry *VSIDIRAz::NextDirEntry() return nullptr; } } + CPLError(CE_Failure, CPLE_AppDefined, + "More than %d consecutive List Blob " + "requests returning no blobs", + ARBITRARY_LIMIT); + return nullptr; } /************************************************************************/