Skip to content

Commit

Permalink
import/get: escape non-pattern LFS path parts (#10416)
Browse files Browse the repository at this point in the history
  • Loading branch information
sisp authored May 13, 2024
1 parent 1848074 commit 5bfacec
Show file tree
Hide file tree
Showing 2 changed files with 80 additions and 1 deletion.
8 changes: 7 additions & 1 deletion dvc/fs/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import glob
from typing import Optional
from urllib.parse import urlparse

Expand Down Expand Up @@ -70,7 +71,12 @@ def download(

if isinstance(fs, DVCFileSystem):
lfs_prefetch(
fs, [f"{fs.normpath(fs_path)}/**" if fs.isdir(fs_path) else fs_path]
fs,
[
f"{fs.normpath(glob.escape(fs_path))}/**"
if fs.isdir(fs_path)
else glob.escape(fs_path)
],
)
cb.set_size(len(from_infos))
jobs = jobs or fs.jobs
Expand Down
73 changes: 73 additions & 0 deletions tests/func/test_download.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import os
from unittest.mock import ANY

import pytest

from dvc.fs import download

lfs_prefetch_params = [
pytest.param("abc", "abc", id="plain"),
pytest.param(
"*",
"[*]",
marks=pytest.mark.skipif(
os.name == "nt",
reason="forbidden character `*` on Windows filesystem",
),
id="escape-*",
),
pytest.param(
"**",
"[*][*]",
marks=pytest.mark.skipif(
os.name == "nt", reason="forbidden character `*` on Windows filesystem"
),
id="escape-**",
),
pytest.param(
"?",
"[?]",
marks=pytest.mark.skipif(
os.name == "nt", reason="forbidden character `?` on Windows filesystem"
),
id="escape-?",
),
pytest.param("[abc]", "[[]abc]", id="escape-[seq]"),
pytest.param("[!abc]", "[[]!abc]", id="escape-[!seq]"),
]


@pytest.mark.parametrize("dirname, include_name", lfs_prefetch_params)
def test_lfs_prefetch_directory(tmp_dir, dvc, scm, mocker, dirname, include_name):
mock_fetch = mocker.patch("scmrepo.git.lfs.fetch")
tmp_dir.scm_gen(
{
".gitattributes": "data/**/* filter=lfs diff=lfs merge=lfs -text",
f"data/{dirname}/test.txt": "test data",
},
commit="init lfs",
)
rev = scm.get_rev()
with dvc.switch(rev):
download(dvc.dvcfs, f"data/{dirname}", "data")
mock_fetch.assert_called_once_with(
scm, [rev], include=[f"/data/{include_name}/**"], progress=ANY
)


@pytest.mark.parametrize("basename, include_name", lfs_prefetch_params)
def test_lfs_prefetch_file(tmp_dir, dvc, scm, mocker, basename, include_name):
mock_fetch = mocker.patch("scmrepo.git.lfs.fetch")
tmp_dir.scm_gen(
{
".gitattributes": "data/**/* filter=lfs diff=lfs merge=lfs -text",
f"data/{basename}.txt": "test data",
},
commit="init lfs",
)
rev = scm.get_rev()
with dvc.switch(rev):
download(dvc.dvcfs, f"data/{basename}.txt", "data")
mock_fetch.assert_called_once_with(
scm, [rev], include=[f"/data/{include_name}.txt"], progress=ANY
)

0 comments on commit 5bfacec

Please sign in to comment.