Merge pull request #151 from achaikou/layout_tests

Layout tests

lib/src/io.cpp

@@ -83,7 +83,7 @@ long long findvrl( mio::mmap_source& file, long long from ) noexcept (false) {
 
         case DLIS_NOTFOUND: {
             const auto msg = "searched {} bytes, but could not find "
-                             "visbile record envelope pattern [0xFF 0x01]"
+                             "visible record envelope pattern [0xFF 0x01]"
             ;
             throw dl::not_found(fmt::format(msg, search_limit));
         }
@@ -449,9 +449,9 @@ findfdata(mio::mmap_source& file,
 noexcept (false) {
 
     const auto* ptr = file.data();
+    const auto* end = file.data() + file.size();
     std::vector< std::pair< std::string, int > > xs;
 
-    char fingerprint[280] = {};
     char name[256] = {};
 
     for (auto i : candidates) {
@@ -467,11 +467,17 @@ noexcept (false) {
         std::int32_t origin;
         std::uint8_t copy;
         std::int32_t idlen;
-        dlis_obname(ptr + tell + offset, &origin, &copy, &idlen, name);
-        std::memset(fingerprint, 0, sizeof(fingerprint));
-        dlis_object_fingerprint(5, "FRAME", idlen, name, origin, copy, fingerprint);
+        auto cur = dlis_obname(ptr + tell + offset, &origin, &copy, &idlen, name);
+        if (std::distance( cur, end ) < 0)
+        {
+            auto msg = "File corrupted. Error on reading fdata obname";
+            throw std::runtime_error(msg);
+        }
 
-        xs.emplace_back(std::string(fingerprint), i);
+        dl::obname tmp{ dl::origin{ origin },
+                        dl::ushort{ copy },
+                        dl::ident{ std::string{ name, name + idlen } } };
+        xs.emplace_back(tmp.fingerprint("FRAME"), i);
     }
 
     return xs;

python/data/layout/nondlis.txt

@@ -0,0 +1 @@
+Lets pretend as if it was a DLIS file, even though it is not. 

python/tests/__init__.py

@@ -8,11 +8,6 @@ def DWL206():
     with dlisio.load('data/206_05a-_3_DWL_DWL_WIRE_258276498.DLIS') as (f,):
         yield f
 
-@pytest.fixture(scope="module", name="only_channels")
-def only_channels():
-    with dlisio.load('data/only-channels.dlis') as (f,):
-        yield f
-
 @pytest.fixture(scope="module")
 def merge_files():
     def merge(fpath, flist, lrs_offset = None):

python/tests/test_core.py

@@ -4,64 +4,7 @@
 
 import dlisio
 
-from . import DWL206, only_channels
-
-def test_sul(DWL206):
-    label = ''.join([
-                '   1',
-                'V1.00',
-                'RECORD',
-                ' 8192',
-                'Default Storage Set                                         ',
-            ])
-
-    sul = dlisio.core.storage_label(label.encode('ascii'))
-    d = {
-        'sequence': 1,
-        'version': '1.0',
-        'maxlen': 8192,
-        'layout': 'record',
-        'id': 'Default Storage Set                                         ',
-    }
-
-    assert sul == d
-
-    assert DWL206.storage_label() == d
-
-
-def test_sul_error_values():
-    label = "too short"
-    with pytest.raises(ValueError) as excinfo:
-       dlisio.core.storage_label(label.encode('ascii'))
-    assert 'buffer to small' in str(excinfo.value)
-
-    label = ''.join([
-                '   1',
-                'V2.00',
-                'RECORD',
-                ' 8192',
-                'Default Storage Set                                         ',
-            ])
-
-    with pytest.raises(ValueError) as excinfo:
-        dlisio.core.storage_label(label.encode('ascii'))
-    assert 'unable to parse' in str(excinfo.value)
-
-
-    label = ''.join([
-                '  2 ',
-                'V1.00',
-                'TRASH1',
-                'ZZZZZ',
-                'Default Storage Set                                         ',
-    ])
-
-    with pytest.warns(RuntimeWarning) as warninfo:
-        sul = dlisio.core.storage_label(label.encode('ascii'))
-    assert len(warninfo) == 1
-    assert "label inconsistent" in warninfo[0].message.args[0]
-    assert sul['layout'] == 'unknown'
-
+from . import DWL206
 
 # The example record from the specification
 stdrecord = bytearray([
@@ -423,55 +366,6 @@ def test_dtype(DWL206):
                                ('TENS_SL', np.float32),
                                ('DEPT_SL', np.float32)])
 
-def test_load_pre_sul_garbage(only_channels):
-    with dlisio.load('data/pre-sul-garbage.dlis') as (f,):
-        assert f.storage_label() == f.storage_label()
-        assert f.sul_offset == 12
-
-def test_load_pre_vrl_garbage(only_channels):
-    with dlisio.load('data/pre-sul-pre-vrl-garbage.dlis') as (f,):
-        assert f.storage_label() == f.storage_label()
-        assert f.sul_offset == 12
-
-def test_load_file_with_broken_utf8():
-    with dlisio.load('data/broken-degree-symbol.dlis') as (f, *tail):
-        pass
-
-def test_padbytes_as_large_as_record():
-    # 180-byte long explicit record with padding, and padbytes are set to 180
-    # (leaving the resulting len(data) == 0)
-    try:
-        f = dlisio.open('data/padbytes-large-as-record.dlis')
-        f.reindex([0], [180])
-
-        rec = f.extract([0])[0]
-        assert rec.explicit
-        assert len(memoryview(rec)) == 0
-    finally:
-        f.close()
-
-def test_padbytes_as_large_as_segment():
-    # 180-byte long explicit record with padding, and padbytes are set to 176
-    # record is expected to not be present
-    try:
-        f = dlisio.open('data/padbytes-large-as-segment-body.dlis')
-        f.reindex([0], [180])
-
-        rec = f.extract([0])[0]
-        assert len(memoryview(rec)) == 0
-    finally:
-        f.close()
-
-def test_load_small_file():
-    # <4K files infinite loop bug check
-    with dlisio.load('data/example-record.dlis'):
-        pass
-
-def test_load_7K_file_with_several_LR():
-    # 4K-8K files infinite loop bug check
-    with dlisio.load('data/7K-file.dlis'):
-        pass
-
 def test_record_attributes():
     stream = dlisio.open('data/3-syntactic-logical-records.dlis')
     tells = [80, 116, 148]

python/tests/test_layout.py

@@ -0,0 +1,173 @@
+import dlisio
+import pytest
+from . import DWL206
+
+def test_sul(DWL206):
+    label = ''.join([
+                '   1',
+                'V1.00',
+                'RECORD',
+                ' 8192',
+                'Default Storage Set                                         ',
+            ])
+
+    sul = dlisio.core.storage_label(label.encode('ascii'))
+    d = {
+        'sequence': 1,
+        'version': '1.0',
+        'maxlen': 8192,
+        'layout': 'record',
+        'id': 'Default Storage Set                                         ',
+    }
+
+    assert sul == d
+
+    assert DWL206.storage_label() == d
+
+
+def test_sul_error_values():
+    label = "too short"
+    with pytest.raises(ValueError) as excinfo:
+       dlisio.core.storage_label(label.encode('ascii'))
+    assert 'buffer to small' in str(excinfo.value)
+
+    label = ''.join([
+                '   1',
+                'V2.00',
+                'RECORD',
+                ' 8192',
+                'Default Storage Set                                         ',
+            ])
+
+    with pytest.raises(ValueError) as excinfo:
+        dlisio.core.storage_label(label.encode('ascii'))
+    assert 'unable to parse' in str(excinfo.value)
+
+
+    label = ''.join([
+                '  2 ',
+                'V1.00',
+                'TRASH1',
+                'ZZZZZ',
+                'Default Storage Set                                         ',
+    ])
+
+    with pytest.warns(RuntimeWarning) as warninfo:
+        sul = dlisio.core.storage_label(label.encode('ascii'))
+    assert len(warninfo) == 1
+    assert "label inconsistent" in warninfo[0].message.args[0]
+    assert sul['layout'] == 'unknown'
+
+def test_load_pre_sul_garbage():
+    with dlisio.load('data/layout/pre-sul-garbage.dlis') as (f,):
+        assert f.storage_label() == f.storage_label()
+        assert f.sul_offset == 12
+
+def test_notdlis():
+    with pytest.raises(RuntimeError) as excinfo:
+        _ = dlisio.load('data/layout/nondlis.txt')
+    assert "could not find storage label" in str(excinfo.value)
+
+def test_broken_sul():
+    with pytest.raises(RuntimeError) as excinfo:
+        _ = dlisio.load('data/layout/incomplete-sul.dlis')
+    assert "file may be corrupted" in str(excinfo.value)
+
+def test_load_pre_vrl_garbage():
+    with dlisio.load('data/layout/pre-sul-pre-vrl-garbage.dlis') as (f,):
+        assert f.storage_label() == f.storage_label()
+        assert f.sul_offset == 12
+
+def test_old_vrs():
+    with pytest.raises(RuntimeError) as excinfo:
+        _ = dlisio.load('data/layout/old-vr.dlis')
+    assert "could not find visible record" in str(excinfo.value)
+
+def test_broken_vr():
+    with pytest.raises(RuntimeError) as excinfo:
+        _ = dlisio.load('data/layout/incomplete-vr.dlis')
+    assert "file may be corrupted" in str(excinfo.value)
+
+def test_load_small_file():
+    # <4K files infinite loop bug check
+    with dlisio.load('data/layout/small.dlis'):
+        pass
+
+def test_load_7K_file_with_several_LR():
+    # 4K-8K files infinite loop bug check
+    with dlisio.load('data/layout/7K-file.dlis'):
+        pass
+
+def test_truncated():
+    with pytest.raises(RuntimeError) as excinfo:
+        _ = dlisio.load('data/layout/truncated.dlis')
+    assert "file truncated" in str(excinfo.value)
+
+def test_too_small_record():
+    with pytest.raises(RuntimeError) as excinfo:
+        _ = dlisio.load('data/layout/too-small-record.dlis')
+    assert "in record 0 corrupted" in str(excinfo.value)
+
+def test_padbytes_as_large_as_record():
+    # 180-byte long explicit record with padding, and padbytes are set to 180
+    # (leaving the resulting len(data) == 0)
+    f = dlisio.open('data/layout/padbytes-large-as-record.dlis')
+    try:
+        f.reindex([0], [180])
+
+        rec = f.extract([0])[0]
+        assert rec.explicit
+        assert len(memoryview(rec)) == 0
+    finally:
+        f.close()
+
+def test_padbytes_as_large_as_segment():
+    # 180-byte long explicit record with padding, and padbytes are set to 176
+    # record is expected to not be present
+    f = dlisio.open('data/layout/padbytes-large-as-segment-body.dlis')
+    try:
+        f.reindex([0], [180])
+
+        rec = f.extract([0])[0]
+        assert len(memoryview(rec)) == 0
+    finally:
+        f.close()
+
+def test_bad_padbytes():
+    with pytest.raises(RuntimeError) as excinfo:
+        _ = dlisio.load('data/layout/padbytes-bad.dlis')
+    assert "bad segment trim" in str(excinfo.value)
+
+def test_padbytes_encrypted():
+    with dlisio.load('data/layout/padbytes-encrypted.dlis'):
+        pass
+
+def test_load_fdata_VR_aligned():
+    with dlisio.load('data/layout/fdata-vr-aligned.dlis') as (f, *_):
+        assert len(f.fdata_index) == 1
+        assert f.fdata_index['T.FRAME-I.DLIS-FRAME-O.3-C.1'] == [0]
+
+def test_load_fdata_many_in_same_VR():
+    with dlisio.load('data/layout/fdata-many-in-same-vr.dlis') as (f, *_):
+        assert len(f.fdata_index) == 2
+        assert f.fdata_index['T.FRAME-I.DLIS-FRAME-O.3-C.1'] == [0, 1]
+        ident = '3'*255
+        fingerprint = 'T.FRAME-I.'+ident+'-O.1073741823-C.255'
+        assert f.fdata_index[fingerprint] == [3]
+
+def test_3lrs_in_lr_in_vr():
+    with dlisio.load('data/layout/example-record.dlis'):
+        pass
+
+def test_2_lr_in_vr():
+    with dlisio.load('data/layout/2lr-in-vr.dlis'):
+        pass
+
+def test_lr_in_2vrs():
+    with dlisio.load('data/layout/lr-in-2vrs.dlis'):
+        pass
+
+def test_vrl_and_lrsh_mismatch():
+    with pytest.raises(RuntimeError) as excinfo:
+        _ = dlisio.load('data/layout/wrong-lrhs.dlis')
+    assert "visible record/segment inconsistency" in str(excinfo.value)

python/tests/test_parse.py

@@ -8,8 +8,8 @@
 
 @pytest.fixture
 def merge(merge_files):
-    def merge(path, content):
-        merge_files(path, content, 104)
+    def merge(path, content, lrs_offset=104):
+        merge_files(path, content, lrs_offset)
     return merge
 
 @pytest.mark.future_test_attributes
@@ -598,3 +598,25 @@ def test_cut_middle_template_attribute(tmpdir, merge):
     with pytest.raises(IndexError) as excinfo:
         dlisio.load(path)
     assert "unexpected end-of-record" in str(excinfo.value)
+
+def test_value_broken_utf8(tmpdir, merge):
+    path = os.path.join(str(tmpdir), 'value_broken_utf8.dlis')
+    content = [
+        'data/parse/start.dlis.part',
+        'data/parse/template/broken-utf8.dlis.part',
+    ]
+    merge(path, content)
+    with dlisio.load(path):
+        pass
+
+def test_fdata_bad_ident(tmpdir, merge):
+    path = os.path.join(str(tmpdir), 'fdata_bad_ident.dlis')
+    content = [
+        'data/parse/sul.dlis.part',
+        'data/parse/implicit/fdata-bad-ident.dlis.part',
+    ]
+    merge(path, content, lrs_offset=None)
+    with pytest.raises(RuntimeError) as excinfo:
+        dlisio.load(path)
+
+    assert "fdata obname" in str(excinfo.value)