Error: DatasetGenerationError: An error occurred while generating the dataset

[hissain]

What I have done?

• I have cloned the project in my Macbook Pro and using Jupyter Notebook, I have run the file https://github.com/hkproj/pytorch-transformer/blob/main/Local_Train.ipynb

How to solve the code that did produce the following error:

Error:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
File ~/anaconda3/lib/python3.11/site-packages/datasets/builder.py:1873, in ArrowBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, job_id)
   1866     writer = writer_class(
   1867         features=writer._features,
   1868         path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
   (...)
   1871         embed_local_files=embed_local_files,
   1872     )
-> 1873 writer.write_table(table)
   1874 num_examples_progress_update += len(table)

File ~/anaconda3/lib/python3.11/site-packages/datasets/arrow_writer.py:568, in ArrowWriter.write_table(self, pa_table, writer_batch_size)
    567 pa_table = pa_table.combine_chunks()
--> 568 pa_table = table_cast(pa_table, self._schema)
    569 if self.embed_local_files:

File ~/anaconda3/lib/python3.11/site-packages/datasets/table.py:2290, in table_cast(table, schema)
   2289 if table.schema != schema:
-> 2290     return cast_table_to_schema(table, schema)
   2291 elif table.schema.metadata != schema.metadata:

File ~/anaconda3/lib/python3.11/site-packages/datasets/table.py:2249, in cast_table_to_schema(table, schema)
   2248     raise ValueError(f"Couldn't cast\n{table.schema}\nto\n{features}\nbecause column names don't match")
-> 2249 arrays = [cast_array_to_feature(table[name], feature) for name, feature in features.items()]
   2250 return pa.Table.from_arrays(arrays, schema=schema)

File ~/anaconda3/lib/python3.11/site-packages/datasets/table.py:2249, in <listcomp>(.0)
   2248     raise ValueError(f"Couldn't cast\n{table.schema}\nto\n{features}\nbecause column names don't match")
-> 2249 arrays = [cast_array_to_feature(table[name], feature) for name, feature in features.items()]
   2250 return pa.Table.from_arrays(arrays, schema=schema)

File ~/anaconda3/lib/python3.11/site-packages/datasets/table.py:1817, in _wrap_for_chunked_arrays.<locals>.wrapper(array, *args, **kwargs)
   1816 if isinstance(array, pa.ChunkedArray):
-> 1817     return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks])
   1818 else:

File ~/anaconda3/lib/python3.11/site-packages/datasets/table.py:1817, in <listcomp>(.0)
   1816 if isinstance(array, pa.ChunkedArray):
-> 1817     return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks])
   1818 else:

File ~/anaconda3/lib/python3.11/site-packages/datasets/table.py:2109, in cast_array_to_feature(array, feature, allow_number_to_str)
   2108 elif not isinstance(feature, (Sequence, dict, list, tuple)):
-> 2109     return array_cast(array, feature(), allow_number_to_str=allow_number_to_str)
   2110 raise TypeError(f"Couldn't cast array of type\n{array.type}\nto\n{feature}")

File ~/anaconda3/lib/python3.11/site-packages/datasets/table.py:1819, in _wrap_for_chunked_arrays.<locals>.wrapper(array, *args, **kwargs)
   1818 else:
-> 1819     return func(array, *args, **kwargs)

File ~/anaconda3/lib/python3.11/site-packages/datasets/table.py:2000, in array_cast(array, pa_type, allow_number_to_str)
   1999     return array.cast(pa_type)
-> 2000 raise TypeError(f"Couldn't cast array of type\n{array.type}\nto\n{pa_type}")

TypeError: Couldn't cast array of type
struct<ca: string, en: string>
to
struct<ca: string, de: string>

The above exception was the direct cause of the following exception:

DatasetGenerationError                    Traceback (most recent call last)
Cell In[1], line 9
      5 cfg['num_epochs'] = 1
      7 from train import train_model
----> 9 train_model(cfg)

File ~/git/github/pytorch-transformer/train.py:198, in train_model(config)
    195 # Make sure the weights folder exists
    196 Path(f"{config['datasource']}_{config['model_folder']}").mkdir(parents=True, exist_ok=True)
--> 198 train_dataloader, val_dataloader, tokenizer_src, tokenizer_tgt = get_ds(config)
    199 model = get_model(config, tokenizer_src.get_vocab_size(), tokenizer_tgt.get_vocab_size()).to(device)
    200 # Tensorboard

File ~/git/github/pytorch-transformer/train.py:143, in get_ds(config)
    141 def get_ds(config):
    142     # It only has the train split, so we divide it overselves
--> 143     ds_raw = load_dataset(f"{config['datasource']}", f"{config['lang_src']}-{config['lang_tgt']}", split='train')
    145     # Build tokenizers
    146     tokenizer_src = get_or_build_tokenizer(config, ds_raw, config['lang_src'])

File ~/anaconda3/lib/python3.11/site-packages/datasets/load.py:1797, in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, verification_mode, ignore_verifications, keep_in_memory, save_infos, revision, use_auth_token, task, streaming, num_proc, storage_options, **config_kwargs)
   1794 try_from_hf_gcs = path not in _PACKAGED_DATASETS_MODULES
   1796 # Download and prepare data
-> 1797 builder_instance.download_and_prepare(
   1798     download_config=download_config,
   1799     download_mode=download_mode,
   1800     verification_mode=verification_mode,
   1801     try_from_hf_gcs=try_from_hf_gcs,
   1802     num_proc=num_proc,
   1803     storage_options=storage_options,
   1804 )
   1806 # Build dataset for splits
   1807 keep_in_memory = (
   1808     keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size)
   1809 )

File ~/anaconda3/lib/python3.11/site-packages/datasets/builder.py:890, in DatasetBuilder.download_and_prepare(self, output_dir, download_config, download_mode, verification_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, file_format, max_shard_size, num_proc, storage_options, **download_and_prepare_kwargs)
    888     if num_proc is not None:
    889         prepare_split_kwargs["num_proc"] = num_proc
--> 890     self._download_and_prepare(
    891         dl_manager=dl_manager,
    892         verification_mode=verification_mode,
    893         **prepare_split_kwargs,
    894         **download_and_prepare_kwargs,
    895     )
    896 # Sync info
    897 self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())

File ~/anaconda3/lib/python3.11/site-packages/datasets/builder.py:985, in DatasetBuilder._download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs)
    981 split_dict.add(split_generator.split_info)
    983 try:
    984     # Prepare split will record examples associated to the split
--> 985     self._prepare_split(split_generator, **prepare_split_kwargs)
    986 except OSError as e:
    987     raise OSError(
    988         "Cannot find data file. "
    989         + (self.manual_download_instructions or "")
    990         + "\nOriginal error:\n"
    991         + str(e)
    992     ) from None

File ~/anaconda3/lib/python3.11/site-packages/datasets/builder.py:1746, in ArrowBasedBuilder._prepare_split(self, split_generator, file_format, num_proc, max_shard_size)
   1744 job_id = 0
   1745 with pbar:
-> 1746     for job_id, done, content in self._prepare_split_single(
   1747         gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
   1748     ):
   1749         if done:
   1750             result = content

File ~/anaconda3/lib/python3.11/site-packages/datasets/builder.py:1891, in ArrowBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, job_id)
   1889     if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
   1890         e = e.__context__
-> 1891     raise DatasetGenerationError("An error occurred while generating the dataset") from e
   1893 yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths)

DatasetGenerationError: An error occurred while generating the dataset

My '!pip list' output in Jypyter Notebook

----------------------------- ------------
absl-py                       2.0.0
accelerate                    0.28.0
aiobotocore                   2.5.0
aiofiles                      22.1.0
aiohttp                       3.8.5
aioitertools                  0.7.1
aiosignal                     1.2.0
aiosqlite                     0.18.0
alabaster                     0.7.12
alembic                       1.13.1
anaconda-anon-usage           0.4.2
anaconda-catalogs             0.2.0
anaconda-client               1.12.1
anaconda-cloud-auth           0.1.3
anaconda-navigator            2.5.0
anaconda-project              0.11.1
anyio                         3.5.0
appdirs                       1.4.4
applaunchservices             0.3.0
appnope                       0.1.2
appscript                     1.1.2
argon2-cffi                   21.3.0
argon2-cffi-bindings          21.2.0
arrow                         1.2.3
astroid                       2.14.2
astropy                       5.1
asttokens                     2.0.5
astunparse                    1.6.3
async-timeout                 4.0.2
atomicwrites                  1.4.0
attrs                         22.1.0
audioread                     3.0.1
Automat                       20.2.0
autopep8                      1.6.0
Babel                         2.11.0
backcall                      0.2.0
backports.functools-lru-cache 1.6.4
backports.tempfile            1.0
backports.weakref             1.0.post1
bcrypt                        3.2.0
beautifulsoup4                4.12.2
binaryornot                   0.4.4
black                         0.0
bleach                        4.1.0
bokeh                         3.2.1
boltons                       23.0.0
botocore                      1.29.76
Bottleneck                    1.3.5
brotlipy                      0.7.0
cachetools                    5.3.1
certifi                       2023.7.22
cffi                          1.15.1
chardet                       4.0.0
charset-normalizer            2.0.4
click                         8.0.4
cloudpickle                   2.2.1
clyent                        1.2.2
colorama                      0.4.6
colorcet                      3.0.1
colorlog                      6.8.2
comm                          0.1.2
conda                         23.7.4
conda-build                   3.26.1
conda-content-trust           0.2.0
conda_index                   0.3.0
conda-libmamba-solver         23.9.1
conda-pack                    0.6.0
conda-package-handling        2.2.0
conda_package_streaming       0.9.0
conda-repo-cli                1.0.75
conda-token                   0.4.0
conda-verify                  3.4.2
constantly                    15.1.0
contourpy                     1.0.5
cookiecutter                  1.7.3
cryptography                  41.0.3
cssselect                     1.1.0
cycler                        0.11.0
cytoolz                       0.12.0
dask                          2023.6.0
datasets                      2.12.0
datashader                    0.15.2
datashape                     0.5.4
debugpy                       1.6.7
decorator                     5.1.1
defusedxml                    0.7.1
diff-match-patch              20200713
diffuser                      0.0.1
diffusers                     0.27.0
dill                          0.3.6
distributed                   2023.6.0
docstring-to-markdown         0.11
docutils                      0.18.1
entrypoints                   0.4
et-xmlfile                    1.1.0
executing                     0.8.3
fastjsonschema                2.16.2
filelock                      3.9.0
filterpy                      1.4.5
flake8                        6.0.0
Flask                         2.2.2
flatbuffers                   23.5.26
fonttools                     4.25.0
frozenlist                    1.3.3
fsspec                        2023.5.0
future                        0.18.3
gast                          0.5.4
gensim                        4.3.0
glob2                         0.7
gmpy2                         2.1.2
google-auth                   2.23.3
google-auth-oauthlib          1.0.0
google-pasta                  0.2.0
graphviz                      0.20.3
greenlet                      2.0.1
grpcio                        1.59.0
h5py                          3.9.0
HeapDict                      1.0.1
holoviews                     1.17.1
huggingface-hub               0.21.4
hvplot                        0.8.4
hyperlink                     21.0.0
idna                          3.4
imagecodecs                   2023.1.23
imageio                       2.31.1
imagesize                     1.4.1
imbalanced-learn              0.10.1
importlib-metadata            6.0.0
incremental                   21.3.0
inflection                    0.5.1
iniconfig                     1.1.1
intake                        0.6.8
intervaltree                  3.1.0
ipykernel                     6.25.0
ipython                       8.15.0
ipython-genutils              0.2.0
ipywidgets                    8.0.4
isort                         5.9.3
itemadapter                   0.3.0
itemloaders                   1.0.4
itsdangerous                  2.0.1
jaraco.classes                3.2.1
jedi                          0.18.1
jellyfish                     1.0.1
Jinja2                        3.1.2
jinja2-time                   0.2.0
jmespath                      0.10.0
joblib                        1.2.0
json5                         0.9.6
jsonpatch                     1.32
jsonpointer                   2.1
jsonschema                    4.17.3
jupyter                       1.0.0
jupyter_client                7.4.9
jupyter-console               6.6.3
jupyter_core                  5.3.0
jupyter-events                0.6.3
jupyter-server                1.23.4
jupyter_server_fileid         0.9.0
jupyter_server_ydoc           0.8.0
jupyter-ydoc                  0.2.4
jupyterlab                    3.6.3
jupyterlab-pygments           0.1.2
jupyterlab_server             2.22.0
jupyterlab-widgets            3.0.5
kaggle                        1.5.16
kaleido                       0.2.1
keras                         2.14.0
keras-tuner                   1.4.6
keyring                       23.13.1
kiwisolver                    1.4.4
kt-legacy                     1.0.5
lazy_loader                   0.2
lazy-object-proxy             1.6.0
libarchive-c                  2.9
libclang                      16.0.6
libmambapy                    1.5.1
librosa                       0.10.1
lightning-utilities           0.11.2
linkify-it-py                 2.0.0
llvmlite                      0.40.0
lmdb                          1.4.1
locket                        1.0.0
lxml                          4.9.3
lz4                           4.3.2
Mako                          1.3.5
Markdown                      3.4.1
markdown-it-py                2.2.0
MarkupSafe                    2.1.1
matplotlib                    3.7.2
matplotlib-inline             0.1.6
mccabe                        0.7.0
mdit-py-plugins               0.3.0
mdurl                         0.1.0
mistune                       0.8.4
ml-dtypes                     0.2.0
more-itertools                8.12.0
mpmath                        1.3.0
msgpack                       1.0.3
multidict                     6.0.2
multipledispatch              0.6.0
multiprocess                  0.70.14
munkres                       1.1.4
mypy-extensions               1.0.0
navigator-updater             0.4.0
nbclassic                     0.5.5
nbclient                      0.5.13
nbconvert                     6.5.4
nbformat                      5.9.2
nest-asyncio                  1.5.6
networkx                      3.1
nltk                          3.8.1
notebook                      6.5.4
notebook_shim                 0.2.2
numba                         0.57.0
numexpr                       2.8.4
numpy                         1.24.3
numpydoc                      1.5.0
oauthlib                      3.2.2
openpyxl                      3.0.10
opt-einsum                    3.3.0
optuna                        3.6.1
packaging                     23.1
pandas                        2.0.3
pandocfilters                 1.5.0
panel                         1.2.3
param                         1.13.0
parsel                        1.6.0
parso                         0.8.3
partd                         1.4.0
pathlib                       1.0.1
pathspec                      0.10.3
patsy                         0.5.3
pep8                          1.7.1
pexpect                       4.8.0
pickleshare                   0.7.5
Pillow                        9.4.0
pip                           23.2.1
pkce                          1.0.3
pkginfo                       1.9.6
platformdirs                  3.10.0
plotly                        5.9.0
pluggy                        1.0.0
ply                           3.11
pooch                         1.8.0
portalocker                   2.8.2
poyo                          0.5.0
prometheus-client             0.14.1
prompt-toolkit                3.0.36
Protego                       0.1.16
protobuf                      4.24.4
psutil                        5.9.0
ptyprocess                    0.7.0
pure-eval                     0.2.2
py-cpuinfo                    8.0.0
pyarrow                       11.0.0
pyasn1                        0.4.8
pyasn1-modules                0.2.8
pycodestyle                   2.10.0
pycosat                       0.6.4
pycparser                     2.21
pyct                          0.5.0
pycurl                        7.45.2
pydantic                      1.10.8
PyDispatcher                  2.0.5
pydocstyle                    6.3.0
pyerfa                        2.0.0
pyflakes                      3.0.1
Pygments                      2.15.1
PyJWT                         2.4.0
pylint                        2.16.2
pylint-venv                   2.3.0
pyls-spyder                   0.4.0
pyobjc-core                   9.0
pyobjc-framework-Cocoa        9.0
pyobjc-framework-CoreServices 9.0
pyobjc-framework-FSEvents     9.0
pyodbc                        4.0.34
pyOpenSSL                     23.2.0
pyparsing                     3.0.9
PyQt5-sip                     12.11.0
pyrsistent                    0.18.0
PySocks                       1.7.1
pytest                        7.4.0
python-dateutil               2.8.2
python-dotenv                 0.21.0
python-json-logger            2.0.7
python-lsp-black              1.2.1
python-lsp-jsonrpc            1.0.0
python-lsp-server             1.7.2
python-slugify                5.0.2
python-snappy                 0.6.1
pytoolconfig                  1.2.5
pytz                          2023.3.post1
pyviz-comms                   2.3.0
PyWavelets                    1.4.1
PyYAML                        6.0
pyzmq                         23.2.0
QDarkStyle                    3.0.2
qstylizer                     0.2.2
QtAwesome                     1.2.2
qtconsole                     5.4.2
QtPy                          2.2.0
queuelib                      1.5.0
regex                         2022.7.9
requests                      2.31.0
requests-file                 1.5.1
requests-oauthlib             1.3.1
requests-toolbelt             1.0.0
resampy                       0.4.2
responses                     0.13.3
rfc3339-validator             0.1.4
rfc3986-validator             0.1.1
rope                          1.7.0
rsa                           4.9
Rtree                         1.0.1
ruamel.yaml                   0.17.21
ruamel-yaml-conda             0.17.21
s3fs                          2023.4.0
safetensors                   0.3.2
scikit-image                  0.20.0
scikit-learn                  1.3.0
scipy                         1.11.1
Scrapy                        2.8.0
seaborn                       0.12.2
Send2Trash                    1.8.0
service-identity              18.1.0
setuptools                    68.0.0
sip                           6.6.2
six                           1.16.0
smart-open                    5.2.1
sniffio                       1.2.0
snowballstemmer               2.2.0
sortedcontainers              2.4.0
soundfile                     0.12.1
soupsieve                     2.4
soxr                          0.3.7
Sphinx                        5.0.2
sphinxcontrib-applehelp       1.0.2
sphinxcontrib-devhelp         1.0.2
sphinxcontrib-htmlhelp        2.0.0
sphinxcontrib-jsmath          1.0.1
sphinxcontrib-qthelp          1.0.3
sphinxcontrib-serializinghtml 1.1.5
spyder                        5.4.3
spyder-kernels                2.4.4
SQLAlchemy                    1.4.39
stack-data                    0.2.0
statsmodels                   0.14.0
sympy                         1.11.1
tables                        3.8.0
tabulate                      0.8.10
tblib                         1.7.0
tenacity                      8.2.2
tensorboard                   2.14.1
tensorboard-data-server       0.7.1
tensorflow                    2.14.0
tensorflow-estimator          2.14.0
tensorflow-io                 0.34.0
tensorflow-io-gcs-filesystem  0.33.0
termcolor                     2.3.0
terminado                     0.17.1
text-unidecode                1.3
textdistance                  4.2.1
threadpoolctl                 2.2.0
three-merge                   0.1.1
tifffile                      2023.4.12
tinycss2                      1.2.1
tldextract                    3.2.0
tokenizers                    0.13.2
toml                          0.10.2
tomlkit                       0.11.1
toolz                         0.12.0
torch                         2.2.2
torchdata                     0.7.1
torchmetrics                  1.4.0.post0
torchtext                     0.17.2
torchvision                   0.17.2
torchviz                      0.0.2
tornado                       6.3.2
tqdm                          4.65.0
traitlets                     5.7.1
transformers                  4.32.1
Twisted                       22.10.0
typing_extensions             4.10.0
tzdata                        2023.3
uc-micro-py                   1.0.1
ujson                         5.4.0
Unidecode                     1.2.0
urllib3                       1.26.16
w3lib                         1.21.0
watchdog                      2.1.6
wcwidth                       0.2.5
webencodings                  0.5.1
websocket-client              0.58.0
Werkzeug                      2.2.3
whatthepatch                  1.0.2
wheel                         0.38.4
widgetsnbextension            4.0.5
wrapt                         1.14.1
wurlitzer                     3.0.2
xarray                        2023.6.0
xlwings                       0.29.1
xxhash                        2.0.2
xyzservices                   2022.9.0
y-py                          0.5.9
yapf                          0.31.0
yarl                          1.8.1
ypy-websocket                 0.8.2
zict                          2.2.0
zipp                          3.11.0
zope.interface                5.4.0
zstandard                     0.19.0 ```

[SleepTight-Cat]

Hi, if you do
get_dataset_config_names("opus_books")
and get only
['ca-de']
then this maybe caused by using an old version of datasets.
For me the code fails with datasets-2.13.2, yet works with datasets-3.2.0. So you may consider update your datasets package as I saw that you have datasets-2.12.0.

You may also refer to this post which is basically just me asking the same question in the Huggingface forum.

Hope these help!