Module neuralops.data.custom

Custom dataset loading utilities.

Functions

def load_custom_dataset(path: str,
split: str | None = None,
data_files: dict[str, str] | None = None,
**kwargs) ‑> datasets.arrow_dataset.Dataset | datasets.dataset_dict.DatasetDict

Expand source code

def load_custom_dataset(
    path: str,
    split: str | None = None,
    data_files: dict[str, str] | None = None,
    **kwargs,
) -> Dataset | DatasetDict:
    """
    Load any dataset from a local or remote path.
    Supports CSV, JSON, Parquet, text, and HuggingFace hub datasets.
    """
    if path.endswith((".csv", ".json", ".parquet", ".txt")):
        dataset = load_dataset(path.split(".")[-1], data_files=data_files or path, **kwargs)
    else:
        dataset = load_dataset(path, data_files=data_files, **kwargs)
    if split:
        return dataset[split]
    return dataset

Load any dataset from a local or remote path. Supports CSV, JSON, Parquet, text, and HuggingFace hub datasets.