torch_em.data.datasets.histopathology.histo_miner

  1"""The Histo-Miner dataset contains annotations for nucleus instance and semantic segmentation
  2(NucSeg) and for tumor region segmentation (TumSeg) in H&E stained cutaneous squamous cell
  3carcinoma (cSCC) histopathology images.
  4
  5NOTE: The public deposit only exposes train and validation splits; the held-out TumSeg test
  6split is released in a separate Zenodo record and is not covered by this loader.
  7
  8This dataset is located at https://zenodo.org/records/15973142.
  9This dataset is from the publication https://doi.org/10.1371/journal.pcbi.1013907.
 10Please cite it if you use this dataset for your research.
 11"""
 12
 13import os
 14from glob import glob
 15from natsort import natsorted
 16from typing import Union, Literal, Tuple, List
 17
 18import numpy as np
 19import imageio.v3 as imageio
 20
 21from torch.utils.data import Dataset, DataLoader
 22
 23import torch_em
 24
 25from .. import util
 26
 27
 28URLS = {
 29    "nuclei": "https://zenodo.org/api/records/15973142/files/NucSeg_OriginalFormat.zip/content",
 30    "tumor": "https://zenodo.org/api/records/15973142/files/TumSeg.zip/content",
 31}
 32
 33CHECKSUMS = {
 34    "nuclei": "6316b027ef50ce874e3f147f20a069f6c5ad9af5688d919c1ecf836301e6eccb",
 35    "tumor": "8091907d84ef75cfa7cf5deff113f6d9d2a76f9269c29f095703165c4f91f682",
 36}
 37
 38DATA_DIRNAMES = {"nuclei": "NucSeg_OriginalFormat", "tumor": "TumSeg"}
 39
 40
 41def get_histo_miner_data(
 42    path: Union[os.PathLike, str], task: Literal["nuclei", "tumor"], download: bool = False
 43) -> str:
 44    """Download the Histo-Miner data.
 45
 46    Args:
 47        path: Filepath to a folder where the downloaded data will be saved.
 48        task: The choice of task, either nucleus segmentation ('nuclei') or tumor region segmentation ('tumor').
 49        download: Whether to download the data if it is not present.
 50
 51    Returns:
 52        Filepath where the dataset is downloaded and stored for further preprocessing.
 53    """
 54    data_dir = os.path.join(path, DATA_DIRNAMES[task])
 55    if os.path.exists(data_dir):
 56        return data_dir
 57
 58    os.makedirs(path, exist_ok=True)
 59    zip_path = os.path.join(path, f"histo_miner_{task}.zip")
 60    util.download_source(path=zip_path, url=URLS[task], download=download, checksum=CHECKSUMS[task])
 61    util.unzip(zip_path=zip_path, dst=path)
 62
 63    return data_dir
 64
 65
 66def _convert_nuclei_npy_to_tif(data_dir, split_dir, label_choice):
 67    label_dirname = "InstanceMaps" if label_choice == "instances" else "ClassMaps"
 68    raw_dir = os.path.join(data_dir, split_dir, "RawImages")
 69    label_dir = os.path.join(data_dir, split_dir, label_dirname)
 70
 71    converted_raw_dir = os.path.join(data_dir, split_dir, "RawImages_tif")
 72    converted_label_dir = os.path.join(data_dir, split_dir, f"{label_dirname}_tif")
 73    os.makedirs(converted_raw_dir, exist_ok=True)
 74    os.makedirs(converted_label_dir, exist_ok=True)
 75
 76    raw_paths, label_paths = [], []
 77    for raw_path in natsorted(glob(os.path.join(raw_dir, "*.npy"))):
 78        fname = os.path.basename(raw_path).replace(".npy", ".tif")
 79
 80        out_raw_path = os.path.join(converted_raw_dir, fname)
 81        if not os.path.exists(out_raw_path):
 82            imageio.imwrite(out_raw_path, np.load(raw_path))
 83        raw_paths.append(out_raw_path)
 84
 85        label_path = os.path.join(label_dir, os.path.basename(raw_path))
 86        out_label_path = os.path.join(converted_label_dir, fname)
 87        if not os.path.exists(out_label_path):
 88            imageio.imwrite(out_label_path, np.load(label_path).astype("int32"))
 89        label_paths.append(out_label_path)
 90
 91    return raw_paths, label_paths
 92
 93
 94def get_histo_miner_paths(
 95    path: Union[os.PathLike, str],
 96    split: Literal["train", "val"],
 97    task: Literal["nuclei", "tumor"] = "nuclei",
 98    label_choice: Literal["instances", "semantic"] = "instances",
 99    download: bool = False,
100) -> Tuple[List[str], List[str]]:
101    """Get paths to the Histo-Miner data.
102
103    Args:
104        path: Filepath to a folder where the downloaded data will be saved.
105        split: The choice of data split.
106        task: The choice of task, either nucleus segmentation ('nuclei') or tumor region segmentation ('tumor').
107        label_choice: The choice of label representation for the 'nuclei' task, either instance or semantic labels.
108        download: Whether to download the data if it is not present.
109
110    Returns:
111        List of filepaths to the image data.
112        List of filepaths to the label data.
113    """
114    data_dir = get_histo_miner_data(path, task, download)
115    split_dir = "Train" if split == "train" else "Val"
116
117    if task == "nuclei":
118        raw_paths, label_paths = _convert_nuclei_npy_to_tif(data_dir, split_dir, label_choice)
119    else:
120        raw_paths = natsorted(glob(os.path.join(data_dir, split_dir, "images", "*.tif")))
121        label_paths = natsorted(glob(os.path.join(data_dir, split_dir, "annotations", "*.png")))
122
123    assert len(raw_paths) == len(label_paths) and len(raw_paths) > 0
124    return raw_paths, label_paths
125
126
127def get_histo_miner_dataset(
128    path: Union[os.PathLike, str],
129    patch_shape: Tuple[int, int],
130    split: Literal["train", "val"],
131    task: Literal["nuclei", "tumor"] = "nuclei",
132    label_choice: Literal["instances", "semantic"] = "instances",
133    resize_inputs: bool = False,
134    download: bool = False,
135    **kwargs
136) -> Dataset:
137    """Get the Histo-Miner dataset for nucleus or tumor region segmentation.
138
139    Args:
140        path: Filepath to a folder where the downloaded data will be saved.
141        patch_shape: The patch shape to use for training.
142        split: The choice of data split.
143        task: The choice of task, either nucleus segmentation ('nuclei') or tumor region segmentation ('tumor').
144        label_choice: The choice of label representation for the 'nuclei' task, either instance or semantic labels.
145        resize_inputs: Whether to resize the inputs.
146        download: Whether to download the data if it is not present.
147        kwargs: Additional keyword arguments for `torch_em.default_segmentation_dataset`.
148
149    Returns:
150        The segmentation dataset.
151    """
152    raw_paths, label_paths = get_histo_miner_paths(path, split, task, label_choice, download)
153
154    if resize_inputs:
155        resize_kwargs = {"patch_shape": patch_shape, "is_rgb": True}
156        kwargs, patch_shape = util.update_kwargs_for_resize_trafo(
157            kwargs=kwargs, patch_shape=patch_shape, resize_inputs=resize_inputs, resize_kwargs=resize_kwargs
158        )
159
160    return torch_em.default_segmentation_dataset(
161        raw_paths=raw_paths,
162        raw_key=None,
163        label_paths=label_paths,
164        label_key=None,
165        is_seg_dataset=False,
166        patch_shape=patch_shape,
167        with_channels=True,
168        ndim=2,
169        **kwargs
170    )
171
172
173def get_histo_miner_loader(
174    path: Union[os.PathLike, str],
175    batch_size: int,
176    patch_shape: Tuple[int, int],
177    split: Literal["train", "val"],
178    task: Literal["nuclei", "tumor"] = "nuclei",
179    label_choice: Literal["instances", "semantic"] = "instances",
180    resize_inputs: bool = False,
181    download: bool = False,
182    **kwargs
183) -> DataLoader:
184    """Get the Histo-Miner dataloader for nucleus or tumor region segmentation.
185
186    Args:
187        path: Filepath to a folder where the downloaded data will be saved.
188        batch_size: The batch size for training.
189        patch_shape: The patch shape to use for training.
190        split: The choice of data split.
191        task: The choice of task, either nucleus segmentation ('nuclei') or tumor region segmentation ('tumor').
192        label_choice: The choice of label representation for the 'nuclei' task, either instance or semantic labels.
193        resize_inputs: Whether to resize the inputs.
194        download: Whether to download the data if it is not present.
195        kwargs: Additional keyword arguments for `torch_em.default_segmentation_dataset` or for the PyTorch DataLoader.
196
197    Returns:
198        The DataLoader.
199    """
200    ds_kwargs, loader_kwargs = util.split_kwargs(torch_em.default_segmentation_dataset, **kwargs)
201    dataset = get_histo_miner_dataset(
202        path, patch_shape, split, task, label_choice, resize_inputs, download, **ds_kwargs
203    )
204    return torch_em.get_data_loader(dataset, batch_size, **loader_kwargs)