torch_em.data.datasets.histopathology.sinus

  1"""The SiNuS dataset contains annotations for singular nucleus segmentation in
  2Dual In Situ Hybridization (DISH) images of breast cancer tissue.
  3
  4NOTE: This dataset is sparsely annotated. It contains annotations for expert-selected
  5singular nuclei suitable for HER2 grading, rather than for all nuclei in each image.
  6
  7The dataset is located at https://data.mendeley.com/datasets/gtjrgwbntc/2.
  8This dataset is from the publication https://doi.org/10.1016/j.dib.2026.112934.
  9Please cite it if you use this dataset for your research.
 10"""
 11
 12import json
 13import os
 14from glob import glob
 15from pathlib import Path
 16from tqdm import tqdm
 17from natsort import natsorted
 18from typing import List, Literal, Tuple, Union
 19
 20import numpy as np
 21import imageio.v3 as imageio
 22from skimage.draw import polygon
 23
 24from torch.utils.data import DataLoader, Dataset
 25
 26import torch_em
 27
 28from .. import util
 29
 30
 31URL = "https://data.mendeley.com/public-api/zip/gtjrgwbntc/download/2"
 32CHECKSUM = "aecd1399192ee511ba29f6c23e6f858b4e6a8328028c1ae91f9ee5a826728c5c"
 33
 34
 35def get_sinus_data(path: Union[os.PathLike, str], download: bool = False) -> str:
 36    """Download the SiNuS dataset.
 37
 38    Args:
 39        path: Filepath to a folder where the downloaded data will be saved.
 40        download: Whether to download the data if it is not present.
 41
 42    Returns:
 43        Filepath where the dataset is downloaded.
 44    """
 45    data_dir = os.path.join(path, "SiNuS A Comprehensive Dataset for Singular Nuclei", "SiNuS")
 46    if os.path.exists(data_dir):
 47        return data_dir
 48
 49    os.makedirs(path, exist_ok=True)
 50
 51    zip_path = os.path.join(path, "sinus.zip")
 52    util.download_source(path=zip_path, url=URL, download=download, checksum=CHECKSUM)
 53    util.unzip(zip_path=zip_path, dst=path)
 54
 55    return data_dir
 56
 57
 58def _create_instance_labels(annotation_path: str, label_path: str) -> None:
 59    with open(annotation_path) as f:
 60        annotation = json.load(f)["annotation"]
 61
 62    shape = (annotation["size"]["height"], annotation["size"]["width"])
 63    labels = np.zeros(shape, dtype="uint16")
 64    for label_id, annotated_object in enumerate(annotation["objects"], 1):
 65        points = np.asarray(annotated_object["points"]["exterior"])
 66        rr, cc = polygon(points[:, 1], points[:, 0], shape=shape)
 67        labels[rr, cc] = label_id
 68
 69        for interior in annotated_object["points"]["interior"]:
 70            points = np.asarray(interior)
 71            rr, cc = polygon(points[:, 1], points[:, 0], shape=shape)
 72            labels[rr, cc] = 0
 73
 74    imageio.imwrite(label_path, labels, compression="zlib")
 75
 76
 77def get_sinus_paths(
 78    path: Union[os.PathLike, str],
 79    annotation_choice: Literal["inclusive", "exclusive"] = "inclusive",
 80    download: bool = False,
 81) -> Tuple[List[str], List[str]]:
 82    """Get paths to the SiNuS data.
 83
 84    Args:
 85        path: Filepath to a folder where the downloaded data will be saved.
 86        annotation_choice: The annotation selection. The inclusive annotations contain nuclei selected by at least
 87            one expert, while the exclusive annotations contain nuclei selected by all experts.
 88        download: Whether to download the data if it is not present.
 89
 90    Returns:
 91        List of filepaths for the image data.
 92        List of filepaths for the label data.
 93    """
 94    if annotation_choice not in ("inclusive", "exclusive"):
 95        raise ValueError(f"'{annotation_choice}' is not a valid annotation choice.")
 96
 97    data_dir = get_sinus_data(path, download)
 98    raw_paths = natsorted(glob(os.path.join(data_dir, "Original", "*.JPG")))
 99
100    selection = f"{annotation_choice.capitalize()} Nuclei Selection"
101    suffix = "ins" if annotation_choice == "inclusive" else "ens"
102    annotation_paths = natsorted(glob(os.path.join(data_dir, selection, "Image *", f"*_annotation_{suffix}.json")))
103
104    label_dir = os.path.join(data_dir, "preprocessed_labels", annotation_choice)
105    os.makedirs(label_dir, exist_ok=True)
106    label_paths = []
107    for annotation_path in tqdm(annotation_paths, desc=f"Preprocessing {annotation_choice} SiNuS labels"):
108        image_name = Path(annotation_path).name.split("_annotation")[0]
109        label_path = os.path.join(label_dir, f"{image_name}.tif")
110        label_paths.append(label_path)
111        if os.path.exists(label_path):
112            continue
113
114        _create_instance_labels(annotation_path, label_path)
115
116    assert len(raw_paths) == len(label_paths) and len(raw_paths) > 0
117    assert all(Path(raw_path).stem == Path(label_path).stem for raw_path, label_path in zip(raw_paths, label_paths))
118
119    return raw_paths, label_paths
120
121
122def get_sinus_dataset(
123    path: Union[os.PathLike, str],
124    patch_shape: Tuple[int, int],
125    annotation_choice: Literal["inclusive", "exclusive"] = "inclusive",
126    resize_inputs: bool = False,
127    download: bool = False,
128    **kwargs,
129) -> Dataset:
130    """Get the SiNuS dataset for singular nucleus segmentation.
131
132    Args:
133        path: Filepath to a folder where the downloaded data will be saved.
134        patch_shape: The patch shape to use for training.
135        annotation_choice: The annotation selection. The inclusive annotations contain nuclei selected by at least
136            one expert, while the exclusive annotations contain nuclei selected by all experts.
137        resize_inputs: Whether to resize the inputs.
138        download: Whether to download the data if it is not present.
139        kwargs: Additional keyword arguments for `torch_em.default_segmentation_dataset`.
140
141    Returns:
142        The segmentation dataset.
143    """
144    raw_paths, label_paths = get_sinus_paths(path, annotation_choice, download)
145
146    if resize_inputs:
147        resize_kwargs = {"patch_shape": patch_shape, "is_rgb": True}
148        kwargs, patch_shape = util.update_kwargs_for_resize_trafo(
149            kwargs=kwargs, patch_shape=patch_shape, resize_inputs=resize_inputs, resize_kwargs=resize_kwargs
150        )
151
152    return torch_em.default_segmentation_dataset(
153        raw_paths=raw_paths,
154        raw_key=None,
155        label_paths=label_paths,
156        label_key=None,
157        is_seg_dataset=False,
158        patch_shape=patch_shape,
159        ndim=2,
160        with_channels=True,
161        **kwargs,
162    )
163
164
165def get_sinus_loader(
166    path: Union[os.PathLike, str],
167    batch_size: int,
168    patch_shape: Tuple[int, int],
169    annotation_choice: Literal["inclusive", "exclusive"] = "inclusive",
170    resize_inputs: bool = False,
171    download: bool = False,
172    **kwargs,
173) -> DataLoader:
174    """Get the SiNuS dataloader for singular nucleus segmentation.
175
176    Args:
177        path: Filepath to a folder where the downloaded data will be saved.
178        batch_size: The batch size for training.
179        patch_shape: The patch shape to use for training.
180        annotation_choice: The annotation selection. The inclusive annotations contain nuclei selected by at least
181            one expert, while the exclusive annotations contain nuclei selected by all experts.
182        resize_inputs: Whether to resize the inputs.
183        download: Whether to download the data if it is not present.
184        kwargs: Additional keyword arguments for `torch_em.default_segmentation_dataset` or for the PyTorch DataLoader.
185
186    Returns:
187        The DataLoader.
188    """
189    ds_kwargs, loader_kwargs = util.split_kwargs(torch_em.default_segmentation_dataset, **kwargs)
190    dataset = get_sinus_dataset(
191        path=path,
192        patch_shape=patch_shape,
193        annotation_choice=annotation_choice,
194        resize_inputs=resize_inputs,
195        download=download,
196        **ds_kwargs,
197    )
198    return torch_em.get_data_loader(dataset, batch_size, **loader_kwargs)