torch_em.data.datasets.histopathology.cd47_ihcnusc

  1"""The CD47-IHCNuSC dataset contains nucleus annotations for CD47 immunohistochemistry (IHC) images
  2of esophageal cancer.
  3
  4The dataset holds 86 patches of 1024 x 1024 pixels at 40x magnification (0.25 micrometer per pixel),
  570 for training and 16 for validation / testing, with 18,483 annotated nuclei.
  6The annotations were made in a human-in-the-loop process: HoverFast created coarse nucleus masks, which were
  7refined by hand in QuPath over several rounds and then validated by experts. Every nucleus carries one of
  8seven types (see 'TYPES'). The 'ambiguous' type of the annotation protocol is not part of the release,
  9so ambiguous nuclei are background in the labels.
 10
 11The dataset is located at https://doi.org/10.5524/102830 (CC0 1.0).
 12This dataset is from the publication https://doi.org/10.1093/gigascience/giag091.
 13Please cite it if you use this dataset for your research.
 14"""
 15
 16import os
 17from glob import glob
 18from tqdm import tqdm
 19from natsort import natsorted
 20from typing import Union, Tuple, List, Literal
 21
 22import imageio.v3 as imageio
 23
 24from torch.utils.data import Dataset, DataLoader
 25
 26import torch_em
 27
 28from .. import util
 29
 30
 31BASE_URL = "https://s3.ap-northeast-1.wasabisys.com/gigadb-datasets/live/pub/10.5524/102000_103000/102830"
 32ARCHIVES = {
 33    "CD47-IHCNuSC_images.zip": "016e2f8f0e6efcb8d55ecf9fb22eeffcefd28ef4127e804e6dffe27ce12a21cb",
 34    "CD47-IHCNuSC_annotations.zip": "90c4e2c799a48a80ceaaa1e0a03d9632ca47b8f82bd7e464d49a4654a879d5e9",
 35}
 36TYPES = {
 37    1: "positive_tumor", 2: "positive_immune", 3: "positive_others", 4: "negative_tumor",
 38    5: "negative_immune", 6: "negative_others", 7: "negative_stroma",
 39}
 40
 41
 42def _create_h5_files(path):
 43    import h5py
 44    from scipy.io import loadmat
 45
 46    for split in ("train", "val"):
 47        out_dir = os.path.join(path, "preprocessed", split)
 48        os.makedirs(out_dir, exist_ok=True)
 49        image_paths = natsorted(glob(os.path.join(path, "images", split, "*.tif")))
 50        for image_path in tqdm(image_paths, desc=f"Preprocess CD47-IHCNuSC {split} images"):
 51            name = os.path.splitext(os.path.basename(image_path))[0]
 52            image = imageio.imread(image_path)[..., :3]
 53            labels = loadmat(os.path.join(path, "annotations", split, f"{name}.mat"))
 54            instances, types = labels["inst_map"].astype("uint32"), labels["type_map"].astype("uint8")
 55            assert image.shape[:2] == instances.shape == types.shape, name
 56
 57            with h5py.File(os.path.join(out_dir, f"{name}.h5"), "w") as f:
 58                f.create_dataset("raw", data=image.transpose(2, 0, 1), compression="gzip")
 59                f.create_dataset("labels/instances", data=instances, compression="gzip")
 60                f.create_dataset("labels/semantic", data=types, compression="gzip")
 61
 62
 63def get_cd47_ihcnusc_data(path: Union[os.PathLike, str], download: bool = False) -> str:
 64    """Download the CD47-IHCNuSC dataset.
 65
 66    Args:
 67        path: Filepath to a folder where the downloaded data is saved.
 68        download: Whether to download the data if it is not present.
 69
 70    Returns:
 71        Filepath where the preprocessed data is stored.
 72    """
 73    data_dir = os.path.join(path, "preprocessed")
 74    if os.path.exists(data_dir):
 75        return data_dir
 76
 77    os.makedirs(path, exist_ok=True)
 78    for name, checksum in ARCHIVES.items():
 79        zip_path = os.path.join(path, name)
 80        util.download_source(zip_path, f"{BASE_URL}/{name}", download, checksum=checksum)
 81        util.unzip(zip_path, path, remove=False)
 82
 83    _create_h5_files(path)
 84    return data_dir
 85
 86
 87def get_cd47_ihcnusc_paths(
 88    path: Union[os.PathLike, str], split: Literal["train", "val"], download: bool = False
 89) -> List[str]:
 90    """Get paths to the CD47-IHCNuSC data.
 91
 92    Args:
 93        path: Filepath to a folder where the downloaded data is saved.
 94        split: The choice of data split. Either 'train' or 'val' (the validation / test split of the authors).
 95        download: Whether to download the data if it is not present.
 96
 97    Returns:
 98        List of filepaths for the input data.
 99    """
100    assert split in ("train", "val"), split
101    data_dir = get_cd47_ihcnusc_data(path, download)
102    return natsorted(glob(os.path.join(data_dir, split, "*.h5")))
103
104
105def get_cd47_ihcnusc_dataset(
106    path: Union[os.PathLike, str],
107    patch_shape: Tuple[int, int],
108    split: Literal["train", "val"],
109    label_choice: Literal["instances", "semantic"] = "instances",
110    download: bool = False,
111    **kwargs
112) -> Dataset:
113    """Get the CD47-IHCNuSC dataset for nucleus segmentation.
114
115    Args:
116        path: Filepath to a folder where the downloaded data is saved.
117        patch_shape: The patch shape to use for training.
118        split: The choice of data split. Either 'train' or 'val' (the validation / test split of the authors).
119        label_choice: The choice of labels, 'instances' or 'semantic' (the nucleus types, see 'TYPES').
120        download: Whether to download the data if it is not present.
121        kwargs: Additional keyword arguments for `torch_em.default_segmentation_dataset`.
122
123    Returns:
124        The segmentation dataset.
125    """
126    assert label_choice in ("instances", "semantic"), label_choice
127    paths = get_cd47_ihcnusc_paths(path, split, download)
128    return torch_em.default_segmentation_dataset(
129        raw_paths=paths,
130        raw_key="raw",
131        label_paths=paths,
132        label_key=f"labels/{label_choice}",
133        patch_shape=patch_shape,
134        ndim=2,
135        with_channels=True,
136        **kwargs
137    )
138
139
140def get_cd47_ihcnusc_loader(
141    path: Union[os.PathLike, str],
142    batch_size: int,
143    patch_shape: Tuple[int, int],
144    split: Literal["train", "val"],
145    label_choice: Literal["instances", "semantic"] = "instances",
146    download: bool = False,
147    **kwargs
148) -> DataLoader:
149    """Get the CD47-IHCNuSC dataloader for nucleus segmentation.
150
151    Args:
152        path: Filepath to a folder where the downloaded data is saved.
153        batch_size: The batch size for training.
154        patch_shape: The patch shape to use for training.
155        split: The choice of data split. Either 'train' or 'val' (the validation / test split of the authors).
156        label_choice: The choice of labels, 'instances' or 'semantic' (the nucleus types, see 'TYPES').
157        download: Whether to download the data if it is not present.
158        kwargs: Additional keyword arguments for `torch_em.default_segmentation_dataset` or for the PyTorch DataLoader.
159
160    Returns:
161        The DataLoader.
162    """
163    ds_kwargs, loader_kwargs = util.split_kwargs(torch_em.default_segmentation_dataset, **kwargs)
164    dataset = get_cd47_ihcnusc_dataset(path, patch_shape, split, label_choice, download, **ds_kwargs)
165    return torch_em.get_data_loader(dataset, batch_size, **loader_kwargs)