Newer
Older
TIASJudgeModel / tests / data / test_split.py
"""患者単位分割のテスト(SPEC_02「データ分割」・SPEC_01「patient_id について」).

最重要: 同一 patient_id が複数の分割に跨らないこと.1 患者に複数枚ある manifest でも同様.
"""

import logging
import random
from collections import Counter

import pytest

from tias_judge_model.data.classes import CLASS_SLUGS, VALID_SEVERITIES
from tias_judge_model.data.manifest import ManifestRecord
from tias_judge_model.data.split import (
    Fold,
    SplitResult,
    count_by_class,
    count_by_class_severity,
    make_kfold_splits,
    patient_labels,
    select_records,
    split_train_val_test,
)

SEED = 42


def make_records(
    patients_per_class: int | dict[str, int], images_per_patient: int = 1, prefix: str = "pt"
) -> list[ManifestRecord]:
    """クラスごとに指定人数の患者を持つ合成 manifest を作る(1 患者 images_per_patient 枚)."""
    if isinstance(patients_per_class, int):
        patients_per_class = dict.fromkeys(CLASS_SLUGS, patients_per_class)
    records = []
    for label, n_patients in patients_per_class.items():
        severities = VALID_SEVERITIES[label]
        for p in range(n_patients):
            patient_id = f"{prefix}_{label}_{p:04d}"
            for k in range(images_per_patient):
                severity = severities[(p + k) % len(severities)]
                records.append(
                    ManifestRecord(
                        image_path=f"{label}_{severity}/{patient_id}_{k}.png",
                        patient_id=patient_id,
                        label=label,
                        severity=severity,
                    )
                )
    return records


def shuffled(records: list[ManifestRecord], seed: int) -> list[ManifestRecord]:
    out = list(records)
    random.Random(seed).shuffle(out)
    return out


def all_patients(records: list[ManifestRecord]) -> set[str]:
    return {r.patient_id for r in records}


def labels_in(patient_ids: tuple[str, ...], records: list[ManifestRecord]) -> Counter[str]:
    mapping = patient_labels(records)
    return Counter(mapping[p] for p in patient_ids)


# --- 単一分割 (train / val / test) ---


@pytest.mark.parametrize("images_per_patient", [1, 3])
def test_single_split_keeps_patients_disjoint(images_per_patient: int) -> None:
    records = make_records(20, images_per_patient=images_per_patient)
    result = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=SEED)

    train, val, test = set(result.train), set(result.val), set(result.test)
    assert not train & val
    assert not train & test
    assert not val & test
    assert train | val | test == all_patients(records)
    # 各分割は患者 ID の重複を含まない
    for part in (result.train, result.val, result.test):
        assert len(part) == len(set(part))


def test_single_split_puts_all_images_of_a_patient_in_same_split() -> None:
    records = make_records(12, images_per_patient=4)
    result = split_train_val_test(records, val_ratio=0.2, test_ratio=0.2, seed=SEED)

    selected = {
        name: select_records(records, ids)
        for name, ids in (("train", result.train), ("val", result.val), ("test", result.test))
    }
    assert sum(len(v) for v in selected.values()) == len(records)
    for name, recs in selected.items():
        for r in recs:
            images_of_patient = [x for x in records if x.patient_id == r.patient_id]
            assert all(x in recs for x in images_of_patient), name


def test_single_split_with_uneven_images_per_patient_stays_disjoint() -> None:
    # 1 患者の枚数がばらつく manifest(将来の複数枚追加を想定)
    base = make_records(10, images_per_patient=1, prefix="one")
    many = make_records(10, images_per_patient=5, prefix="many")
    records = base + many
    result = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=SEED)

    groups = [set(result.train), set(result.val), set(result.test)]
    for patient in all_patients(records):
        assert sum(patient in g for g in groups) == 1


def test_single_split_is_reproducible_with_same_seed() -> None:
    records = make_records(20)
    a = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=SEED)
    b = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=SEED)
    assert a == b


def test_single_split_changes_with_different_seed() -> None:
    records = make_records(30)
    a = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=0)
    b = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=1)
    assert a != b


@pytest.mark.parametrize("images_per_patient", [1, 3])
def test_single_split_does_not_depend_on_input_order(images_per_patient: int) -> None:
    records = make_records(20, images_per_patient=images_per_patient)
    expected = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=SEED)
    for shuffle_seed in range(3):
        actual = split_train_val_test(
            shuffled(records, shuffle_seed), val_ratio=0.15, test_ratio=0.15, seed=SEED
        )
        assert actual == expected
    assert (
        split_train_val_test(list(reversed(records)), val_ratio=0.15, test_ratio=0.15, seed=SEED)
        == expected
    )


def test_single_split_is_stratified_by_class() -> None:
    # 実データに近い不均衡な分布(TEST_01 の 1/10 程度)
    per_class = {"pale": 10, "light_red": 34, "red": 12, "dark_red": 38, "purple": 15}
    records = make_records(per_class)
    result = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=SEED)

    for part in (result.train, result.val, result.test):
        counts = labels_in(part, records)
        assert set(counts) == set(CLASS_SLUGS), "各分割に全クラスが含まれること"

    # 各クラスで val / test の人数が比率どおり(端数は 1 人以内)
    for part in (result.val, result.test):
        counts = labels_in(part, records)
        for label, n in per_class.items():
            assert abs(counts[label] - n * 0.15) <= 1, label


def test_single_split_sizes_follow_ratios() -> None:
    records = make_records(20)
    result = split_train_val_test(records, val_ratio=0.15, test_ratio=0.15, seed=SEED)
    # 各クラス 20 人 × 0.15 = 3 人
    assert len(result.val) == 15
    assert len(result.test) == 15
    assert len(result.train) == 70


def test_single_split_to_dict_contains_all_parts() -> None:
    result = split_train_val_test(make_records(10), val_ratio=0.2, test_ratio=0.2, seed=SEED)
    d = result.to_dict()
    assert set(d) == {"train", "val", "test"}
    assert d["train"] == list(result.train)
    assert isinstance(result, SplitResult)


def test_single_split_rejects_empty_records() -> None:
    with pytest.raises(ValueError):
        split_train_val_test([], val_ratio=0.15, test_ratio=0.15, seed=SEED)


@pytest.mark.parametrize(
    ("val_ratio", "test_ratio"), [(0.5, 0.5), (0.6, 0.5), (0.0, 0.2), (0.2, -0.1), (1.0, 0.1)]
)
def test_single_split_rejects_invalid_ratios(val_ratio: float, test_ratio: float) -> None:
    with pytest.raises(ValueError):
        split_train_val_test(make_records(20), val_ratio, test_ratio, seed=SEED)


def test_single_split_rejects_patient_with_conflicting_labels() -> None:
    records = make_records(10)
    conflict = ManifestRecord(
        image_path="purple_mild/extra.png",
        patient_id=records[0].patient_id,
        label="purple",
        severity="mild",
    )
    assert records[0].label != "purple"
    with pytest.raises(ValueError):
        split_train_val_test([*records, conflict], val_ratio=0.15, test_ratio=0.15, seed=SEED)


def test_split_logs_have_no_patient_id(caplog: pytest.LogCaptureFixture) -> None:
    records = make_records(10, prefix="SECRETPT")
    with caplog.at_level(logging.DEBUG):
        split_train_val_test(records, val_ratio=0.2, test_ratio=0.2, seed=SEED)
        make_kfold_splits(records, n_folds=5, seed=SEED)
    assert "SECRETPT" not in caplog.text


# --- K-fold 交差検証 ---


@pytest.mark.parametrize("images_per_patient", [1, 3])
def test_kfold_held_out_covers_all_patients_exactly_once(images_per_patient: int) -> None:
    records = make_records(12, images_per_patient=images_per_patient)
    folds = make_kfold_splits(records, n_folds=5, seed=SEED)

    assert len(folds) == 5
    assert [f.index for f in folds] == list(range(5))
    held_out_all = [p for f in folds for p in f.held_out]
    assert len(held_out_all) == len(set(held_out_all)), "held_out が fold 間で重複しない"
    assert set(held_out_all) == all_patients(records)


@pytest.mark.parametrize("images_per_patient", [1, 3])
def test_kfold_train_and_held_out_are_disjoint(images_per_patient: int) -> None:
    records = make_records(12, images_per_patient=images_per_patient)
    for fold in make_kfold_splits(records, n_folds=5, seed=SEED):
        assert isinstance(fold, Fold)
        assert not set(fold.train) & set(fold.held_out)
        assert set(fold.train) | set(fold.held_out) == all_patients(records)


def test_kfold_with_uneven_images_per_patient_stays_disjoint() -> None:
    records = make_records(6, prefix="one") + make_records(6, images_per_patient=4, prefix="many")
    for fold in make_kfold_splits(records, n_folds=5, seed=SEED):
        train_recs = select_records(records, fold.train)
        held_recs = select_records(records, fold.held_out)
        assert not {r.patient_id for r in train_recs} & {r.patient_id for r in held_recs}
        assert len(train_recs) + len(held_recs) == len(records)


def test_kfold_is_reproducible_with_same_seed() -> None:
    records = make_records(12)
    assert make_kfold_splits(records, 5, seed=SEED) == make_kfold_splits(records, 5, seed=SEED)


def test_kfold_changes_with_different_seed() -> None:
    records = make_records(12)
    assert make_kfold_splits(records, 5, seed=0) != make_kfold_splits(records, 5, seed=1)


@pytest.mark.parametrize("images_per_patient", [1, 3])
def test_kfold_does_not_depend_on_input_order(images_per_patient: int) -> None:
    records = make_records(12, images_per_patient=images_per_patient)
    expected = make_kfold_splits(records, 5, seed=SEED)
    for shuffle_seed in range(3):
        assert make_kfold_splits(shuffled(records, shuffle_seed), 5, seed=SEED) == expected


def test_kfold_is_stratified_by_class() -> None:
    per_class = {"pale": 10, "light_red": 34, "red": 12, "dark_red": 38, "purple": 15}
    records = make_records(per_class)
    for fold in make_kfold_splits(records, n_folds=5, seed=SEED):
        counts = labels_in(fold.held_out, records)
        assert set(counts) == set(CLASS_SLUGS), "各 fold の held_out に全クラスが含まれること"
        for label, n in per_class.items():
            assert abs(counts[label] - n / 5) <= 1, label


@pytest.mark.parametrize("n_folds", [1, 0, -1])
def test_kfold_rejects_n_folds_below_two(n_folds: int) -> None:
    with pytest.raises(ValueError):
        make_kfold_splits(make_records(10), n_folds=n_folds, seed=SEED)


def test_kfold_rejects_empty_records() -> None:
    with pytest.raises(ValueError):
        make_kfold_splits([], n_folds=5, seed=SEED)


def test_kfold_rejects_patient_with_conflicting_labels() -> None:
    records = make_records(10)
    conflict = ManifestRecord(
        image_path="purple_mild/extra.png",
        patient_id=records[0].patient_id,
        label="purple",
        severity="mild",
    )
    with pytest.raises(ValueError):
        make_kfold_splits([*records, conflict], n_folds=5, seed=SEED)


# --- 集計 ---


def test_patient_labels_maps_each_patient_once() -> None:
    records = make_records(3, images_per_patient=2)
    mapping = patient_labels(records)
    assert len(mapping) == 15
    for r in records:
        assert mapping[r.patient_id] == r.label


def test_count_by_class_lists_all_classes_in_id_order() -> None:
    records = make_records({"red": 2, "purple": 1}, images_per_patient=2)
    counts = count_by_class(records)
    assert list(counts) == list(CLASS_SLUGS)
    assert counts == {"pale": 0, "light_red": 0, "red": 4, "dark_red": 0, "purple": 2}


def test_count_by_class_severity_counts_pairs() -> None:
    records = make_records({"red": 3, "light_red": 2})
    counts = count_by_class_severity(records)
    assert counts == {("light_red", "normal"): 2, ("red", "strong"): 2, ("red", "mild"): 1}
    # クラス ID 順に並ぶ
    assert [label for label, _ in counts][0] == "light_red"