Newer
Older
TIASDataHub / tests / test_store.py
"""ストア(SPEC_01 基本方針 2・SPEC_02「assets」,testing.md 必須対象 2).

- 置き場所は `store/<SHA-256 の先頭 2 文字>/<SHA-256>.<拡張子>` で中身から決まる
- 同じ中身は 1 つにまとまる.コピー後にハッシュを照合する.既存のファイルを上書きしない
- 原本(取り込み元)を変更しない.置いたファイルは読み取り専用にする
- 例外は ValueError.メッセージにパス・元のファイル名を出さない
"""

import hashlib
import os
import stat
from pathlib import Path

import pytest

from tiasdata import store
from tiasdata.store import (
    CHUNK_SIZE,
    TEMP_PREFIX,
    StoredFile,
    StoredFileStatus,
    add_file,
    normalize_ext,
    sha256_file,
    store_path,
    verify_stored,
)

SHA_A = "ab" + "0" * 62


def _sha(data: bytes) -> str:
    return hashlib.sha256(data).hexdigest()


def _all_files(directory: Path) -> list[Path]:
    return sorted(p for p in directory.rglob("*") if p.is_file())


def _make_writable(path: Path) -> None:
    path.chmod(path.stat().st_mode | stat.S_IWUSR)


def _write(path: Path, data: bytes) -> Path:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_bytes(data)
    return path


def _assert_no_leak(message: str, tmp_path: Path, *names: str) -> None:
    assert str(tmp_path) not in message
    assert tmp_path.as_posix() not in message
    for name in names:
        assert name not in message


# ---------------------------------------------------------------- normalize_ext


class TestNormalizeExt:
    @pytest.mark.parametrize(
        ("ext", "expected"),
        [("png", "png"), (".png", "png"), (".PNG", "png"), ("JpG", "jpg"), (".tif2", "tif2")],
    )
    def test_normalizes_to_lower_without_dot(self, ext: str, expected: str) -> None:
        assert normalize_ext(ext) == expected

    @pytest.mark.parametrize("ext", ["", ".", "p-g", ".tar.gz", "pn g", "png/", "画像", "..png"])
    def test_rejects_empty_or_non_alnum(self, ext: str) -> None:
        with pytest.raises(ValueError):
            normalize_ext(ext)


# ---------------------------------------------------------------- store_path


class TestStorePath:
    def test_layout_is_first_two_chars_then_sha_and_ext(self, tmp_path: Path) -> None:
        path = store_path(tmp_path, SHA_A, "png")
        assert path == tmp_path / "ab" / f"{SHA_A}.png"

    def test_ext_is_normalized(self, tmp_path: Path) -> None:
        assert store_path(tmp_path, SHA_A, ".PNG") == tmp_path / "ab" / f"{SHA_A}.png"

    def test_is_pure_and_creates_nothing(self, tmp_path: Path) -> None:
        missing = tmp_path / "no-store"
        store_path(missing, SHA_A, "png")
        assert not missing.exists()
        assert list(tmp_path.iterdir()) == []

    @pytest.mark.parametrize(
        "sha",
        [
            "",
            "ab",
            "AB" + "0" * 62,  # 大文字
            "a" * 63,
            "a" * 65,
            "g" * 64,
            "../" + "a" * 61,
            " " + "a" * 63,
        ],
    )
    def test_rejects_malformed_sha256(self, tmp_path: Path, sha: str) -> None:
        with pytest.raises(ValueError):
            store_path(tmp_path, sha, "png")

    def test_rejects_bad_ext(self, tmp_path: Path) -> None:
        with pytest.raises(ValueError):
            store_path(tmp_path, SHA_A, "")


# ---------------------------------------------------------------- sha256_file


class TestSha256File:
    def test_matches_hashlib(self, tmp_path: Path) -> None:
        data = b"tongue image dummy"
        assert sha256_file(_write(tmp_path / "a.bin", data)) == _sha(data)

    def test_empty_file(self, tmp_path: Path) -> None:
        assert sha256_file(_write(tmp_path / "e.bin", b"")) == _sha(b"")

    @pytest.mark.parametrize(
        "size", [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, 2 * CHUNK_SIZE + 7]
    )
    def test_across_chunk_boundary(self, tmp_path: Path, size: int) -> None:
        data = bytes(i % 251 for i in range(size))
        assert sha256_file(_write(tmp_path / "big.bin", data)) == _sha(data)

    def test_does_not_modify_file(self, tmp_path: Path) -> None:
        path = _write(tmp_path / "a.bin", b"abc")
        before = path.stat()
        sha256_file(path)
        after = path.stat()
        assert (after.st_mtime_ns, after.st_mode, after.st_size) == (
            before.st_mtime_ns,
            before.st_mode,
            before.st_size,
        )


# ---------------------------------------------------------------- add_file: 正常系


class TestAddFile:
    def test_first_add_copies_to_sha_named_path(self, store_dir: Path, make_png) -> None:
        source = make_png(store_dir.parent / "src" / "1.png", seed=1)
        data = source.read_bytes()

        result = add_file(store_dir, source)

        assert isinstance(result, StoredFile)
        assert result.sha256 == _sha(data)
        assert result.ext == "png"
        assert result.size_bytes == len(data)
        assert result.copied is True
        target = store_dir / result.sha256[:2] / f"{result.sha256}.png"
        assert target == store_path(store_dir, result.sha256, result.ext)
        assert target.read_bytes() == data
        assert sha256_file(target) == result.sha256
        assert _all_files(store_dir) == [target]

    def test_second_add_of_same_file_does_not_copy(self, store_dir: Path, make_png) -> None:
        source = make_png(store_dir.parent / "src" / "1.png", seed=1)
        first = add_file(store_dir, source)
        target = store_path(store_dir, first.sha256, first.ext)
        mtime = target.stat().st_mtime_ns

        second = add_file(store_dir, source)

        assert second == first._replace(copied=False)
        assert _all_files(store_dir) == [target]
        assert target.stat().st_mtime_ns == mtime

    def test_same_content_elsewhere_is_deduplicated(self, store_dir: Path, make_png) -> None:
        a = make_png(store_dir.parent / "src" / "淡紅(正常)" / "1.png", seed=5)
        b = make_png(store_dir.parent / "other" / "別名.png", seed=5)
        assert a.read_bytes() == b.read_bytes()

        ra = add_file(store_dir, a)
        rb = add_file(store_dir, b)

        assert ra.copied is True
        assert rb.copied is False
        assert ra.sha256 == rb.sha256
        assert len(_all_files(store_dir)) == 1

    def test_same_content_with_upper_ext_is_deduplicated(self, store_dir: Path, make_png) -> None:
        a = make_png(store_dir.parent / "src" / "1.png", seed=3)
        b = _write(store_dir.parent / "src2" / "1.PNG", a.read_bytes())

        ra = add_file(store_dir, a)
        rb = add_file(store_dir, b)

        assert rb.ext == "png"
        assert rb.copied is False
        assert ra.sha256 == rb.sha256
        assert len(_all_files(store_dir)) == 1

    def test_different_contents_are_stored_separately(self, store_dir: Path, tongue_color_source):
        results = [add_file(store_dir, img.path) for img in tongue_color_source.images]

        assert all(r.copied for r in results)
        assert len({r.sha256 for r in results}) == len(results)
        files = _all_files(store_dir)
        assert len(files) == len(results)
        for r in results:
            assert verify_stored(store_dir, r.sha256, r.ext) is StoredFileStatus.OK

    def test_stored_file_is_read_only(self, store_dir: Path, make_png) -> None:
        source = make_png(store_dir.parent / "src" / "1.png", seed=1)
        result = add_file(store_dir, source)
        target = store_path(store_dir, result.sha256, result.ext)

        assert not os.access(target, os.W_OK)
        assert not target.stat().st_mode & stat.S_IWUSR
        with pytest.raises(PermissionError):
            target.write_bytes(b"x")
        assert sha256_file(target) == result.sha256

    def test_no_temp_files_remain(self, store_dir: Path, tongue_color_source) -> None:
        for img in tongue_color_source.images:
            add_file(store_dir, img.path)
        leftovers = [p for p in store_dir.rglob("*") if p.name.startswith(TEMP_PREFIX)]
        assert leftovers == []

    def test_large_file_across_chunks(self, store_dir: Path) -> None:
        data = os.urandom(2 * CHUNK_SIZE + 123)
        source = _write(store_dir.parent / "src" / "big.bin", data)

        result = add_file(store_dir, source)

        assert result.sha256 == _sha(data)
        assert result.size_bytes == len(data)
        assert store_path(store_dir, result.sha256, "bin").read_bytes() == data

    def test_empty_file_can_be_stored(self, store_dir: Path) -> None:
        source = _write(store_dir.parent / "src" / "empty.png", b"")
        result = add_file(store_dir, source)
        assert result.sha256 == _sha(b"")
        assert result.size_bytes == 0
        assert verify_stored(store_dir, result.sha256, "png") is StoredFileStatus.OK


# ---------------------------------------------------------------- add_file: 原本を変えない


class TestAddFileKeepsSource:
    def test_source_content_mtime_mode_unchanged(self, store_dir: Path, make_png) -> None:
        source = make_png(store_dir.parent / "src" / "1.png", seed=1)
        os.utime(source, ns=(1_600_000_000_000_000_000, 1_600_000_000_000_000_000))
        data = source.read_bytes()
        before = source.stat()

        add_file(store_dir, source)
        add_file(store_dir, source)  # 既にある場合の経路でも

        after = source.stat()
        assert source.read_bytes() == data
        assert after.st_mtime_ns == before.st_mtime_ns
        assert after.st_mode == before.st_mode
        assert after.st_size == before.st_size
        assert os.access(source, os.W_OK)  # 原本に読み取り専用を付けない

    def test_read_only_source_can_be_added_and_stays_read_only(
        self, store_dir: Path, make_png
    ) -> None:
        source = make_png(store_dir.parent / "src" / "1.png", seed=2)
        source.chmod(stat.S_IREAD)
        data = source.read_bytes()
        before = source.stat()
        try:
            result = add_file(store_dir, source)

            after = source.stat()
            assert result.copied is True
            assert source.read_bytes() == data
            assert after.st_mode == before.st_mode
            assert after.st_mtime_ns == before.st_mtime_ns
            assert not os.access(source, os.W_OK)
        finally:
            _make_writable(source)

    def test_source_folder_gets_no_new_files(self, store_dir: Path, tongue_color_source) -> None:
        root = tongue_color_source.root
        before = _all_files(root)
        for img in tongue_color_source.images:
            add_file(store_dir, img.path)
        assert _all_files(root) == before


# ---------------------------------------------------------------- add_file: 異常系


class TestAddFileErrors:
    def test_missing_store_dir_raises_and_creates_nothing(self, tmp_path: Path, make_png) -> None:
        source = make_png(tmp_path / "src" / "1.png", seed=1)
        missing = tmp_path / "tiasdata" / "store"

        with pytest.raises(ValueError) as exc:
            add_file(missing, source)

        assert not missing.exists()
        assert not missing.parent.exists()
        _assert_no_leak(str(exc.value), tmp_path, "1.png")

    def test_store_dir_is_a_file_raises(self, tmp_path: Path, make_png) -> None:
        source = make_png(tmp_path / "src" / "1.png", seed=1)
        not_dir = _write(tmp_path / "store", b"")
        with pytest.raises(ValueError):
            add_file(not_dir, source)

    def test_source_is_directory_raises(self, store_dir: Path, tmp_path: Path) -> None:
        folder = tmp_path / "src" / "淡紅(正常).png"
        folder.mkdir(parents=True)
        with pytest.raises(ValueError) as exc:
            add_file(store_dir, folder)
        assert _all_files(store_dir) == []
        _assert_no_leak(str(exc.value), tmp_path, "淡紅(正常)")

    def test_source_missing_raises(self, store_dir: Path, tmp_path: Path) -> None:
        with pytest.raises(ValueError) as exc:
            add_file(store_dir, tmp_path / "src" / "患者ABC123.png")
        assert _all_files(store_dir) == []
        _assert_no_leak(str(exc.value), tmp_path, "患者ABC123")

    @pytest.mark.parametrize("name", ["noext", "weird.p-g", "trailing."])
    def test_source_with_bad_ext_raises(self, store_dir: Path, tmp_path: Path, name: str) -> None:
        source = _write(tmp_path / "src" / name, b"data")
        with pytest.raises(ValueError) as exc:
            add_file(store_dir, source)
        assert list(store_dir.iterdir()) == []
        _assert_no_leak(str(exc.value), tmp_path, name)
        assert source.read_bytes() == b"data"

    def test_existing_different_content_is_not_overwritten(
        self, store_dir: Path, tmp_path: Path, make_png
    ) -> None:
        source = make_png(tmp_path / "src" / "患者XYZ999.png", seed=1)
        sha = sha256_file(source)
        target = store_path(store_dir, sha, "png")
        _write(target, b"corrupted content")
        target.chmod(stat.S_IREAD)
        before = target.stat()

        with pytest.raises(ValueError) as exc:
            add_file(store_dir, source)

        assert target.read_bytes() == b"corrupted content"
        assert target.stat().st_mtime_ns == before.st_mtime_ns
        assert _all_files(store_dir) == [target]
        message = str(exc.value)
        assert sha in message
        _assert_no_leak(message, tmp_path, "患者XYZ999")

    def test_existing_directory_at_target_raises(
        self, store_dir: Path, tmp_path: Path, make_png
    ) -> None:
        source = make_png(tmp_path / "src" / "1.png", seed=1)
        target = store_path(store_dir, sha256_file(source), "png")
        target.mkdir(parents=True)
        with pytest.raises(ValueError):
            add_file(store_dir, source)
        assert target.is_dir()

    def test_verify_failure_after_copy_leaves_no_file(
        self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        source = make_png(tmp_path / "src" / "患者QQQ.png", seed=1)
        sha = sha256_file(source)
        real = store.sha256_file

        def broken(path: Path) -> str:
            # 一時ファイルの読み直しだけ不一致にする(ディスクへの書き込みの誤りを模す)
            if path.name.startswith(TEMP_PREFIX):
                return "0" * 64
            return real(path)

        monkeypatch.setattr(store, "sha256_file", broken)

        with pytest.raises(ValueError) as exc:
            add_file(store_dir, source)

        assert not store_path(store_dir, sha, "png").exists()
        assert [p for p in store_dir.rglob("*") if p.is_file()] == []
        _assert_no_leak(str(exc.value), tmp_path, "患者QQQ")

    def test_source_changed_during_copy_stops(
        self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        source = make_png(tmp_path / "src" / "1.png", seed=1)
        fake = "cd" + "1" * 62
        real = store.sha256_file

        def first_call_differs(path: Path) -> str:
            # 1 回目(取り込み元)の計算の後に中身が変わった状況を模す
            if path == source:
                return fake
            return real(path)

        monkeypatch.setattr(store, "sha256_file", first_call_differs)

        with pytest.raises(ValueError) as exc:
            add_file(store_dir, source)

        assert _all_files(store_dir) == []
        _assert_no_leak(str(exc.value), tmp_path, "1.png")

    def test_concurrent_same_content_is_reused(
        self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        source = make_png(tmp_path / "src" / "1.png", seed=1)
        data = source.read_bytes()
        sha = _sha(data)
        target = store_path(store_dir, sha, "png")
        real_copy = store._copy_to_temp

        def copy_then_race(src: Path, directory: Path):
            result = real_copy(src, directory)
            _write(target, data)  # コピーの間に別の取り込みが同じ中身を置いた
            return result

        monkeypatch.setattr(store, "_copy_to_temp", copy_then_race)

        result = add_file(store_dir, source)

        assert result.copied is False
        assert result.sha256 == sha
        assert _all_files(store_dir) == [target]

    def test_concurrent_different_content_is_not_overwritten(
        self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        source = make_png(tmp_path / "src" / "1.png", seed=1)
        target = store_path(store_dir, sha256_file(source), "png")
        real_copy = store._copy_to_temp

        def copy_then_race(src: Path, directory: Path):
            result = real_copy(src, directory)
            _write(target, b"other")
            return result

        monkeypatch.setattr(store, "_copy_to_temp", copy_then_race)

        with pytest.raises(ValueError):
            add_file(store_dir, source)

        assert target.read_bytes() == b"other"
        assert _all_files(store_dir) == [target]


# ---------------------------------------------------------------- add_file: 再利用時の読み取り専用


def _is_read_only(path: Path) -> bool:
    return not os.access(path, os.W_OK) and not path.stat().st_mode & stat.S_IWUSR


class TestAddFileReuseRestoresReadOnly:
    """SPEC_01「ストアの規則」: 再利用するときも読み取り専用を付け直す(中身には触らない)."""

    def test_reuse_restores_read_only_when_removed(self, store_dir: Path, make_png) -> None:
        source = make_png(store_dir.parent / "src" / "1.png", seed=1)
        first = add_file(store_dir, source)
        target = store_path(store_dir, first.sha256, first.ext)
        data = target.read_bytes()
        _make_writable(target)  # 復元・手作業等で読み取り専用が外れた状況
        assert os.access(target, os.W_OK)
        mtime = target.stat().st_mtime_ns

        try:
            second = add_file(store_dir, source)

            assert second == first._replace(copied=False)
            assert _is_read_only(target)
            assert target.read_bytes() == data
            assert sha256_file(target) == first.sha256
            assert target.stat().st_mtime_ns == mtime
            assert _all_files(store_dir) == [target]
        finally:
            _make_writable(target)

    def test_reuse_of_already_read_only_file_keeps_it_read_only(
        self, store_dir: Path, make_png
    ) -> None:
        source = make_png(store_dir.parent / "src" / "1.png", seed=2)
        first = add_file(store_dir, source)
        target = store_path(store_dir, first.sha256, first.ext)
        assert _is_read_only(target)

        second = add_file(store_dir, source)
        third = add_file(store_dir, source)

        assert second.copied is False
        assert third.copied is False
        assert _is_read_only(target)
        assert sha256_file(target) == first.sha256
        assert _all_files(store_dir) == [target]

    def test_reuse_from_other_source_with_same_content_restores_read_only(
        self, store_dir: Path, make_png
    ) -> None:
        a = make_png(store_dir.parent / "src" / "1.png", seed=3)
        b = _write(store_dir.parent / "other" / "別名.PNG", a.read_bytes())
        first = add_file(store_dir, a)
        target = store_path(store_dir, first.sha256, first.ext)
        _make_writable(target)

        try:
            result = add_file(store_dir, b)

            assert result.copied is False
            assert _is_read_only(target)
        finally:
            _make_writable(target)

    def test_concurrent_reuse_makes_existing_read_only(
        self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        source = make_png(tmp_path / "src" / "1.png", seed=1)
        data = source.read_bytes()
        sha = _sha(data)
        target = store_path(store_dir, sha, "png")
        real_copy = store._copy_to_temp

        def copy_then_race(src: Path, directory: Path):
            result = real_copy(src, directory)
            # 別の取り込みが同じ中身を置いた(まだ読み取り専用を付ける前の状態)
            _write(target, data)
            assert os.access(target, os.W_OK)
            return result

        monkeypatch.setattr(store, "_copy_to_temp", copy_then_race)

        try:
            result = add_file(store_dir, source)

            assert result.copied is False
            assert result.sha256 == sha
            assert result.size_bytes == len(data)
            assert _is_read_only(target)
            assert target.read_bytes() == data
            assert _all_files(store_dir) == [target]
        finally:
            if target.exists():
                _make_writable(target)

    def test_mismatch_keeps_existing_content_and_writable_attribute(
        self, store_dir: Path, tmp_path: Path, make_png
    ) -> None:
        source = make_png(tmp_path / "src" / "患者XYZ999.png", seed=1)
        sha = sha256_file(source)
        target = store_path(store_dir, sha, "png")
        _write(target, b"corrupted content")
        assert os.access(target, os.W_OK)
        before = target.stat()

        with pytest.raises(ValueError) as exc:
            add_file(store_dir, source)

        after = target.stat()
        assert target.read_bytes() == b"corrupted content"
        assert after.st_mode == before.st_mode
        assert after.st_mtime_ns == before.st_mtime_ns
        assert os.access(target, os.W_OK)  # 照合が合わないときは属性も変えない
        assert _all_files(store_dir) == [target]
        _assert_no_leak(str(exc.value), tmp_path, "患者XYZ999")

    def test_concurrent_mismatch_keeps_existing_writable(
        self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        source = make_png(tmp_path / "src" / "1.png", seed=1)
        target = store_path(store_dir, sha256_file(source), "png")
        real_copy = store._copy_to_temp

        def copy_then_race(src: Path, directory: Path):
            result = real_copy(src, directory)
            _write(target, b"other")
            return result

        monkeypatch.setattr(store, "_copy_to_temp", copy_then_race)

        with pytest.raises(ValueError):
            add_file(store_dir, source)

        assert target.read_bytes() == b"other"
        assert os.access(target, os.W_OK)
        assert target.stat().st_mode & stat.S_IWUSR
        assert _all_files(store_dir) == [target]


# ---------------------------------------------------------------- verify_stored


class TestVerifyStored:
    def test_ok(self, store_dir: Path, make_png) -> None:
        result = add_file(store_dir, make_png(store_dir.parent / "s" / "1.png", seed=1))
        assert verify_stored(store_dir, result.sha256, result.ext) is StoredFileStatus.OK
        assert verify_stored(store_dir, result.sha256, ".PNG") is StoredFileStatus.OK

    def test_missing(self, store_dir: Path) -> None:
        assert verify_stored(store_dir, SHA_A, "png") is StoredFileStatus.MISSING

    def test_missing_when_only_other_ext_exists(self, store_dir: Path, make_png) -> None:
        result = add_file(store_dir, make_png(store_dir.parent / "s" / "1.png", seed=1))
        assert verify_stored(store_dir, result.sha256, "jpg") is StoredFileStatus.MISSING

    def test_mismatch(self, store_dir: Path, make_png) -> None:
        result = add_file(store_dir, make_png(store_dir.parent / "s" / "1.png", seed=1))
        target = store_path(store_dir, result.sha256, result.ext)
        _make_writable(target)
        target.write_bytes(b"tampered")
        assert verify_stored(store_dir, result.sha256, result.ext) is StoredFileStatus.MISMATCH

    def test_status_values(self) -> None:
        assert [s.value for s in StoredFileStatus] == ["ok", "missing", "mismatch"]

    def test_malformed_sha_raises(self, store_dir: Path) -> None:
        with pytest.raises(ValueError):
            verify_stored(store_dir, "XYZ", "png")