"""ストア(SPEC_01 基本方針 2・SPEC_02「assets」,testing.md 必須対象 2).
- 置き場所は `store/<SHA-256 の先頭 2 文字>/<SHA-256>.<拡張子>` で中身から決まる
- 同じ中身は 1 つにまとまる.コピー後にハッシュを照合する.既存のファイルを上書きしない
- 原本(取り込み元)を変更しない.置いたファイルは読み取り専用にする
- 例外は ValueError.メッセージにパス・元のファイル名を出さない
"""
import hashlib
import os
import stat
from pathlib import Path
import pytest
from tiasdata import store
from tiasdata.store import (
CHUNK_SIZE,
TEMP_PREFIX,
StoredFile,
StoredFileStatus,
add_file,
normalize_ext,
sha256_file,
store_path,
verify_stored,
)
SHA_A = "ab" + "0" * 62
def _sha(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def _all_files(directory: Path) -> list[Path]:
return sorted(p for p in directory.rglob("*") if p.is_file())
def _make_writable(path: Path) -> None:
path.chmod(path.stat().st_mode | stat.S_IWUSR)
def _write(path: Path, data: bytes) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
return path
def _assert_no_leak(message: str, tmp_path: Path, *names: str) -> None:
assert str(tmp_path) not in message
assert tmp_path.as_posix() not in message
for name in names:
assert name not in message
# ---------------------------------------------------------------- normalize_ext
class TestNormalizeExt:
@pytest.mark.parametrize(
("ext", "expected"),
[("png", "png"), (".png", "png"), (".PNG", "png"), ("JpG", "jpg"), (".tif2", "tif2")],
)
def test_normalizes_to_lower_without_dot(self, ext: str, expected: str) -> None:
assert normalize_ext(ext) == expected
@pytest.mark.parametrize("ext", ["", ".", "p-g", ".tar.gz", "pn g", "png/", "画像", "..png"])
def test_rejects_empty_or_non_alnum(self, ext: str) -> None:
with pytest.raises(ValueError):
normalize_ext(ext)
# ---------------------------------------------------------------- store_path
class TestStorePath:
def test_layout_is_first_two_chars_then_sha_and_ext(self, tmp_path: Path) -> None:
path = store_path(tmp_path, SHA_A, "png")
assert path == tmp_path / "ab" / f"{SHA_A}.png"
def test_ext_is_normalized(self, tmp_path: Path) -> None:
assert store_path(tmp_path, SHA_A, ".PNG") == tmp_path / "ab" / f"{SHA_A}.png"
def test_is_pure_and_creates_nothing(self, tmp_path: Path) -> None:
missing = tmp_path / "no-store"
store_path(missing, SHA_A, "png")
assert not missing.exists()
assert list(tmp_path.iterdir()) == []
@pytest.mark.parametrize(
"sha",
[
"",
"ab",
"AB" + "0" * 62, # 大文字
"a" * 63,
"a" * 65,
"g" * 64,
"../" + "a" * 61,
" " + "a" * 63,
],
)
def test_rejects_malformed_sha256(self, tmp_path: Path, sha: str) -> None:
with pytest.raises(ValueError):
store_path(tmp_path, sha, "png")
def test_rejects_bad_ext(self, tmp_path: Path) -> None:
with pytest.raises(ValueError):
store_path(tmp_path, SHA_A, "")
# ---------------------------------------------------------------- sha256_file
class TestSha256File:
def test_matches_hashlib(self, tmp_path: Path) -> None:
data = b"tongue image dummy"
assert sha256_file(_write(tmp_path / "a.bin", data)) == _sha(data)
def test_empty_file(self, tmp_path: Path) -> None:
assert sha256_file(_write(tmp_path / "e.bin", b"")) == _sha(b"")
@pytest.mark.parametrize(
"size", [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, 2 * CHUNK_SIZE + 7]
)
def test_across_chunk_boundary(self, tmp_path: Path, size: int) -> None:
data = bytes(i % 251 for i in range(size))
assert sha256_file(_write(tmp_path / "big.bin", data)) == _sha(data)
def test_does_not_modify_file(self, tmp_path: Path) -> None:
path = _write(tmp_path / "a.bin", b"abc")
before = path.stat()
sha256_file(path)
after = path.stat()
assert (after.st_mtime_ns, after.st_mode, after.st_size) == (
before.st_mtime_ns,
before.st_mode,
before.st_size,
)
# ---------------------------------------------------------------- add_file: 正常系
class TestAddFile:
def test_first_add_copies_to_sha_named_path(self, store_dir: Path, make_png) -> None:
source = make_png(store_dir.parent / "src" / "1.png", seed=1)
data = source.read_bytes()
result = add_file(store_dir, source)
assert isinstance(result, StoredFile)
assert result.sha256 == _sha(data)
assert result.ext == "png"
assert result.size_bytes == len(data)
assert result.copied is True
target = store_dir / result.sha256[:2] / f"{result.sha256}.png"
assert target == store_path(store_dir, result.sha256, result.ext)
assert target.read_bytes() == data
assert sha256_file(target) == result.sha256
assert _all_files(store_dir) == [target]
def test_second_add_of_same_file_does_not_copy(self, store_dir: Path, make_png) -> None:
source = make_png(store_dir.parent / "src" / "1.png", seed=1)
first = add_file(store_dir, source)
target = store_path(store_dir, first.sha256, first.ext)
mtime = target.stat().st_mtime_ns
second = add_file(store_dir, source)
assert second == first._replace(copied=False)
assert _all_files(store_dir) == [target]
assert target.stat().st_mtime_ns == mtime
def test_same_content_elsewhere_is_deduplicated(self, store_dir: Path, make_png) -> None:
a = make_png(store_dir.parent / "src" / "淡紅(正常)" / "1.png", seed=5)
b = make_png(store_dir.parent / "other" / "別名.png", seed=5)
assert a.read_bytes() == b.read_bytes()
ra = add_file(store_dir, a)
rb = add_file(store_dir, b)
assert ra.copied is True
assert rb.copied is False
assert ra.sha256 == rb.sha256
assert len(_all_files(store_dir)) == 1
def test_same_content_with_upper_ext_is_deduplicated(self, store_dir: Path, make_png) -> None:
a = make_png(store_dir.parent / "src" / "1.png", seed=3)
b = _write(store_dir.parent / "src2" / "1.PNG", a.read_bytes())
ra = add_file(store_dir, a)
rb = add_file(store_dir, b)
assert rb.ext == "png"
assert rb.copied is False
assert ra.sha256 == rb.sha256
assert len(_all_files(store_dir)) == 1
def test_different_contents_are_stored_separately(self, store_dir: Path, tongue_color_source):
results = [add_file(store_dir, img.path) for img in tongue_color_source.images]
assert all(r.copied for r in results)
assert len({r.sha256 for r in results}) == len(results)
files = _all_files(store_dir)
assert len(files) == len(results)
for r in results:
assert verify_stored(store_dir, r.sha256, r.ext) is StoredFileStatus.OK
def test_stored_file_is_read_only(self, store_dir: Path, make_png) -> None:
source = make_png(store_dir.parent / "src" / "1.png", seed=1)
result = add_file(store_dir, source)
target = store_path(store_dir, result.sha256, result.ext)
assert not os.access(target, os.W_OK)
assert not target.stat().st_mode & stat.S_IWUSR
with pytest.raises(PermissionError):
target.write_bytes(b"x")
assert sha256_file(target) == result.sha256
def test_no_temp_files_remain(self, store_dir: Path, tongue_color_source) -> None:
for img in tongue_color_source.images:
add_file(store_dir, img.path)
leftovers = [p for p in store_dir.rglob("*") if p.name.startswith(TEMP_PREFIX)]
assert leftovers == []
def test_large_file_across_chunks(self, store_dir: Path) -> None:
data = os.urandom(2 * CHUNK_SIZE + 123)
source = _write(store_dir.parent / "src" / "big.bin", data)
result = add_file(store_dir, source)
assert result.sha256 == _sha(data)
assert result.size_bytes == len(data)
assert store_path(store_dir, result.sha256, "bin").read_bytes() == data
def test_empty_file_can_be_stored(self, store_dir: Path) -> None:
source = _write(store_dir.parent / "src" / "empty.png", b"")
result = add_file(store_dir, source)
assert result.sha256 == _sha(b"")
assert result.size_bytes == 0
assert verify_stored(store_dir, result.sha256, "png") is StoredFileStatus.OK
# ---------------------------------------------------------------- add_file: 原本を変えない
class TestAddFileKeepsSource:
def test_source_content_mtime_mode_unchanged(self, store_dir: Path, make_png) -> None:
source = make_png(store_dir.parent / "src" / "1.png", seed=1)
os.utime(source, ns=(1_600_000_000_000_000_000, 1_600_000_000_000_000_000))
data = source.read_bytes()
before = source.stat()
add_file(store_dir, source)
add_file(store_dir, source) # 既にある場合の経路でも
after = source.stat()
assert source.read_bytes() == data
assert after.st_mtime_ns == before.st_mtime_ns
assert after.st_mode == before.st_mode
assert after.st_size == before.st_size
assert os.access(source, os.W_OK) # 原本に読み取り専用を付けない
def test_read_only_source_can_be_added_and_stays_read_only(
self, store_dir: Path, make_png
) -> None:
source = make_png(store_dir.parent / "src" / "1.png", seed=2)
source.chmod(stat.S_IREAD)
data = source.read_bytes()
before = source.stat()
try:
result = add_file(store_dir, source)
after = source.stat()
assert result.copied is True
assert source.read_bytes() == data
assert after.st_mode == before.st_mode
assert after.st_mtime_ns == before.st_mtime_ns
assert not os.access(source, os.W_OK)
finally:
_make_writable(source)
def test_source_folder_gets_no_new_files(self, store_dir: Path, tongue_color_source) -> None:
root = tongue_color_source.root
before = _all_files(root)
for img in tongue_color_source.images:
add_file(store_dir, img.path)
assert _all_files(root) == before
# ---------------------------------------------------------------- add_file: 異常系
class TestAddFileErrors:
def test_missing_store_dir_raises_and_creates_nothing(self, tmp_path: Path, make_png) -> None:
source = make_png(tmp_path / "src" / "1.png", seed=1)
missing = tmp_path / "tiasdata" / "store"
with pytest.raises(ValueError) as exc:
add_file(missing, source)
assert not missing.exists()
assert not missing.parent.exists()
_assert_no_leak(str(exc.value), tmp_path, "1.png")
def test_store_dir_is_a_file_raises(self, tmp_path: Path, make_png) -> None:
source = make_png(tmp_path / "src" / "1.png", seed=1)
not_dir = _write(tmp_path / "store", b"")
with pytest.raises(ValueError):
add_file(not_dir, source)
def test_source_is_directory_raises(self, store_dir: Path, tmp_path: Path) -> None:
folder = tmp_path / "src" / "淡紅(正常).png"
folder.mkdir(parents=True)
with pytest.raises(ValueError) as exc:
add_file(store_dir, folder)
assert _all_files(store_dir) == []
_assert_no_leak(str(exc.value), tmp_path, "淡紅(正常)")
def test_source_missing_raises(self, store_dir: Path, tmp_path: Path) -> None:
with pytest.raises(ValueError) as exc:
add_file(store_dir, tmp_path / "src" / "患者ABC123.png")
assert _all_files(store_dir) == []
_assert_no_leak(str(exc.value), tmp_path, "患者ABC123")
@pytest.mark.parametrize("name", ["noext", "weird.p-g", "trailing."])
def test_source_with_bad_ext_raises(self, store_dir: Path, tmp_path: Path, name: str) -> None:
source = _write(tmp_path / "src" / name, b"data")
with pytest.raises(ValueError) as exc:
add_file(store_dir, source)
assert list(store_dir.iterdir()) == []
_assert_no_leak(str(exc.value), tmp_path, name)
assert source.read_bytes() == b"data"
def test_existing_different_content_is_not_overwritten(
self, store_dir: Path, tmp_path: Path, make_png
) -> None:
source = make_png(tmp_path / "src" / "患者XYZ999.png", seed=1)
sha = sha256_file(source)
target = store_path(store_dir, sha, "png")
_write(target, b"corrupted content")
target.chmod(stat.S_IREAD)
before = target.stat()
with pytest.raises(ValueError) as exc:
add_file(store_dir, source)
assert target.read_bytes() == b"corrupted content"
assert target.stat().st_mtime_ns == before.st_mtime_ns
assert _all_files(store_dir) == [target]
message = str(exc.value)
assert sha in message
_assert_no_leak(message, tmp_path, "患者XYZ999")
def test_existing_directory_at_target_raises(
self, store_dir: Path, tmp_path: Path, make_png
) -> None:
source = make_png(tmp_path / "src" / "1.png", seed=1)
target = store_path(store_dir, sha256_file(source), "png")
target.mkdir(parents=True)
with pytest.raises(ValueError):
add_file(store_dir, source)
assert target.is_dir()
def test_verify_failure_after_copy_leaves_no_file(
self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
) -> None:
source = make_png(tmp_path / "src" / "患者QQQ.png", seed=1)
sha = sha256_file(source)
real = store.sha256_file
def broken(path: Path) -> str:
# 一時ファイルの読み直しだけ不一致にする(ディスクへの書き込みの誤りを模す)
if path.name.startswith(TEMP_PREFIX):
return "0" * 64
return real(path)
monkeypatch.setattr(store, "sha256_file", broken)
with pytest.raises(ValueError) as exc:
add_file(store_dir, source)
assert not store_path(store_dir, sha, "png").exists()
assert [p for p in store_dir.rglob("*") if p.is_file()] == []
_assert_no_leak(str(exc.value), tmp_path, "患者QQQ")
def test_source_changed_during_copy_stops(
self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
) -> None:
source = make_png(tmp_path / "src" / "1.png", seed=1)
fake = "cd" + "1" * 62
real = store.sha256_file
def first_call_differs(path: Path) -> str:
# 1 回目(取り込み元)の計算の後に中身が変わった状況を模す
if path == source:
return fake
return real(path)
monkeypatch.setattr(store, "sha256_file", first_call_differs)
with pytest.raises(ValueError) as exc:
add_file(store_dir, source)
assert _all_files(store_dir) == []
_assert_no_leak(str(exc.value), tmp_path, "1.png")
def test_concurrent_same_content_is_reused(
self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
) -> None:
source = make_png(tmp_path / "src" / "1.png", seed=1)
data = source.read_bytes()
sha = _sha(data)
target = store_path(store_dir, sha, "png")
real_copy = store._copy_to_temp
def copy_then_race(src: Path, directory: Path):
result = real_copy(src, directory)
_write(target, data) # コピーの間に別の取り込みが同じ中身を置いた
return result
monkeypatch.setattr(store, "_copy_to_temp", copy_then_race)
result = add_file(store_dir, source)
assert result.copied is False
assert result.sha256 == sha
assert _all_files(store_dir) == [target]
def test_concurrent_different_content_is_not_overwritten(
self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
) -> None:
source = make_png(tmp_path / "src" / "1.png", seed=1)
target = store_path(store_dir, sha256_file(source), "png")
real_copy = store._copy_to_temp
def copy_then_race(src: Path, directory: Path):
result = real_copy(src, directory)
_write(target, b"other")
return result
monkeypatch.setattr(store, "_copy_to_temp", copy_then_race)
with pytest.raises(ValueError):
add_file(store_dir, source)
assert target.read_bytes() == b"other"
assert _all_files(store_dir) == [target]
# ---------------------------------------------------------------- add_file: 再利用時の読み取り専用
def _is_read_only(path: Path) -> bool:
return not os.access(path, os.W_OK) and not path.stat().st_mode & stat.S_IWUSR
class TestAddFileReuseRestoresReadOnly:
"""SPEC_01「ストアの規則」: 再利用するときも読み取り専用を付け直す(中身には触らない)."""
def test_reuse_restores_read_only_when_removed(self, store_dir: Path, make_png) -> None:
source = make_png(store_dir.parent / "src" / "1.png", seed=1)
first = add_file(store_dir, source)
target = store_path(store_dir, first.sha256, first.ext)
data = target.read_bytes()
_make_writable(target) # 復元・手作業等で読み取り専用が外れた状況
assert os.access(target, os.W_OK)
mtime = target.stat().st_mtime_ns
try:
second = add_file(store_dir, source)
assert second == first._replace(copied=False)
assert _is_read_only(target)
assert target.read_bytes() == data
assert sha256_file(target) == first.sha256
assert target.stat().st_mtime_ns == mtime
assert _all_files(store_dir) == [target]
finally:
_make_writable(target)
def test_reuse_of_already_read_only_file_keeps_it_read_only(
self, store_dir: Path, make_png
) -> None:
source = make_png(store_dir.parent / "src" / "1.png", seed=2)
first = add_file(store_dir, source)
target = store_path(store_dir, first.sha256, first.ext)
assert _is_read_only(target)
second = add_file(store_dir, source)
third = add_file(store_dir, source)
assert second.copied is False
assert third.copied is False
assert _is_read_only(target)
assert sha256_file(target) == first.sha256
assert _all_files(store_dir) == [target]
def test_reuse_from_other_source_with_same_content_restores_read_only(
self, store_dir: Path, make_png
) -> None:
a = make_png(store_dir.parent / "src" / "1.png", seed=3)
b = _write(store_dir.parent / "other" / "別名.PNG", a.read_bytes())
first = add_file(store_dir, a)
target = store_path(store_dir, first.sha256, first.ext)
_make_writable(target)
try:
result = add_file(store_dir, b)
assert result.copied is False
assert _is_read_only(target)
finally:
_make_writable(target)
def test_concurrent_reuse_makes_existing_read_only(
self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
) -> None:
source = make_png(tmp_path / "src" / "1.png", seed=1)
data = source.read_bytes()
sha = _sha(data)
target = store_path(store_dir, sha, "png")
real_copy = store._copy_to_temp
def copy_then_race(src: Path, directory: Path):
result = real_copy(src, directory)
# 別の取り込みが同じ中身を置いた(まだ読み取り専用を付ける前の状態)
_write(target, data)
assert os.access(target, os.W_OK)
return result
monkeypatch.setattr(store, "_copy_to_temp", copy_then_race)
try:
result = add_file(store_dir, source)
assert result.copied is False
assert result.sha256 == sha
assert result.size_bytes == len(data)
assert _is_read_only(target)
assert target.read_bytes() == data
assert _all_files(store_dir) == [target]
finally:
if target.exists():
_make_writable(target)
def test_mismatch_keeps_existing_content_and_writable_attribute(
self, store_dir: Path, tmp_path: Path, make_png
) -> None:
source = make_png(tmp_path / "src" / "患者XYZ999.png", seed=1)
sha = sha256_file(source)
target = store_path(store_dir, sha, "png")
_write(target, b"corrupted content")
assert os.access(target, os.W_OK)
before = target.stat()
with pytest.raises(ValueError) as exc:
add_file(store_dir, source)
after = target.stat()
assert target.read_bytes() == b"corrupted content"
assert after.st_mode == before.st_mode
assert after.st_mtime_ns == before.st_mtime_ns
assert os.access(target, os.W_OK) # 照合が合わないときは属性も変えない
assert _all_files(store_dir) == [target]
_assert_no_leak(str(exc.value), tmp_path, "患者XYZ999")
def test_concurrent_mismatch_keeps_existing_writable(
self, store_dir: Path, tmp_path: Path, make_png, monkeypatch: pytest.MonkeyPatch
) -> None:
source = make_png(tmp_path / "src" / "1.png", seed=1)
target = store_path(store_dir, sha256_file(source), "png")
real_copy = store._copy_to_temp
def copy_then_race(src: Path, directory: Path):
result = real_copy(src, directory)
_write(target, b"other")
return result
monkeypatch.setattr(store, "_copy_to_temp", copy_then_race)
with pytest.raises(ValueError):
add_file(store_dir, source)
assert target.read_bytes() == b"other"
assert os.access(target, os.W_OK)
assert target.stat().st_mode & stat.S_IWUSR
assert _all_files(store_dir) == [target]
# ---------------------------------------------------------------- verify_stored
class TestVerifyStored:
def test_ok(self, store_dir: Path, make_png) -> None:
result = add_file(store_dir, make_png(store_dir.parent / "s" / "1.png", seed=1))
assert verify_stored(store_dir, result.sha256, result.ext) is StoredFileStatus.OK
assert verify_stored(store_dir, result.sha256, ".PNG") is StoredFileStatus.OK
def test_missing(self, store_dir: Path) -> None:
assert verify_stored(store_dir, SHA_A, "png") is StoredFileStatus.MISSING
def test_missing_when_only_other_ext_exists(self, store_dir: Path, make_png) -> None:
result = add_file(store_dir, make_png(store_dir.parent / "s" / "1.png", seed=1))
assert verify_stored(store_dir, result.sha256, "jpg") is StoredFileStatus.MISSING
def test_mismatch(self, store_dir: Path, make_png) -> None:
result = add_file(store_dir, make_png(store_dir.parent / "s" / "1.png", seed=1))
target = store_path(store_dir, result.sha256, result.ext)
_make_writable(target)
target.write_bytes(b"tampered")
assert verify_stored(store_dir, result.sha256, result.ext) is StoredFileStatus.MISMATCH
def test_status_values(self) -> None:
assert [s.value for s in StoredFileStatus] == ["ok", "missing", "mismatch"]
def test_malformed_sha_raises(self, store_dir: Path) -> None:
with pytest.raises(ValueError):
verify_stored(store_dir, "XYZ", "png")