Newer
Older
TIASDataHub / tests / test_check.py
"""目録とストアの点検(`tiasdata check`.SPEC_03「CLI」・PLAN_02 Step 3).

- ストアのファイルのハッシュの照合,目録にあってストアに無いファイル・その逆,所見の参照切れ,
  凍結した版の構成の欠け,読み取り専用が外れたストアのファイルを数える
- 書きかけの一時ファイル・想定外のファイルも問題として数える(ユーザー承認済み)
- 止めずに数え,問題があれば終了コード 0 以外(error-handling.md)
- 件数だけを表示する(パスを出さない).対応表を開かない
"""

import hashlib
import sqlite3
import stat
from contextlib import closing
from pathlib import Path

import pytest
from conftest import ImportEnv

from tiasdata.check import run_check
from tiasdata.cli import main
from tiasdata.config import load_config
from tiasdata.importers import run_import


def _setup(env: ImportEnv) -> None:
    assert main(["init"]) == 0
    run_import(load_config(env.config_path), env.source_name)


def _store(env: ImportEnv) -> Path:
    return env.catalog_root / "store"


def _check(env: ImportEnv):
    conn = sqlite3.connect(env.catalog_root / "catalog.sqlite")
    with closing(conn):
        return run_check(conn, _store(env))


def _stored_files(env: ImportEnv) -> list[Path]:
    return sorted(p for p in _store(env).rglob("*") if p.is_file())


def _make_writable(path: Path) -> None:
    path.chmod(path.stat().st_mode | stat.S_IWUSR)


def _make_read_only(path: Path) -> None:
    path.chmod(path.stat().st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))


def _put_unregistered(env: ImportEnv, data: bytes, ext: str = "png") -> Path:
    sha256 = hashlib.sha256(data).hexdigest()
    path = _store(env) / sha256[:2] / f"{sha256}.{ext}"
    path.parent.mkdir(exist_ok=True)
    path.write_bytes(data)
    _make_read_only(path)
    return path


def _raw_catalog(env: ImportEnv) -> sqlite3.Connection:
    # 外部キー制約を有効にしない接続(壊れた目録を作るため)
    return sqlite3.connect(env.catalog_root / "catalog.sqlite")


class TestHealthy:
    def test_empty_catalog_has_no_problems(self, import_env: ImportEnv) -> None:
        assert main(["init"]) == 0
        result = _check(import_env)
        assert result.catalog_files == 0
        assert result.store_files == 0
        assert result.problems == 0

    def test_after_import_has_no_problems(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        result = _check(import_env)
        assert result.catalog_files == 18
        assert result.hash_ok == 18
        assert result.store_files == 18
        assert result.hash_mismatch == 0
        assert result.missing_in_store == 0
        assert result.not_in_catalog == 0
        assert result.writable == 0
        assert result.temp_files == 0
        assert result.unexpected_in_store == 0
        assert result.dangling_annotations == 0
        assert result.broken_dataset_members == 0
        assert result.problems == 0

    def test_cli_exit_0_when_no_problems(
        self, import_env: ImportEnv, capsys: pytest.CaptureFixture[str]
    ) -> None:
        _setup(import_env)
        capsys.readouterr()
        assert main(["check"]) == 0


class TestProblems:
    def test_hash_mismatch(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        path = _stored_files(import_env)[0]
        _make_writable(path)
        path.write_bytes(b"tampered")
        _make_read_only(path)
        result = _check(import_env)
        assert result.hash_mismatch == 1
        assert result.hash_ok == 17
        assert result.problems == 1

    def test_missing_in_store(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        path = _stored_files(import_env)[0]
        _make_writable(path)
        path.unlink()
        result = _check(import_env)
        assert result.missing_in_store == 1
        assert result.hash_ok == 17
        assert result.problems == 1

    def test_not_in_catalog(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        _put_unregistered(import_env, b"not registered")
        result = _check(import_env)
        assert result.not_in_catalog == 1
        assert result.store_files == 19
        assert result.problems == 1

    def test_registered_content_with_other_ext_is_not_in_catalog(
        self, import_env: ImportEnv
    ) -> None:
        # ストアでは拡張子が違うと別のファイル(SPEC_02「assets」)
        _setup(import_env)
        data = import_env.source.images[0].path.read_bytes()
        _put_unregistered(import_env, data, ext="jpg")
        result = _check(import_env)
        assert result.not_in_catalog == 1
        assert result.problems == 1

    def test_writable_store_file(self, import_env: ImportEnv) -> None:
        # PLAN_02 Step 3: 読み取り専用が外れたストアのファイルの数を報告する
        _setup(import_env)
        files = _stored_files(import_env)
        _make_writable(files[0])
        _make_writable(files[1])
        result = _check(import_env)
        assert result.writable == 2
        assert result.hash_ok == 18
        assert result.problems == 2

    def test_temp_file(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        prefix_dir = _stored_files(import_env)[0].parent
        (prefix_dir / ".tmp-abc123").write_bytes(b"partial")
        result = _check(import_env)
        assert result.temp_files == 1
        assert result.store_files == 18
        assert result.problems == 1

    @pytest.mark.parametrize(
        "relative",
        ["README.txt", "zz/file.png", "ab/notes.txt", "ab/" + "c" * 64 + ".png"],
        ids=["file-at-top", "bad-prefix-dir", "bad-name", "prefix-mismatch"],
    )
    def test_unexpected_file(self, import_env: ImportEnv, relative: str) -> None:
        _setup(import_env)
        path = _store(import_env) / relative
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_bytes(b"x")
        result = _check(import_env)
        assert result.unexpected_in_store == 1
        assert result.problems >= 1

    def test_unexpected_directory(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        (_store(import_env) / "subdir").mkdir()
        result = _check(import_env)
        assert result.unexpected_in_store == 1

    def test_dangling_annotation(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        with closing(_raw_catalog(import_env)) as conn:
            conn.execute(
                "INSERT INTO annotations (kind, capture_id, value, source, created_at)"
                " VALUES ('tongue_color', 'C999999',"
                ' \'{"label":"red","severity":"mild"}\', \'test\', \'2026-10-06T00:00:00Z\')'
            )
            conn.commit()
        result = _check(import_env)
        assert result.dangling_annotations == 1
        assert result.problems == 1

    def test_broken_dataset_member(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        with closing(_raw_catalog(import_env)) as conn:
            conn.execute(
                "INSERT INTO dataset_members (name, version, sha256) VALUES ('ds', 1, ?)",
                ("f" * 64,),
            )
            conn.commit()
        result = _check(import_env)
        assert result.broken_dataset_members >= 1
        assert result.problems >= 1

    def test_all_problems_are_counted_without_stopping(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        files = _stored_files(import_env)
        _make_writable(files[0])
        files[0].unlink()  # 欠け
        _make_writable(files[1])
        files[1].write_bytes(b"tampered")  # 不一致(読み取り専用も外れている)
        _put_unregistered(import_env, b"extra")  # 余分
        (files[2].parent / ".tmp-x").write_bytes(b"")  # 一時ファイル
        (_store(import_env) / "junk.txt").write_bytes(b"")  # 想定外
        result = _check(import_env)
        assert result.missing_in_store == 1
        assert result.hash_mismatch == 1
        assert result.writable == 1
        assert result.not_in_catalog == 1
        assert result.temp_files == 1
        assert result.unexpected_in_store == 1
        assert result.problems == 6

    def test_cli_exit_nonzero_and_reports_count(
        self, import_env: ImportEnv, capsys: pytest.CaptureFixture[str]
    ) -> None:
        _setup(import_env)
        _make_writable(_stored_files(import_env)[0])
        capsys.readouterr()
        assert main(["check"]) != 0
        out = capsys.readouterr().out
        assert "読み取り専用が外れたストアのファイル: 1 件" in out


class TestCheckEnvironment:
    def test_check_does_not_need_identity_db(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        import_env.identity_path.unlink()
        assert main(["check"]) == 0
        assert not import_env.identity_path.exists()

    def test_check_does_not_modify_store(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        files = _stored_files(import_env)
        _make_writable(files[0])
        before = {p: (p.read_bytes(), p.stat().st_mode) for p in files}
        _check(import_env)
        assert {p: (p.read_bytes(), p.stat().st_mode) for p in _stored_files(import_env)} == before

    def test_missing_store_dir_is_error(
        self, import_env: ImportEnv, capsys: pytest.CaptureFixture[str]
    ) -> None:
        assert main(["init"]) == 0
        _store(import_env).rmdir()
        capsys.readouterr()
        assert main(["check"]) == 1
        err = capsys.readouterr().err
        assert len(err.strip().splitlines()) == 1

    def test_check_before_init_is_error(
        self, import_env: ImportEnv, capsys: pytest.CaptureFixture[str]
    ) -> None:
        assert main(["check"]) == 1
        assert main(["stats"]) == 1


class TestDatasetVersions:
    """凍結した版の manifest を構成から作り直して記録と照合する(SPEC_03「CLI」の check)."""

    def test_versions_without_mismatch(self, import_env: ImportEnv, datasets_env) -> None:
        _setup(import_env)
        assert main(["dataset", "freeze", "tongue_color"]) == 0
        result = _check(import_env)
        assert result.dataset_versions == 1
        assert result.manifest_mismatch == 0
        assert result.problems == 0

    def test_no_versions(self, import_env: ImportEnv) -> None:
        _setup(import_env)
        result = _check(import_env)
        assert result.dataset_versions == 0
        assert result.manifest_mismatch == 0

    def test_changed_member_is_counted_and_reported(
        self, import_env: ImportEnv, datasets_env, capsys: pytest.CaptureFixture[str]
    ) -> None:
        # 版の行はトリガで書き換えられないので,構成が参照するファイルの種類を書き換えて
        # 作り直した manifest が記録と食い違う状態を作る
        _setup(import_env)
        assert main(["dataset", "freeze", "tongue_color"]) == 0
        with closing(_raw_catalog(import_env)) as conn:
            conn.execute(
                "UPDATE assets SET kind = 'tongue_raw'"
                " WHERE sha256 = (SELECT MIN(sha256) FROM dataset_members)"
            )
            conn.commit()
        result = _check(import_env)
        assert result.dataset_versions == 1
        assert result.manifest_mismatch == 1
        assert result.problems == 1
        capsys.readouterr()
        assert main(["check"]) == 1
        out = capsys.readouterr().out
        assert "凍結した版の manifest の不一致: 1 件(版 1 件)" in out