Newer
Older
TIASDataHub / src / tiasdata / catalog / versions.py
"""データセットと版(`datasets`・`dataset_versions`・`dataset_members`・`dataset_member_annotations`).

版とその構成は INSERT だけを行い,UPDATE・DELETE しない(スキーマのトリガでも拒否される.
SPEC_02「スキーマで強制する規則」).そのため版の行は manifest の SHA-256 を計算してから
構成と一緒に INSERT する.
"""

import sqlite3

from pydantic import BaseModel, ConfigDict


class _Row(BaseModel):
    model_config = ConfigDict(frozen=True)


class MemberAssetRow(_Row):
    """版に入れる(入れた)ファイルと,その撮影・症例.

    Attributes:
        sha256: ファイルの SHA-256.
        ext: 拡張子.
        kind: ファイルの種類.
        capture_id: 撮影 ID(撮影にひも付かないファイルは None).
        subject_id: 研究用の症例 ID(撮影が無ければ None).
    """

    sha256: str
    ext: str
    kind: str
    capture_id: str | None
    subject_id: str | None


class AnnotationRow(_Row):
    """所見の行(版の評価に使う列だけ).

    Attributes:
        annotation_id: 所見の ID.
        kind: 所見の種類.
        capture_id: 撮影に付く所見の撮影 ID.
        sha256: ファイルに付く所見のファイルの SHA-256.
        value: 値(JSON の文字列).
    """

    annotation_id: int
    kind: str
    capture_id: str | None
    sha256: str | None
    value: str


class MemberAnnotationRow(_Row):
    """版の構成で,ファイルに対して採用した所見."""

    member_sha256: str
    annotation: AnnotationRow


class DatasetRecord(_Row):
    """`datasets` の 1 行."""

    name: str
    description: str | None
    created_at: str


class VersionRecord(_Row):
    """`dataset_versions` の 1 行."""

    name: str
    version: int
    definition: str
    hub_commit: str
    manifest_sha256: str
    created_at: str
    note: str | None


class VersionSummary(_Row):
    """版の行と構成の件数.

    Attributes:
        record: 版の行.
        members: 構成のファイルの数.
        member_annotations: 採用した所見の数.
    """

    record: VersionRecord
    members: int
    member_annotations: int


_VERSION_COLUMNS = "name, version, definition, hub_commit, manifest_sha256, created_at, note"


def _version(row: tuple) -> VersionRecord:
    return VersionRecord(
        name=row[0],
        version=row[1],
        definition=row[2],
        hub_commit=row[3],
        manifest_sha256=row[4],
        created_at=row[5],
        note=row[6],
    )


def _annotation(row: tuple) -> AnnotationRow:
    return AnnotationRow(
        annotation_id=row[0], kind=row[1], capture_id=row[2], sha256=row[3], value=row[4]
    )


def kind_categories(conn: sqlite3.Connection) -> dict[str, str]:
    """種類名 → 分類(`asset`・`annotation`)."""
    rows = conn.execute("SELECT kind, category FROM kinds ORDER BY kind")
    return {kind: category for kind, category in rows}


def list_assets_of_kind(conn: sqlite3.Connection, kind: str) -> list[MemberAssetRow]:
    """種類が一致するファイルと,その撮影・症例を SHA-256 の順に返す."""
    rows = conn.execute(
        "SELECT a.sha256, a.ext, a.kind, a.capture_id, c.subject_id FROM assets AS a"
        " LEFT JOIN captures AS c ON c.capture_id = a.capture_id"
        " WHERE a.kind = ? ORDER BY a.sha256",
        (kind,),
    )
    return [
        MemberAssetRow(sha256=r[0], ext=r[1], kind=r[2], capture_id=r[3], subject_id=r[4])
        for r in rows
    ]


def list_valid_annotations(conn: sqlite3.Connection, kind: str) -> list[AnnotationRow]:
    """種類が一致する有効な所見(訂正されていないもの)を ID の順に返す."""
    rows = conn.execute(
        "SELECT annotation_id, kind, capture_id, sha256, value FROM annotations"
        " WHERE kind = ? AND superseded_by IS NULL ORDER BY annotation_id",
        (kind,),
    )
    return [_annotation(r) for r in rows]


def get_dataset(conn: sqlite3.Connection, name: str) -> DatasetRecord | None:
    """データセットの行を引く(無ければ None)."""
    row = conn.execute(
        "SELECT name, description, created_at FROM datasets WHERE name = ?", (name,)
    ).fetchone()
    if row is None:
        return None
    return DatasetRecord(name=row[0], description=row[1], created_at=row[2])


def list_datasets(conn: sqlite3.Connection) -> list[DatasetRecord]:
    """データセットの行を名前の順に返す."""
    rows = conn.execute("SELECT name, description, created_at FROM datasets ORDER BY name")
    return [DatasetRecord(name=r[0], description=r[1], created_at=r[2]) for r in rows]


def insert_dataset(
    conn: sqlite3.Connection, name: str, description: str | None, created_at: str
) -> None:
    """データセットの行を追加する."""
    conn.execute(
        "INSERT INTO datasets (name, description, created_at) VALUES (?, ?, ?)",
        (name, description, created_at),
    )


def get_version(conn: sqlite3.Connection, name: str, version: int) -> VersionRecord | None:
    """版の行を引く(無ければ None)."""
    row = conn.execute(
        f"SELECT {_VERSION_COLUMNS} FROM dataset_versions WHERE name = ? AND version = ?",
        (name, version),
    ).fetchone()
    return None if row is None else _version(row)


def get_latest_version(conn: sqlite3.Connection, name: str) -> VersionRecord | None:
    """データセットの最新の版の行を引く(版が無ければ None)."""
    row = conn.execute(
        f"SELECT {_VERSION_COLUMNS} FROM dataset_versions WHERE name = ?"
        " ORDER BY version DESC LIMIT 1",
        (name,),
    ).fetchone()
    return None if row is None else _version(row)


def list_versions(conn: sqlite3.Connection, name: str | None = None) -> list[VersionSummary]:
    """版の行と構成の件数を,名前・版の順に返す.

    Args:
        conn: 目録への接続.
        name: データセット名(省略時はすべてのデータセット).

    Returns:
        版ごとの行と件数.
    """
    rows = conn.execute(
        "SELECT v.name, v.version, v.definition, v.hub_commit, v.manifest_sha256,"
        " v.created_at, v.note,"
        " (SELECT COUNT(*) FROM dataset_members AS m"
        "  WHERE m.name = v.name AND m.version = v.version),"
        " (SELECT COUNT(*) FROM dataset_member_annotations AS d"
        "  WHERE d.name = v.name AND d.version = v.version)"
        " FROM dataset_versions AS v"
        " WHERE ? IS NULL OR v.name = ?"
        " ORDER BY v.name, v.version",
        (name, name),
    )
    return [
        VersionSummary(record=_version(r[:7]), members=r[7], member_annotations=r[8]) for r in rows
    ]


def version_members(conn: sqlite3.Connection, name: str, version: int) -> list[MemberAssetRow]:
    """版の構成のファイルと,その撮影・症例を SHA-256 の順に返す.

    目録にファイルが無い構成の行は含めない(`check` が manifest の不一致として見つける).
    """
    rows = conn.execute(
        "SELECT a.sha256, a.ext, a.kind, a.capture_id, c.subject_id"
        " FROM dataset_members AS m"
        " JOIN assets AS a ON a.sha256 = m.sha256"
        " LEFT JOIN captures AS c ON c.capture_id = a.capture_id"
        " WHERE m.name = ? AND m.version = ? ORDER BY a.sha256",
        (name, version),
    )
    return [
        MemberAssetRow(sha256=r[0], ext=r[1], kind=r[2], capture_id=r[3], subject_id=r[4])
        for r in rows
    ]


def version_member_annotations(
    conn: sqlite3.Connection, name: str, version: int
) -> list[MemberAnnotationRow]:
    """版の構成で採用した所見を,ファイルの SHA-256・所見の ID の順に返す.

    訂正された所見(`superseded_by` あり)も版を作った時点の値のまま返す.
    """
    rows = conn.execute(
        "SELECT d.sha256, a.annotation_id, a.kind, a.capture_id, a.sha256, a.value"
        " FROM dataset_member_annotations AS d"
        " JOIN annotations AS a ON a.annotation_id = d.annotation_id"
        " WHERE d.name = ? AND d.version = ? ORDER BY d.sha256, a.annotation_id",
        (name, version),
    )
    return [MemberAnnotationRow(member_sha256=r[0], annotation=_annotation(r[1:])) for r in rows]


def insert_version(
    conn: sqlite3.Connection,
    record: VersionRecord,
    members: list[str],
    member_annotations: list[tuple[str, int]],
) -> None:
    """版の行と構成を追加する(呼び出し側のトランザクションの中で呼ぶ).

    Args:
        conn: 目録への接続.
        record: 版の行(manifest の SHA-256 を計算済み).
        members: 構成のファイルの SHA-256.
        member_annotations: 採用した所見(ファイルの SHA-256, 所見の ID).
    """
    conn.execute(
        f"INSERT INTO dataset_versions ({_VERSION_COLUMNS}) VALUES (?, ?, ?, ?, ?, ?, ?)",
        (
            record.name,
            record.version,
            record.definition,
            record.hub_commit,
            record.manifest_sha256,
            record.created_at,
            record.note,
        ),
    )
    conn.executemany(
        "INSERT INTO dataset_members (name, version, sha256) VALUES (?, ?, ?)",
        [(record.name, record.version, sha256) for sha256 in members],
    )
    conn.executemany(
        "INSERT INTO dataset_member_annotations (name, version, sha256, annotation_id)"
        " VALUES (?, ?, ?, ?)",
        [
            (record.name, record.version, sha256, annotation_id)
            for sha256, annotation_id in member_annotations
        ],
    )