using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.Eventing.Reader;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Runtime.Remoting.Channels;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Controls;
using System.Windows.Forms;
using System.Media;
using OpenCvSharp;
using OpenCvSharp.Aruco;
namespace TIASshot {
internal abstract class CameraBase {
// インターフェース
public abstract bool Connect();
public abstract void Disconnect();
protected abstract void Shot(int numImages=1, int interval=0);
// プロパティ
public string DeviceName { get; protected set; } = "Unknown";
public string SerialNumber { get; protected set; }
public string ErrorMsg { get; protected set; }
public bool IsCalibrating => _calibrating > 0;
// 派生クラスで使用するメンバ
protected Form1 _form;
protected int _calibrating = 0;
protected bool _calibrated = false;
protected Bitmap[] _bmps = new Bitmap[2];
protected int _bmpIndex = 0;
protected bool _isPreview = false;
protected List<Mat> _chartMasks = new List<Mat>();
protected Dictionary<int, Mat> _convRGB2SRGB = new Dictionary<int, Mat>();
protected string _saveFolder = "";
protected List<Mat> _shots = new List<Mat>();
protected Dictionary<string, string> _ShotInfo = new Dictionary<string, string>(); // 設定値保存用
protected Rect _roi;
// プライベートメンバ
readonly Dictionary ARDict = CvAruco.GetPredefinedDictionary(PredefinedDictionaryName.Dict4X4_50);
readonly Point2f[] PointsDst40 = new Point2f[] {
new Point2f(345, 130),new Point2f(465, 130),new Point2f(465, 250),new Point2f(345, 250),
};
readonly Point2f[] PointsDst41 = new Point2f[]{
new Point2f(345, 1200), new Point2f(465, 1200), new Point2f(465, 1320), new Point2f(345, 1320),
};
readonly int[][] ExtendChannels = new int[][]
{
new int[] { 3, 3, 3 }, // 1
new int[] { 0, 3, 3 }, // R
new int[] { 1, 3, 3 }, // G
new int[] { 2, 3, 3 }, // B
new int[] { 0, 1, 3 }, // RG
new int[] { 0, 2, 3 }, // RB
new int[] { 1, 2, 3 }, // GB
new int[] { 0, 0, 3 }, // RR
new int[] { 1, 1, 3 }, // GG
new int[] { 2, 2, 3 }, // BB
new int[] { 0, 0, 2 }, // RRB
new int[] { 0, 0, 1 }, // RRG
new int[] { 1, 1, 0 }, // GGR
new int[] { 1, 1, 2 }, // GGB
new int[] { 2, 2, 0 }, // BBR
new int[] { 2, 2, 1 }, // BBG
new int[] { 0, 1, 2 }, // RGB
};
readonly int[] ChannelList; // 処理するチャネル数
readonly Mat TCC_SRGB;
readonly Mat TCC_XYZ;
readonly float UpdateRate;
int _detectionCount = 0;
Point2f _lastPosition = new Point2f(0, 0);
/// <summary>
/// カメラの基本クラス
/// </summary>
public CameraBase(Form1 form) {
_form = form;
UpdateRate = Config.GetFloat("Calib/UpdateRate");
TCC_SRGB = LoadMatFromCsv(Config.GetString("File/TccSrgbRef"));
TCC_XYZ = LoadMatFromCsv(Config.GetString("File/TccXyzRef"));
ChannelList = Config.GetString("Calib/ConversionChannels")
.Split(',')
.Select(x => int.Parse(x.Trim()))
.Where(x => x >= 4 && x <= 17)
.Distinct()
.OrderBy(x => x)
.ToArray();
}
/// <summary>
/// 起動チェック
/// </summary>
/// <returns></returns>
protected bool BootCheck() {
if (!Config.IsLoaded()) {
ErrorMsg = "設定ファイル（Config.xml）の読み込みに失敗しました．\r\n終了します．";
return false;
}
if (TCC_SRGB is null) {
ErrorMsg = $"ファイル（{Config.GetString("File/TccSrgbRef")}）の読み込みに失敗しました．\r\n終了します．";
return false;
}
if (TCC_XYZ is null) {
ErrorMsg = $"ファイル（{Config.GetString("File/TccXyzRef")}）の読み込みに失敗しました．\r\n終了します．";
return false;
}
return true;
}
/// <summary>
/// 画像撮影１枚
/// </summary>
public void ShotOne() {
SetSaveFolder(_form.GetDataName());
SetInfo("データ名", _form.GetDataName());
SetInfo("撮影枚数", "1");
Shot();
EventSound(Config.GetString("Sound/OneShotDone"));
_form.ShowMessage("撮影終了");
}
/// <summary>
/// 複数画像撮影
/// </summary>
public void ShotMulti() {
SetSaveFolder(_form.GetDataName());
SetInfo("データ名", _form.GetDataName());
SetInfo("撮影枚数", $"{_form.GetNumMultiShots()}");
SetInfo("撮影間隔(ms)", $"{_form.GetMultiShotsInterval()}");
Shot(_form.GetNumMultiShots(), _form.GetMultiShotsInterval());
EventSound(Config.GetString("Sound/MultiShotDone"));
_form.ShowMessage("撮影終了");
}
/// <summary>
/// 画像保存処理
/// </summary>
/// <param name="img"></param>
/// <param name="idx"></param>
protected void SaveImages(Mat img, int idx) {
var filename = Config.GetString("File/RgbImage");
filename = filename.Replace("{NO}", $"{idx + 1:0000}");
Cv2.ImWrite(Path.Combine(_saveFolder, filename), img);
foreach (var channel in ChannelList) {
using (var converted = ConvertImage(img, _convRGB2SRGB[channel])) {
filename = GetFilenameWithChannel("File/SrgbImage", channel);
filename = filename.Replace("{NO}", $"{idx + 1:0000}");
Cv2.ImWrite(Path.Combine(_saveFolder, filename), converted);
}
}
}
/// <summary>
/// 画像保存スレッド処理
/// </summary>
/// <param name="numImages"></param>
protected void SaveThread(int numImages) {
int saveCount = 0;
while (saveCount < numImages) {
while(_shots.Count <= saveCount) {
Thread.Sleep(1);
}
SaveImages(_shots[saveCount], saveCount);
saveCount++;
_form.ShowMessage($"画像{saveCount}/{numImages}枚目保存");
}
_form.ShowMessage("全ての画像保存完了");
_form.EnableShots(true);
}
/// <summary>
/// チャートの検出
/// </summary>
/// <param name="img"></param>
protected void DetectChart(Mat img) {
// ARマーカー検出
CvAruco.DetectMarkers(img, ARDict, out var corners, out var ids,
new DetectorParameters(), out var rejectedImgPoints);
if (ids.Length < 1) return;
// マーカー座標格納
var ptsPict = new List<Point2f>();
var ptsModel = new List<Point2f>();
Point2f position = new Point2f();
float y40 = 0, y41 = 0;
for (int i = 0; i < ids.Length; i++) {
if (ids[i] == 40) {
ptsPict.AddRange(corners[i]);
ptsModel.AddRange(PointsDst40);
position = corners[i][3];
y40 = corners[i][3].Y;
}
if (ids[i] == 41) {
ptsPict.AddRange(corners[i]);
ptsModel.AddRange(PointsDst41);
y41 = corners[i][3].Y;
}
}
if (ptsPict.Count < 8) return;
if (y40 > y41) {
_form.ShowMessage("舌診チャートが上下逆方向です");
return;
}
// チャートの固定判定
_form.ShowMessage("舌診チャートの検出中");
var dist = (float)position.DistanceTo(_lastPosition);
if (dist < Config.GetFloat("Calib/ChartSetCriteria")) {
_detectionCount++;
} else {
_detectionCount = 0;
}
_lastPosition = position;
if (_detectionCount < Config.GetInt("Calib/ChartSetCount")) return;
// ホモグラフィの計算
var matPtsPict = Mat.FromArray(ptsPict);
var matPtsModel = Mat.FromArray(ptsModel);
var matH = Cv2.FindHomography(matPtsModel, matPtsPict);
var imgF = new Mat(1545, 810, MatType.CV_8UC3);
Cv2.WarpPerspective(img, imgF, matH, imgF.Size());
// チャートマスク作成
_chartMasks.Clear();
var roiSize = ptsPict.Count < 8 ? 60 : 80;
for (int i = 0; i < 24; i++) {
var row = i % 6;
var col = i / 6;
var x = 581 - col * 144 + (ptsPict.Count < 8 ? 10 : 0);
var y = 318 + row * 144 + (ptsPict.Count < 8 ? 10 : 0);
var roi = new Rect(x, y, roiSize, roiSize);
using (var mask = new Mat(1545, 810, MatType.CV_8U)) {
Cv2.Rectangle(mask, roi, new Scalar(255), Cv2.FILLED);
var maskF = new Mat(img.Size(), MatType.CV_8U);
Cv2.WarpPerspective(mask, maskF, matH, maskF.Size());
_chartMasks.Add(maskF);
}
}
_form.ShowMessage("舌診チャート検出 校正中");
_calibrating = Config.GetInt("Calib/Frames");
}
/// <summary>
/// ファイル名にチャネル数を追加する
/// </summary>
/// <param name="config"></param>
/// <param name="channel"></param>
/// <returns></returns>
private string GetFilenameWithChannel(string config, int channel) {
var filename = Config.GetString(config);
filename = filename.Replace("{CN}", $"{channel:00}");
return filename;
}
/// <summary>
/// チャートに基づく変換行列の計算
/// </summary>
/// <param name="img"></param>
protected void CalcTcc(Mat img) {
// 変換行列の計算
var tccRgb = GetTccRgb(img);
var imgRois = GetTccRoisImage(img);
// データ保存
SetSaveFolder("校正");
SetInfo("校正データ", _saveFolder);
Cv2.ImWrite(Path.Combine(_saveFolder, Config.GetString("File/TccImage")), img);
Cv2.ImWrite(Path.Combine(_saveFolder, Config.GetString("File/TccRoisImage")), imgRois);
SaveMatToCsv(Path.Combine(_saveFolder, Config.GetString("File/TccRgbValues")), tccRgb);
_convRGB2SRGB.Clear();
foreach (var channel in ChannelList) {
var convRGB2SRGB = CalcConvertMatrix(tccRgb, TCC_SRGB, channel);
SaveMatToCsv(Path.Combine(_saveFolder, GetFilenameWithChannel("File/ConvRgb2Srgb", channel)), convRGB2SRGB);
var convRGB2XYZ = CalcConvertMatrix(tccRgb, TCC_XYZ, channel);
SaveMatToCsv(Path.Combine(_saveFolder, GetFilenameWithChannel("File/ConvRgb2Xyz", channel)), convRGB2XYZ);
if (channel == 17) {
var convOld = new Mat(convRGB2XYZ.Size(), MatType.CV_64FC1);
for (var row = 0; row < convRGB2XYZ.Rows; row++) {
var rowFrom = (row + 1) % convRGB2XYZ.Rows; // 1行ずらす
for (var col = 0; col < convRGB2XYZ.Cols; col++) {
convOld.At<double>(row, col) = convRGB2XYZ.At<double>(rowFrom, col);
}
}
SaveMatToCsv(Path.Combine(_saveFolder, GetFilenameWithChannel("File/ConvRgb2XyzOld", channel)), convOld);
}
var tccSrgb = ConvertColor(tccRgb, convRGB2SRGB);
SaveMatToCsv(Path.Combine(_saveFolder, GetFilenameWithChannel("File/TccSrgbValues", channel)), tccSrgb);
var convSrgb2Xyz = CalcConvertMatrix(tccSrgb, TCC_XYZ, channel);
SaveMatToCsv(Path.Combine(_saveFolder, GetFilenameWithChannel("File/ConvSrgb2Xyz", channel)), convSrgb2Xyz);
// 変換精度検証
var diff = Math.Sqrt(Cv2.Norm(TCC_SRGB, tccSrgb, NormTypes.L2));
Debug.WriteLine($"{channel}次元 RGB→SRGB 変換行列の誤差 = {diff:.000}");
_convRGB2SRGB.Add(channel, convRGB2SRGB);
}
EventSound(Config.GetString("Sound/CalibDone"));
_form.ShowMessage("自動校正完了");
_form.EnableShots();
_calibrated = true;
}
/// <summary>
/// 画像からチャートのRGB値算出
/// </summary>
/// <param name="img"></param>
/// <returns></returns>
private Mat GetTccRgb(Mat img) {
var arrRGB = new Mat(24, 3, MatType.CV_64FC1);
for (int i = 0; i < _chartMasks.Count; i++) {
var rgb = Cv2.Mean(img, _chartMasks[i]);
arrRGB.At<double>(i, 0) = rgb.Val0; // Blue
arrRGB.At<double>(i, 1) = rgb.Val1; // Green
arrRGB.At<double>(i, 2) = rgb.Val2; // Red
}
return arrRGB;
}
/// <summary>
/// TCCのROI画像を取得
/// </summary>
/// <param name="img"></param>
/// <returns></returns>
private Mat GetTccRoisImage(Mat img) {
var imgRois = img.Clone();
for (int i = 0; i < _chartMasks.Count; i++) {
imgRois.SetTo(new Scalar(0, 200, 0), _chartMasks[i]);
}
return imgRois;
}
/// <summary>
/// 変換行列算出
/// </summary>
/// <param name="img">画像</param>
/// <param name="target">変換目標の24x3行列</param>
private Mat CalcConvertMatrix(Mat src, Mat target, int channels) {
var extended = ExtendMat(src, channels);
Mat convMat = new Mat(channels, 3, MatType.CV_64FC1);
Cv2.Solve(extended, target, convMat, DecompTypes.SVD);
return convMat;
}
/// <summary>
/// 色変換
/// </summary>
/// <param name="src"></param>
/// <param name="conv"></param>
/// <returns></returns>
private Mat ConvertColor(Mat src, Mat conv) {
var extended = ExtendMat(src, conv.Rows);
var converted = (extended * conv);
return converted.ToMat();
}
/// <summary>
/// 8bitクリッピング処理 ForEachAsVec3d用
/// </summary>
/// <param name="p"></param>
/// <param name="pos"></param>
unsafe void Clip8bit(Vec3d* p, int* pos) {
p->Item0 = Math.Max(0, Math.Min(255.0, p->Item0));
p->Item1 = Math.Max(0, Math.Min(255.0, p->Item1));
p->Item2 = Math.Max(0, Math.Min(255.0, p->Item2));
}
/// <summary>
/// 画像の色変換
/// </summary>
/// <param name="src"></param>
/// <param name="conv"></param>
/// <returns>double型画像</returns>
protected Mat ConvertImage(Mat src, Mat conv) {
if (src.Type() != MatType.CV_64FC3) {
src.ConvertTo(src, MatType.CV_64FC3);
}
var flatten = src.Reshape(3, src.Height * src.Width);
var extended = ExtendMat(flatten, conv.Rows);
var converted = (extended * conv).ToMat();
var convertedImage = converted.Reshape(3, src.Height);
//Clipping to 8bit range
//unsafe {
// convertedImage.ForEachAsVec3d(Clip8bit);
//}
var convImg8 = new Mat();
convertedImage.ConvertTo(convImg8, MatType.CV_8UC3);
return convImg8;
}
/// <summary>
/// 行列の拡張 3次元→高次元（指定チャンネル数）
/// </summary>
/// <param name="src"></param>
/// <returns></returns>
private Mat ExtendMat(Mat src, int channels) {
if (src.Cols * src.Channels() != 3) return src;
var dst = new Mat(src.Rows, channels, MatType.CV_64FC1);
Parallel.For(0, src.Rows, row => {
var vals = new double[] {
src.Cols == 1 ? src.At<Vec3d>(row, 0)[2] : src.At<double>(row, 2), // R
src.Cols == 1 ? src.At<Vec3d>(row, 0)[1] : src.At<double>(row, 1), // G
src.Cols == 1 ? src.At<Vec3d>(row, 0)[0] : src.At<double>(row, 0), // B
1.0
};
for (int i = 0; i < channels; i++) {
dst.At<double>(row, i) = vals[ExtendChannels[i][0]] * vals[ExtendChannels[i][1]] * vals[ExtendChannels[i][2]];
}
});
return dst;
}
/// <summary>
/// ゲイン値の更新比率計算
/// </summary>
/// <param name="value"></param>
/// <returns></returns>
protected float GetRatio(float value, float target) {
float ratio = target / value;
ratio = (ratio - 1.0f) * (value == 255.0f ? 1.0f : UpdateRate) + 1.0f;
return ratio;
}
/// <summary>
/// csvファイルからMatを読み込む
/// </summary>
/// <param name="csvFile"></param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
private Mat LoadMatFromCsv(string csvFile) {
try {
var arr = new List<double[]>();
int cols = -1;
using (var reader = new StreamReader(csvFile)) {
while (!reader.EndOfStream) {
var line = reader.ReadLine();
var valStrs = line.Split(',');
if (cols == -1) cols = valStrs.Length;
else if (cols != valStrs.Length) throw new Exception("cols != valStrs.Length");
var vals = valStrs.Select(x => double.Parse(x)).ToArray();
arr.Add(vals);
}
}
var m = new Mat(arr.Count, cols, MatType.CV_64FC1);
for (var row = 0; row < arr.Count; row++) {
for (var col = 0; col < cols; col++) {
m.At<double>(row, col) = arr[row][col];
}
}
return m;
} catch (Exception) {
return null;
}
}
/// <summary>
/// Matをcsvファイルに保存
/// </summary>
/// <param name="m"></param>
/// <param name="csvFile"></param>
private void SaveMatToCsv(string csvFile, Mat m) {
using (var writer = new StreamWriter(csvFile)) {
for (var row = 0; row < m.Rows; row++) {
var line = "";
for (var col = 0; col < m.Cols; col++) {
if (col > 0) line += ",";
line += $"{m.At<double>(row, col):0.0000000}";
}
writer.WriteLine(line);
}
}
}
/// <summary>
/// 保存フォルダの取得
/// </summary>
/// <param name="isSeries"></param>
/// <returns></returns>
protected void SetSaveFolder(string note) {
var dt = DateTime.Now;
var paths = new List<string>() {
Config.GetString("File/SaveFolder"),
dt.ToString("yyyy-MM-dd"),
dt.ToString("HH_mm_ss") + $"-{note}",
};
_saveFolder = Path.Combine(paths.ToArray());
Directory.CreateDirectory(_saveFolder);
}
/// <summary>
/// 設定値を保存する
/// </summary>
/// <param name="key"></param>
/// <param name="value"></param>
protected void SetInfo(string key, string value) {
if (_ShotInfo.ContainsKey(key)) {
_ShotInfo[key] = value;
} else {
_ShotInfo.Add(key, value);
}
}
/// <summary>
/// 設定値をCSVファイルに書き込む
/// </summary>
/// <param name="sw"></param>
protected void WriteInfo(StreamWriter sw) {
sw.WriteLine($"撮影日時,{DateTime.Now:yyyy/MM/dd HH:mm:ss}"); // 撮影日時を追加
foreach (var kvp in _ShotInfo) {
sw.WriteLine($"{kvp.Key},{kvp.Value}");
}
}
private void EventSound(string wavfile) {
if (!File.Exists(wavfile)) {
Debug.WriteLine($"音声ファイルが見つかりません: {wavfile}");
return;
}
try {
using (var player = new SoundPlayer(wavfile)) {
player.Play();
}
} catch (Exception ex) {
Debug.WriteLine($"音声再生に失敗しました: {ex.Message}");
}
}
}
}
