using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using Lumenera.USB;
using OpenCvSharp;
using OpenCvSharp.Extensions;
using System.Drawing;
using System.Windows.Interop;
using System.Threading;
using OpenCvSharp.Aruco;
using System.Windows.Media.Animation;
using System.Configuration;
using System.Web.ModelBinding;
using System.Windows.Controls;
using System.IO;
using static System.Resources.ResXFileRef;
using System.Windows.Shapes;
using Path = System.IO.Path;
namespace TIASshot {
/// <summary>
/// Lumeneraカメラクラス
/// </summary>
internal class Lucam {
// Todo: RGB->XYZ 変換行列算出
// Todo: SRGB->XYZ 変換行列算出
// 光源の輝度設定
public string DeviceName { get; private set; } = "Unknown";
public string SerialNumber { get; private set; }
public string ErrorMsg { get; private set; }
readonly Dictionary ARDict = CvAruco.GetPredefinedDictionary(PredefinedDictionaryName.Dict4X4_50);
readonly Point2f[] PointsDst40 = new Point2f[] {
new Point2f(345, 130),new Point2f(465, 130),new Point2f(465, 250),new Point2f(345, 250),
};
readonly Point2f[] PointsDst41 = new Point2f[]{
new Point2f(345, 1200), new Point2f(465, 1200), new Point2f(465, 1320), new Point2f(345, 1320),
};
readonly float RefRGB;
readonly float UpdateRate;
readonly Mat TCC_SRGB;
readonly Mat TCC_XYZ;
IntPtr _hCam = IntPtr.Zero;
PictureBox _picPreview, _picDisplay;
Form1 _form;
bool _isPreview = false;
//bool _check = false;
int _calibrating = 0;
bool _calibrated = false;
dll.LucamSnapshot _snap;
dll.LucamConversion _convert;
dll.LucamRgbPreviewCallback _callbackHandler;
int _callbackId;
Bitmap[] _bmps = new Bitmap[2];
int _bmpIndex = 0;
List<Mat> _chartMasks = new List<Mat>();
int _detectionCount = 0;
Point2f _lastPosition = new Point2f(0, 0);
Mat _convRGB2SRGB;
Mat _tccRois; // TCCのROI検出結果画像
/// <summary>
/// コンストラクタ
/// </summary>
/// <param name="preview"></param>
/// <param name="display"></param>
public Lucam(Form1 form, PictureBox preview, PictureBox display) {
_picPreview = preview;
_picDisplay = display;
_form = form;
Config.Load();
RefRGB = Config.GetFloat("ReferenceValue");
UpdateRate = Config.GetFloat("UpdateRate");
TCC_SRGB = LoadMatFromCsv(Config.GetString("TccSrgbTableFile"));
TCC_XYZ = LoadMatFromCsv(Config.GetString("TccXyzTableFile"));
// カメラパラメータの初期値
_snap.BufferLastFrame = false;
_snap.Exposure = Config.GetFloat("Exposure");
_snap.ExposureDelay = 0;
_snap.flReserved1 = 0;
_snap.flReserved2 = 0;
_snap.Format.BinningX = 1;
_snap.Format.BinningY = 1;
_snap.Format.FlagsX = 0;
_snap.Format.FlagsY = 0;
_snap.Format.Height = 1024;
_snap.Format.PixelFormat = dll.LucamPixelFormat.PF_8;
_snap.Format.SubSampleX = 1;
_snap.Format.SubSampleY = 1;
_snap.Format.Width = 1280;
_snap.Format.X = 0;
_snap.Format.Y = 0;
_snap.Gain = Config.GetFloat("Gain");
_snap.GainBlue = Config.GetFloat("GainB");
_snap.GainGrn1 = Config.GetFloat("GainG");
_snap.GainGrn2 = _snap.GainGrn1;
_snap.GainRed = Config.GetFloat("GainR");
_snap.ShutterType = dll.LucamShutterType.GlobalShutter;
_snap.StrobeDelay = 0.1f;
_snap.StrobeFlags = 0;
_snap.Timeout = 5000;
_snap.ulReserved2 = 0;
_snap.UseHwTrigger = false;
_convert.DemosaicMethod = dll.LucamDemosaicMethod.HIGH_QUALITY;
_convert.CorrectionMatrix = dll.LucamCorrectionMatrix.LED;
}
/// <summary>
/// 起動チェック
/// </summary>
/// <returns></returns>
private bool BootCheck() {
if (!Config.IsLoaded()) {
ErrorMsg = "設定ファイル（Config.xml）の読み込みに失敗しました．\r\n終了します．";
return false;
}
if (TCC_SRGB is null) {
ErrorMsg = $"ファイル（{Config.GetString("TccSrgbTableFile")}）の読み込みに失敗しました．\r\n終了します．";
return false;
}
if (TCC_XYZ is null) {
ErrorMsg = $"ファイル（{Config.GetString("TccXyzTableFile")}）の読み込みに失敗しました．\r\n終了します．";
return false;
}
return true;
}
/// <summary>
/// カメラ接続
/// </summary>
/// <returns></returns>
public bool Connect() {
if (!BootCheck()) return false;
var numCam = dll.LucamNumCameras();
if ( numCam < 1 ) {
ErrorMsg = "カメラが見つかりません．\r\n終了します．";
return false;
}
if (numCam > 1) {
ErrorMsg = "複数のカメラが見つかりました．\r\n正しいカメラを１つ接続してください．\r\n終了します．";
return false;
}
// カメラのデバイス情報取得
var lumVersion = new dll.LucamVersion[numCam];
lumVersion = api.EnumCameras();
var id = 0;
if (lumVersion[id].CameraId == 0x49f) DeviceName = "Lw110";
SerialNumber = lumVersion[id].SerialNumber.ToString();
// カメラを開く
_hCam = api.CameraOpen(1);
if (_hCam == IntPtr.Zero) {
ErrorMsg = "カメラの接続に失敗しました．\r\n他のアプリケーションでカメラを使用していないか確認してください．\r\n終了します．";
return false;
}
// プレビューコールバックの登録
_callbackHandler = new dll.LucamRgbPreviewCallback(PreviewCallback);
_callbackId = dll.LucamAddRgbPreviewCallback(_hCam, _callbackHandler, IntPtr.Zero, dll.LucamPixelFormat.PF_24);
if (_callbackId == -1) {
ErrorMsg = "コールバックの登録に失敗しました．\r\n終了します．";
return false;
}
SetCameraParam(); // カメラパラメータの設定
return true;
}
/// <summary>
/// プレビュー開始・停止
/// </summary>
/// <returns></returns>
public bool StartStopPreview() {
if (_isPreview) {
// プレビュー停止
var ret = dll.LucamStreamVideoControl(_hCam, dll.LucamStreamMode.STOP_STREAMING, _picPreview.Handle.ToInt32());
Debug.WriteLine("プレビュー停止");
if (!ret) return false;
_isPreview = false;
return true;
} else {
// プレビュー開始
var ret = dll.LucamStreamVideoControl(_hCam, dll.LucamStreamMode.START_DISPLAY, _picPreview.Handle.ToInt32());
if (!ret) return false;
_isPreview = true;
return true;
}
}
/// <summary>
/// ゲイン値の更新比率計算
/// </summary>
/// <param name="value"></param>
/// <returns></returns>
private float GetRatio(float value, float target) {
float ratio = target / value;
ratio = (ratio - 1.0f) * (value == 255.0f ? 1.0f : UpdateRate) + 1.0f;
return ratio;
}
/// <summary>
/// チャートの検出
/// </summary>
/// <param name="img"></param>
private void DetectChart(Mat img) {
// ARマーカー検出
CvAruco.DetectMarkers(img, ARDict, out var corners, out var ids,
new DetectorParameters(), out var rejectedImgPoints);
if (ids.Length < 1) return;
// マーカー座標格納
var ptsPict = new List<Point2f>();
var ptsModel = new List<Point2f>();
Point2f position = new Point2f();
float y40 = 0, y41 = 0;
for (int i = 0; i < ids.Length; i++) {
if (ids[i] == 40) {
ptsPict.AddRange(corners[i]);
ptsModel.AddRange(PointsDst40);
position = corners[i][3];
y40 = corners[i][3].Y;
}
if (ids[i] == 41) {
ptsPict.AddRange(corners[i]);
ptsModel.AddRange(PointsDst41);
y41 = corners[i][3].Y;
}
}
if (ptsPict.Count < 8) return;
if (y40 > y41) {
_form.ShowMessage("舌診チャートが上下逆方向です");
return;
}
// チャートの固定判定
_form.ShowMessage("舌診チャートの検出中");
var dist = (float)position.DistanceTo(_lastPosition);
if (dist < Config.GetFloat("ChartSetCriteria")) {
_detectionCount++;
} else {
_detectionCount = 0;
}
_lastPosition = position;
if (_detectionCount < Config.GetInt("ChartSetCount")) return;
// ホモグラフィの計算
var matPtsPict = Mat.FromArray(ptsPict);
var matPtsModel = Mat.FromArray(ptsModel);
var matH = Cv2.FindHomography(matPtsModel, matPtsPict);
var imgF = new Mat(1545, 810, MatType.CV_8UC3);
Cv2.WarpPerspective(img, imgF, matH, imgF.Size());
// チャートマスク作成
_chartMasks.Clear();
var roiSize = ptsPict.Count < 8 ? 60 : 80;
_tccRois = img.Clone();
for (int i = 0; i < 24; i++) {
var row = i % 6;
var col = i / 6;
var x = 581 - col * 144 + (ptsPict.Count < 8 ? 10 : 0);
var y = 318 + row * 144 + (ptsPict.Count < 8 ? 10 : 0);
var roi = new Rect(x, y, roiSize, roiSize);
using (var mask = new Mat(1545, 810, MatType.CV_8U)) {
Cv2.Rectangle(mask, roi, new Scalar(255), Cv2.FILLED);
var maskF = new Mat(_tccRois.Size(), MatType.CV_8U);
Cv2.WarpPerspective(mask, maskF, matH, maskF.Size());
_chartMasks.Add(maskF);
_tccRois.SetTo(new Scalar(0, 200, 0), maskF);
}
}
_form.ShowMessage("舌診チャート検出 校正中");
_calibrating = Config.GetInt("CalibrationFrames");
}
/// <summary>
/// プレビューコールバック
/// </summary>
/// <param name="pContext"></param>
/// <param name="pData">データポインタ</param>
/// <param name="n">データサイズ</param>
/// <param name="unused"></param>
void PreviewCallback(IntPtr pContext, IntPtr pData, int n, uint unused) {
using (Mat img = Mat.FromPixelData(_snap.Format.Height, _snap.Format.Width, MatType.CV_8UC3, pData)) {
using (Mat imgt = img.T()) {
_bmps[_bmpIndex] = imgt.ToBitmap();
if (_calibrating > 0) {
var whitePatch = Cv2.Mean(imgt, _chartMasks[12]);
Debug.WriteLine($"White patch R {whitePatch.Val2:.00} G {whitePatch.Val1:.00} B {whitePatch.Val0:.00}");
if (_calibrating % Config.GetInt("CalibrationUpdateInterval") == 0) {
_snap.GainBlue *= GetRatio((float)whitePatch.Val0, Config.GetFloat("ReferenceB"));
_snap.GainGrn1 *= GetRatio((float)whitePatch.Val1, Config.GetFloat("ReferenceG"));
_snap.GainGrn2 = _snap.GainGrn1;
_snap.GainRed *= GetRatio((float)whitePatch.Val2, Config.GetFloat("ReferenceR"));
SetCameraParam();
}
_calibrating--;
if (_calibrating == 0) {
CalcTcc(imgt);
}
}
if (!_calibrated && _calibrating == 0) {
DetectChart(imgt);
}
}
}
_form.ShowImage(_bmps[_bmpIndex]);
_bmpIndex = (_bmpIndex + 1) % 2;
if (_bmps[_bmpIndex] != null) _bmps[_bmpIndex].Dispose();
}
private void CalcTcc(Mat img) {
_convRGB2SRGB = CalcConvertMatrix(img, TCC_SRGB);
// データ保存
var folder = GetSaveFolder("校正");
Cv2.ImWrite(Path.Combine(folder, Config.GetString("TccSaveFile")), img);
Cv2.ImWrite(Path.Combine(folder, Config.GetString("TccRoisFile")), _tccRois);
SaveMatToCsv(Path.Combine(folder, Config.GetString("ConvSrgbSaveFile")), _convRGB2SRGB);
_form.ShowMessage("自動校正完了");
_form.EnableShots();
_calibrated = true;
}
/// <summary>
/// 変換行列算出
/// </summary>
/// <param name="img">画像</param>
/// <param name="target">変換目標の24x3行列</param>
private Mat CalcConvertMatrix(Mat img, Mat target) {
// 画像からチャートのRGB値算出
var arrRGB = new Mat(24, 3, MatType.CV_64FC1);
for (int i = 0; i < _chartMasks.Count; i++) {
var rgb = Cv2.Mean(img, _chartMasks[i]);
arrRGB.At<double>(i, 0) = rgb.Val0;
arrRGB.At<double>(i, 1) = rgb.Val1;
arrRGB.At<double>(i, 2) = rgb.Val2;
}
var extended = ExtendMat(arrRGB);
//Debug.WriteLine("extended");
//Debug.WriteLine(extended.Dump());
// 変換行列算出
Mat convMat = new Mat(17, 3, MatType.CV_64FC1);
Cv2.Solve(extended, target, convMat, DecompTypes.SVD);
//Debug.WriteLine("convMat");
//Debug.WriteLine(convMat.Dump());
// 変換精度検証
var converted = extended * convMat;
//Debug.WriteLine("converted");
//Debug.WriteLine(converted.ToMat().Dump());
//Mat impSRGB = converted.ToMat().Reshape(3);
//Cv2.ImWrite("impSRGB.png", impSRGB);
var diff = Math.Sqrt(Cv2.Norm(target, converted, NormTypes.L2));
Debug.WriteLine($"変換行列の誤差 = {diff:.000}");
return convMat;
}
/// <summary>
/// 画像撮影１枚
/// </summary>
public void ShotOne() {
Shot();
_form.ShowMessage("1枚撮影 保存しました");
}
/// <summary>
/// 複数画像撮影
/// </summary>
public void ShotMulti() {
Shot(_form.GetNumMultiShots(), _form.GetMultiShotsInterval());
_form.ShowMessage("連続撮影 保存しました");
}
/// <summary>
/// 撮影
/// </summary>
/// <param name="numImages"></param>
/// <param name="interval"></param>
private void Shot(int numImages=1, int interval=0) {
SetSnapParam();
var folder = GetSaveFolder(_form.GetDataName());
dll.LucamEnableFastFrames(_hCam, ref _snap);
var imageSize = _snap.Format.Width * _snap.Format.Height;
var rawImage = new byte[imageSize];
var rgbImage = new byte[imageSize * 3];
for (var i = 0; i < numImages; i++) {
_form.ShowMessage($"連続撮影 {i+1} / {numImages} 枚目");
var ret = dll.LucamTakeFastFrame(_hCam, rawImage);
//Debug.WriteLine(ret);
dll.LucamConvertFrameToRgb24(_hCam, rgbImage, rawImage,
_snap.Format.Width, _snap.Format.Height, dll.LucamPixelFormat.PF_8, ref _convert);
using (Mat img = Mat.FromPixelData(_snap.Format.Height, _snap.Format.Width, MatType.CV_8UC3, rgbImage)) {
using (Mat imgt = img.T()) {
Cv2.ImWrite(Path.Combine(folder, $"Shot{i + 1:0000}.bmp"), imgt);
using (var converted = ConvertImage(imgt, _convRGB2SRGB)) {
Cv2.ImWrite(Path.Combine(folder, $"Srgb{i+1:0000}.jpg"), converted);
}
}
}
if (i < numImages - 1) Thread.Sleep(interval);
}
rgbImage = null;
rawImage = null;
dll.LucamDisableFastFrames(_hCam);
}
/// <summary>
/// 校正 露光時間とホワイトバランスの自動調整
/// </summary>
public void Calibration() {
Debug.WriteLine("校正前");
SetSnapParam();
_calibrating = 30;
}
/// <summary>
/// 撮影パラメータの設定
/// </summary>
private void SetSnapParam() {
dll.LucamPropertyFlag flag;
dll.LucamGetProperty(_hCam, dll.LucamProperty.EXPOSURE, out _snap.Exposure, out flag);
dll.LucamGetProperty(_hCam, dll.LucamProperty.GAIN, out _snap.Gain, out flag);
dll.LucamGetProperty(_hCam, dll.LucamProperty.GAIN_BLUE, out _snap.GainBlue, out flag);
dll.LucamGetProperty(_hCam, dll.LucamProperty.GAIN_GREEN1, out _snap.GainGrn1, out flag);
dll.LucamGetProperty(_hCam, dll.LucamProperty.GAIN_GREEN2, out _snap.GainGrn2, out flag);
dll.LucamGetProperty(_hCam, dll.LucamProperty.GAIN_RED, out _snap.GainRed, out flag);
Debug.WriteLine($"SetSnapParam Exp {_snap.Exposure} Gain {_snap.Gain} GainBlue {_snap.GainBlue} GainGrn1 {_snap.GainGrn1} GainGrn2 {_snap.GainGrn2} GainRed {_snap.GainRed}");
}
/// <summary>
/// カメラパラメータの設定
/// </summary>
private void SetCameraParam() {
var flag = dll.LucamPropertyFlag.NONE;
dll.LucamSetProperty(_hCam, dll.LucamProperty.EXPOSURE, _snap.Exposure, flag);
dll.LucamSetProperty(_hCam, dll.LucamProperty.GAIN, _snap.Gain, flag);
dll.LucamSetProperty(_hCam, dll.LucamProperty.GAIN_BLUE, _snap.GainBlue, flag);
dll.LucamSetProperty(_hCam, dll.LucamProperty.GAIN_GREEN1, _snap.GainGrn1, flag);
dll.LucamSetProperty(_hCam, dll.LucamProperty.GAIN_GREEN2, _snap.GainGrn2, flag);
dll.LucamSetProperty(_hCam, dll.LucamProperty.GAIN_RED, _snap.GainRed, flag);
Debug.WriteLine($"SetCameraParam Exp {_snap.Exposure} Gain {_snap.Gain} GainBlue {_snap.GainBlue} GainGrn1 {_snap.GainGrn1} GainGrn2 {_snap.GainGrn2} GainRed {_snap.GainRed}");
}
/// <summary>
/// csvファイルからMatを読み込む
/// </summary>
/// <param name="csvFile"></param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
private Mat LoadMatFromCsv(string csvFile) {
try {
var arr = new List<double[]>();
int cols = -1;
using (var reader = new StreamReader(csvFile)) {
while (!reader.EndOfStream) {
var line = reader.ReadLine();
var valStrs = line.Split(',');
if (cols == -1) cols = valStrs.Length;
else if (cols != valStrs.Length) throw new Exception("cols != valStrs.Length");
var vals = valStrs.Select(x => double.Parse(x)).ToArray();
arr.Add(vals);
}
}
var m = new Mat(arr.Count, cols, MatType.CV_64FC1);
for (var row = 0; row < arr.Count; row++) {
for (var col = 0; col < cols; col++) {
m.At<double>(row, col) = arr[row][col];
}
}
return m;
} catch (Exception) {
return null;
}
}
/// <summary>
/// Matをcsvファイルに保存
/// </summary>
/// <param name="m"></param>
/// <param name="csvFile"></param>
private void SaveMatToCsv(string csvFile, Mat m) {
using (var writer = new StreamWriter(csvFile)) {
writer.Write(m.Dump(FormatType.CSV));
}
}
/// <summary>
/// 画像の色変換
/// </summary>
/// <param name="src"></param>
/// <param name="conv"></param>
/// <returns>double型画像</returns>
private Mat ConvertImage(Mat src, Mat conv) {
if (src.Type() != MatType.CV_64FC3) {
src.ConvertTo(src, MatType.CV_64FC3);
}
var flatten = src.Reshape(3, src.Height * src.Width);
var extended = ExtendMat(flatten);
var converted = (extended * conv).ToMat();
var convertedImage = converted.Reshape(3, src.Height);
return convertedImage;
}
/// <summary>
/// 行列の拡張 3次元→17次元
/// </summary>
/// <param name="src"></param>
/// <returns></returns>
private Mat ExtendMat(Mat src) {
if (src.Cols * src.Channels() != 3) return src;
var dst = new Mat(src.Rows, 17, MatType.CV_64FC1);
for (int row = 0; row < src.Rows; row++) {
var b = src.Cols == 1 ? src.At<Vec3d>(row, 0)[0] : src.At<double>(row, 0);
var g = src.Cols == 1 ? src.At<Vec3d>(row, 0)[1] : src.At<double>(row, 1);
var r = src.Cols == 1 ? src.At<Vec3d>(row, 0)[2] : src.At<double>(row, 2);
dst.At<double>(row, 0) = r;
dst.At<double>(row, 1) = g;
dst.At<double>(row, 2) = b;
dst.At<double>(row, 3) = r * g;
dst.At<double>(row, 4) = r * b;
dst.At<double>(row, 5) = g * b;
dst.At<double>(row, 6) = r * r;
dst.At<double>(row, 7) = g * g;
dst.At<double>(row, 8) = b * b;
dst.At<double>(row, 9) = r * r * b;
dst.At<double>(row, 10) = r * r * g;
dst.At<double>(row, 11) = g * g * r;
dst.At<double>(row, 12) = g * g * b;
dst.At<double>(row, 13) = b * b * r;
dst.At<double>(row, 14) = b * b * g;
dst.At<double>(row, 15) = r * g * b;
dst.At<double>(row, 16) = 1.0;
}
return dst;
}
/// <summary>
/// 保存フォルダの取得
/// </summary>
/// <param name="isSeries"></param>
/// <returns></returns>
private string GetSaveFolder(string note) {
var dt = DateTime.Now;
var paths = new List<string>() {
Config.GetString("SaveFolder"),
dt.ToString("yyyy-MM-dd"),
dt.ToString("HH_mm_ss") + $"-{note}",
};
var path = Path.Combine(paths.ToArray());
Directory.CreateDirectory(path);
return path;
}
/// <summary>
/// カメラ切断
/// </summary>
public void Disconnect() {
if (_hCam != IntPtr.Zero) {
if (_isPreview) {
StartStopPreview();
}
api.CameraClose(_hCam);
_hCam = IntPtr.Zero;
Debug.WriteLine("カメラ切断");
}
}
}
}
