using System;
using System.Windows.Forms;
using System.IO;
using System.Drawing;
using System.Drawing.Imaging;
using System.ComponentModel;
using System.Runtime.InteropServices;
using System.Threading;
using System.Diagnostics;
using System.Linq;
using System.Collections.Generic;
using TensorFlow;
using OpenCvSharp;
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
namespace Tongue_extraction
{
public partial class Form1 : Form
{
[DllImport("TongueAnalysisDLL.dll")]
private static extern int AnalysisProc2(string path_img, string path_calib, string path_info, string path_output);
[DllImport("TongueAnalysisDLL.dll")]
private static extern int AnalysisProc3(string path_img, string path_calib, string path_output);
//Use the RemoveSmallRegionDLL
[DllImport(@"RemoveSmallRegionDLL.dll", EntryPoint = "RemoveSmallRegion", SetLastError = true, CharSet = CharSet.Ansi, ExactSpelling = false, CallingConvention = CallingConvention.StdCall)]
extern static void RemoveSmallRegion(string name, string name2, int AreaLimit, int CheckMode, int NeihborMode);
// 削除予定
public static Bitmap bitmap_bitch;
float max_score = 0;
private ManualResetEvent manualReset = new ManualResetEvent(true);
string path_detectionModel = @"models/Detection_Normal.pb";
string path_segmentationModel = @"models/ResUNet.onnx";
public Form1()
{
InitializeComponent();
comboBox_mode.DropDownStyle = ComboBoxStyle.DropDownList;
comboBox_mode.SelectedIndex = 0;
}
private void Button_start_Click(object sender, EventArgs e)
{
button_start.Enabled = false;
button_pause.Enabled = true;
pictureBox_input.Image = null;
pictureBox_input.Refresh();
pictureBox_calibration.Image = null;
pictureBox_calibration.Refresh();
pictureBox_detection.Image = null;
pictureBox_detection.Refresh();
pictureBox_segmentation.Image = null;
pictureBox_segmentation.Refresh();
pictureBox_mask.Image = null;
pictureBox_mask.Refresh();
pictureBox_extracted.Image = null;
pictureBox_extracted.Refresh();
pictureBox_regions.Image = null;
pictureBox_regions.Refresh();
pictureBox_last.Image = null;
pictureBox_last.Refresh();
comboBox_mode.Enabled = false;
if (comboBox_mode.SelectedIndex == 0)
{
// 画像のみ入力し，マスク等を得る
this.backgroundWorker1.DoWork += new System.ComponentModel.DoWorkEventHandler(this.BackgroundWorker1_DoWork_Image);
backgroundWorker1.RunWorkerAsync();
}
else if (comboBox_mode.SelectedIndex == 1)
{
// 画像と校正画像を入力し，Lab色を得る
this.backgroundWorker1.DoWork += new System.ComponentModel.DoWorkEventHandler(this.BackgroundWorker1_DoWork_ImageAndCalib);
backgroundWorker1.RunWorkerAsync();
}
else
{
MessageBox.Show("Please select the mode on the combobox");
}
}
private void BackgroundWorker1_DoWork_Image(object sender, DoWorkEventArgs e)
{
// ステータスの表示
Invoke((MethodInvoker)delegate
{
label_state.Text = "準備中..";
});
// Path
var path_imgDir = @"..\Input\TongueImage";
var path_outputDir = @"..\Output/" + DateTime.Now.ToLocalTime().ToString("yyyyMMddhhmmss");
MyMakeDir(path_outputDir);
FolderCopy("template_outputDir", path_outputDir);
var path_outputCsv = path_outputDir + @"\analysis.csv";
var count = 0;
foreach (var path_img in Directory.GetFiles(path_imgDir))
{
var name_file = Path.GetFileNameWithoutExtension(path_img);
// ステータスの表示
Invoke((MethodInvoker)delegate
{
label_processingFileName.Text = "Processing File: " + path_img;
count += 1;
label_totalProgress.Text = "Total Progress: " + count + "/" + Directory.GetFiles(path_imgDir).Length;
});
// Init
manualReset.WaitOne();
Invoke((MethodInvoker)delegate
{
label2.Text = "";
label8.Text = "";
label8.BackColor = Color.White;
label1.BackColor = Color.Red;
});
// 舌画像表示
Mat mat_input = Cv2.ImRead(path_img, ImreadModes.Color);
Invoke((MethodInvoker)delegate
{
label1.BackColor = Color.White;
label3.BackColor = Color.Red;
pictureBox_input.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_input);
pictureBox_input.Refresh();
label_state.Text = "舌検出及び分割";
});
// Detection
var dst_rect = new Rect();
try
{
dst_rect = TongueDetection(mat_input);
}
catch (Exception)
{
using (var sw = new StreamWriter(path_outputCsv, true))
{
var swLine = name_file + "," + "Tongue detection error";
sw.WriteLine(swLine);
}
continue;
}
if ((dst_rect.Width == 0) || (dst_rect.Height == 0))
{
using (var sw = new StreamWriter(path_outputCsv, true))
{
var swLine = name_file + "," + "Tongue detection error";
sw.WriteLine(swLine);
}
continue;
}
var mat_drawBox = mat_input.Clone();
Cv2.Rectangle(mat_drawBox, dst_rect, new Scalar(0, 255, 0), 3);
MyMakeDir(path_outputDir + @"\1-detection\1-drawBox\");
mat_drawBox.SaveImage(path_outputDir + @"\1-detection\1-drawBox\" + name_file + ".png");
// Detection領域crop
var mat_cropped = new Mat(mat_input, dst_rect);
mat_cropped.SaveImage(path_outputDir + @"\1-detection\2-cropped\" + name_file + ".png");
// Resize
var mat_roisize = mat_input.Clone(dst_rect);
var mat_roi = new Mat();
Cv2.Resize(mat_roisize, mat_roi, new OpenCvSharp.Size(256, 256));
mat_roi.SaveImage(path_outputDir + @"\1-detection\3-resized\" + name_file + ".png");
// Detection 表示
Invoke((MethodInvoker)delegate
{
label3.BackColor = Color.White;
label4.BackColor = Color.Red;
pictureBox_detection.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_roi); ;
pictureBox_detection.Refresh();
});
// Segmenation
var mat_output = TongueSegmentation(mat_roi);
Cv2.Threshold(mat_output, mat_output, 128, 255, ThresholdTypes.Otsu | ThresholdTypes.Binary);
Thread.Sleep(100);
GC.Collect();
mat_output.SaveImage(path_outputDir + @"\2-segmentation\1-modelOutput\" + name_file + ".png");
// 2値化
var mat_binOutput = new Mat();
Cv2.Threshold(mat_output, mat_binOutput, 0, 255, ThresholdTypes.Otsu);
mat_binOutput.SaveImage(path_outputDir + @"\2-segmentation\2-binOutput\" + name_file + ".png");
// Segmenation 表示
Invoke((MethodInvoker)delegate
{
label3.BackColor = Color.White;
label4.BackColor = Color.Red;
pictureBox_segmentation.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_binOutput); ;
pictureBox_segmentation.Refresh();
});
// Segmentationの例外処理
var edges = new Mat();
OpenCvSharp.Point[][] contours;
HierarchyIndex[] hierarchyIndexes;
Cv2.Canny(mat_binOutput, edges, 50, 200);
Cv2.FindContours(edges, out contours, out hierarchyIndexes, RetrievalModes.List, ContourApproximationModes.ApproxSimple);
// 1 or 2つ以外の輪郭が検出された場合
var num_obj = contours.Count();
if ((num_obj == 0) || (num_obj > 2))
{
Console.WriteLine("Segmentation result has over 1 area : extracted areas = {0}", num_obj);
try
{
RemoveSmallRegion(path_outputDir + @"\2-segmentation\2-binOutput\" + name_file + ".png", path_outputDir + @"\2-segmentation\2-binOutput(refined)\" + name_file + ".png", 500, 1, 1);
RemoveSmallRegion(path_outputDir + @"\2-segmentation\2-binOutput(refined)\" + name_file + ".png", path_outputDir + @"\2-segmentation\2-binOutput(refined)\" + name_file + ".png", 500, 0, 0);
}
catch
{
MessageBox.Show("Error: Unable to reprocess! Please check is there [RemoveSmallRegionDLL.dll] file in floder?", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
break;
}
// 再度確認
mat_binOutput = Cv2.ImRead(path_outputDir + @"\2-segmentation\2-binOutput(refined)\" + name_file + ".png", ImreadModes.Grayscale);
Cv2.ImShow("dst2", mat_binOutput);
Cv2.WaitKey();
Cv2.Canny(mat_binOutput, edges, 50, 200);
Cv2.FindContours(edges, out contours, out hierarchyIndexes, RetrievalModes.List, ContourApproximationModes.ApproxSimple);
num_obj = contours.Count();
if ((num_obj == 0) || (num_obj > 2))
{
using (var sw = new StreamWriter(path_outputCsv, true))
{
Console.WriteLine("Tongue segmentation error");
var swLine = name_file + "," + "Tongue segmentation error";
sw.WriteLine(swLine);
}
continue;
}
}
// Resize
var mat_resizedOutput = new Mat();
Cv2.Resize(mat_binOutput, mat_resizedOutput, dst_rect.Size);
mat_resizedOutput.SaveImage(path_outputDir + @"\2-segmentation\3-resized\" + name_file + ".png");
// マスクの作成
var mat_mask = new Mat(mat_input.Size(), MatType.CV_8UC1, 0);
for (int i = dst_rect.Top; i < dst_rect.Bottom; i++)
for (int j = dst_rect.Left; j < dst_rect.Right; j++)
{
var value = mat_resizedOutput.At<byte>(i - dst_rect.Top, j - dst_rect.Left);
mat_mask.Set(i, j, value);
}
mat_mask.SaveImage(path_outputDir + @"\2-segmentation\4-mask\" + name_file + ".png");
// マスク 表示
Invoke((MethodInvoker)delegate
{
label4.BackColor = Color.White;
label5.BackColor = Color.Red;
pictureBox_mask.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_mask);
pictureBox_mask.Refresh();
});
// 舌領域の抽出
var mat_extraction = new Mat();
mat_input.CopyTo(mat_extraction, mat_mask);
mat_extraction.SaveImage(path_outputDir + @"\2-segmentation\5-extraction\" + name_file + ".png");
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
pictureBox_extracted.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_extraction);
pictureBox_extracted.Refresh();
label5.BackColor = Color.White;
label6.BackColor = Color.Red;
label_state.Text = "舌特徴抽出中...";
});
// 面積，舌検出座標 保存
var areaCount = 0;
using (var mat_temp = new Mat())
{
Cv2.CvtColor(mat_extraction, mat_temp, ColorConversionCodes.BGR2GRAY);
areaCount = Cv2.CountNonZero(mat_temp);
}
var out_dir = path_outputDir + @"\4-featureExtraction\shape\";
MyMakeDir(out_dir);
using (var sw = new StreamWriter(out_dir + name_file + ".csv"))
{
sw.WriteLine(",width,height,area," +
"topLeftX,topLeftY,bottomRightX,bottomRightY");
sw.Write(
"shape," + dst_rect.Width.ToString() + "," + dst_rect.Height.ToString() + "," + areaCount.ToString() + "," +
dst_rect.TopLeft.X.ToString() + "," + dst_rect.TopLeft.Y.ToString() + "," +
dst_rect.BottomRight.X.ToString() + "," + dst_rect.BottomRight.Y.ToString());
}
// 舌分画に基づく特徴量計算1: Chiu
var chiu = new ColorExtractor_Chiu();
var mat_chiu = chiu.Get3areaMask(mat_mask);
var path_chiuMask = path_outputDir + @"\3-imgProc\devideRegion\chiu\" + name_file + ".png";
mat_chiu.SaveImage(path_outputDir + @"\3-imgProc\devideRegion\chiu\" + name_file + ".png");
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
label6.BackColor = Color.White;
label7.BackColor = Color.Red;
pictureBox_regions.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_chiu);
pictureBox_regions.Refresh();
});
// 舌分画に基づく特徴量計算2: ExtendedChiu
var exChiu = new ColorExtractor_ExtendedChiu();
var mat_exChiu = exChiu.Get4areaMask(mat_mask);
mat_exChiu.SaveImage(path_outputDir + @"\3-imgProc\devideRegion\exChiu\" + name_file + ".png");
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
pictureBox_regions.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_exChiu);
pictureBox_regions.Refresh();
});
// 舌分画に基づく特徴量計算3: fiveclick [ishikawa, 2010]
var fiveClick = new ColorExtractor_fiveClick();
var mat_fiveClick = fiveClick.Get4areaMask(mat_mask);
mat_fiveClick.SaveImage(path_outputDir + @"\3-imgProc\devideRegion\fiveClick\" + name_file + ".png");
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
pictureBox_regions.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_fiveClick);
pictureBox_regions.Refresh();
label6.BackColor = Color.White;
});
var regionMethod = new List<string> { "chiu", "exChiu", "fiveClick" };
// Lab
for (int m = 0; m < regionMethod.Count(); m++)
{
// 色統計量，GLCM計算
var path_regionMask = path_outputDir + @"\3-imgProc\devideRegion\" + regionMethod[m] + "\\" + name_file + ".png";
var path_GLCM = path_outputDir + @"\4-featureExtraction\region\" + regionMethod[m] + @"\GLCM\" + name_file + ".csv";
var path_colorStats = path_outputDir + @"\4-featureExtraction\region\" + regionMethod[m] + @"\colorStats\" + name_file + ".csv";
var pInfo_py = new ProcessStartInfo(@"python\TongueFeatures.exe")
{
UseShellExecute = false,
RedirectStandardOutput = true,
CreateNoWindow = true,
Arguments = path_img + " " + path_regionMask + " " + path_colorStats + " " + path_GLCM
};
using (var p = Process.Start(pInfo_py))
{
p.WaitForExit();
p.Close();
}
}
// CSVの保存
Invoke((MethodInvoker)delegate
{
label7.BackColor = Color.White;
label_state.Text = "保存中...";
});
// featuresを1つにまとめる
using (var sw = new StreamWriter(path_outputCsv, true))
{
var swLine = name_file + ",";
using (var sr = new StreamReader(path_outputDir + @"\4-featureExtraction\shape\" + name_file + ".csv"))
{
sr.ReadLine();
var l = sr.ReadLine();
var l_ = l.Substring(l.IndexOf(',') + 1) + ",";
swLine += l_;
}
for (int m = 0; m < regionMethod.Count(); m++)
{
using (var sr = new StreamReader(path_outputDir + @"\4-featureExtraction\region\" + regionMethod[m] + @"\GLCM\" + name_file + ".csv"))
{
sr.ReadLine();
var l = "";
var l_dst = "";
while ((l = sr.ReadLine()) != null)
{
var l_ = l.Substring(l.IndexOf(',') + 1);
l_dst += l_ + ",";
}
swLine += l_dst;
}
}
for (int m = 0; m < regionMethod.Count(); m++)
{
using (var sr = new StreamReader(path_outputDir + @"\4-featureExtraction\region\" + regionMethod[m] + @"\colorStats\" + name_file + ".csv"))
{
sr.ReadLine();
var l = "";
var l_dst = "";
while ((l = sr.ReadLine()) != null)
{
var l_ = l.Substring(l.IndexOf(',') + 1);
l_dst += l_ + ",";
}
swLine += l_dst;
}
}
swLine = swLine.Substring(0, swLine.LastIndexOf(','));
sw.WriteLine(swLine);
}
// 終了処理
var faster = false;
var waitTime = faster ? 0 : 1000;
Thread.Sleep(waitTime);
Invoke((MethodInvoker)delegate
{
pictureBox_input.Image = null;
pictureBox_input.Refresh();
pictureBox_calibration.Image = null;
pictureBox_calibration.Refresh();
pictureBox_detection.Image = null;
pictureBox_detection.Refresh();
pictureBox_segmentation.Image = null;
pictureBox_segmentation.Refresh();
pictureBox_mask.Image = null;
pictureBox_mask.Refresh();
pictureBox_extracted.Image = null;
pictureBox_extracted.Refresh();
pictureBox_regions.Image = null;
pictureBox_regions.Refresh();
pictureBox_last.Image = null;
pictureBox_last.Refresh();
});
GC.Collect();
}
MessageBox.Show("Finished!");
Invoke((MethodInvoker)delegate
{
button_start.Enabled = true;
button_pause.Enabled = false;
label_processingFileName.Text = "Processing File: None";
});
}
//
private void BackgroundWorker1_DoWork_ImageAndCalib(object sender, DoWorkEventArgs e)
{
// ステータスの表示
Invoke((MethodInvoker)delegate
{
label_state.Text = "準備中..";
});
// Path
var path_imgDir = @"..\Input\TongueImage";
var path_calibDir = @"..\Input\CalibrationColorChart";
var path_outputDir = @"..\Output/" + DateTime.Now.ToLocalTime().ToString("yyyyMMddhhmmss");
MyMakeDir(path_outputDir);
FolderCopy("template_outputDir_withCalib", path_outputDir);
path_outputDir = @"..\Output/" + "debug"; // for debug
var path_outputCsv = path_outputDir + @"\analysis.csv";
var count = 0;
foreach (var path_img in Directory.GetFiles(path_imgDir))
{
var name_file = Path.GetFileNameWithoutExtension(path_img);
// 校正画像探索
var path_calib = Directory.GetFiles(path_calibDir).Where(x => Path.GetFileNameWithoutExtension(path_img) == Path.GetFileNameWithoutExtension(x)).ToList()[0];
if (path_calib == null)
Console.Write("No calib sheet");
// ステータスの表示
Invoke((MethodInvoker)delegate
{
label_processingFileName.Text = "Processing File: " + path_img;
count += 1;
label_totalProgress.Text = "Total Progress: " + count + "/" + Directory.GetFiles(path_imgDir).Length;
});
// Init
manualReset.WaitOne();
Invoke((MethodInvoker)delegate
{
label8.Text = "";
label8.BackColor = Color.White;
label1.BackColor = Color.Red;
});
// 舌画像表示
Mat mat_input = Cv2.ImRead(path_img, ImreadModes.Color);
Invoke((MethodInvoker)delegate
{
label1.BackColor = Color.White;
label2.BackColor = Color.Red;
pictureBox_input.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_input);
pictureBox_input.Refresh();
label_state.Text = "色校正";
});
// 色校正・変換 RGB->lab
var path_labCsv = path_outputDir + @"\3-imgProc\cvtColor\LabCsv\" + name_file + ".csv";
var pInfo = new ProcessStartInfo();
pInfo.Arguments = path_img + " " + path_calib + " " + path_labCsv;
pInfo.FileName = @"rgb2lab\TongueAnalysis.exe";
using (var p = Process.Start(pInfo)) {
p.WaitForExit();
p.Close();
}
// Lab画像読み込み 変換 保存
Mat mat_lab = new Mat(mat_input.Size(), MatType.CV_32FC3);
Mat mat_lab_show = new Mat(mat_input.Size(), MatType.CV_8UC3);
using (var sr_lab = new StreamReader(path_labCsv))
{
for (int i = 0; i < mat_input.Height; i++)
{
var line = sr_lab.ReadLine();
var strs = line.Split(',');
for (int j = 0; j < mat_input.Width; j++)
{
var c = j * 3;
var str_l = strs[c];
var str_a = strs[c + 1];
var str_b = strs[c + 2];
float l_, a_, b_;
float.TryParse(str_l, out l_);
float.TryParse(str_a, out a_);
float.TryParse(str_b, out b_);
mat_lab.Set(i, j, new Vec3f(l_, a_, b_));
}
}
}
Cv2.CvtColor(mat_lab, mat_lab_show, ColorConversionCodes.Lab2BGR);
mat_lab_show.ConvertTo(mat_lab_show, MatType.CV_8UC3, 256.0);
Cv2.ImWrite(path_outputDir + @"\3-imgProc\cvtColor\LabImg\" + name_file + ".png", mat_lab_show);
// Lab画像 表示
Invoke((MethodInvoker)delegate
{
label2.BackColor = Color.White;
label3.BackColor = Color.Red;
pictureBox_calibration.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_lab_show);
pictureBox_calibration.Refresh();
});
// Detection
var dst_rect = new Rect();
try
{
dst_rect = TongueDetection(mat_lab_show);
}
catch (Exception)
{
using (var sw = new StreamWriter(path_outputCsv, true))
{
var swLine = name_file + "," + "Tongue detection error";
sw.WriteLine(swLine);
}
continue;
}
if ((dst_rect.Width == 0) || (dst_rect.Height == 0))
{
using (var sw = new StreamWriter(path_outputCsv, true))
{
var swLine = name_file + "," + "Tongue detection error";
sw.WriteLine(swLine);
}
continue;
}
var mat_drawBox = mat_lab_show.Clone();
Cv2.Rectangle(mat_drawBox, dst_rect, new Scalar(0, 255, 0), 3);
mat_drawBox.SaveImage(path_outputDir + @"\1-detection\1-drawBox\" + name_file + ".png");
// Detection領域crop
var mat_cropped = new Mat(mat_lab_show, dst_rect);
mat_cropped.SaveImage(path_outputDir + @"\1-detection\2-cropped\" + name_file + ".png");
// Resize
var mat_roisize = mat_lab_show.Clone(dst_rect);
var mat_roi = new Mat();
Cv2.Resize(mat_roisize, mat_roi, new OpenCvSharp.Size(256, 256));
mat_roi.SaveImage(path_outputDir + @"\1-detection\3-resized\" + name_file + ".png");
// Detection 表示
Invoke((MethodInvoker)delegate
{
label3.BackColor = Color.White;
label4.BackColor = Color.Red;
pictureBox_detection.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_roi); ;
pictureBox_detection.Refresh();
});
// Segmenation
var mat_output = TongueSegmentation(mat_roi);
Cv2.Threshold(mat_output, mat_output, 128, 255, ThresholdTypes.Otsu | ThresholdTypes.Binary);
Thread.Sleep(100);
GC.Collect();
mat_output.SaveImage(path_outputDir + @"\2-segmentation\1-modelOutput\" + name_file + ".png");
// 2値化
var mat_binOutput = new Mat();
Cv2.Threshold(mat_output, mat_binOutput, 0, 255, ThresholdTypes.Otsu);
mat_binOutput.SaveImage(path_outputDir + @"\2-segmentation\2-binOutput\" + name_file + ".png");
// Segmenation 表示
Invoke((MethodInvoker)delegate
{
label3.BackColor = Color.White;
label4.BackColor = Color.Red;
pictureBox_segmentation.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_binOutput); ;
pictureBox_segmentation.Refresh();
});
// Segmentationの例外処理
var edges = new Mat();
OpenCvSharp.Point[][] contours;
HierarchyIndex[] hierarchyIndexes;
Cv2.Canny(mat_binOutput, edges, 50, 200);
Cv2.FindContours(edges, out contours, out hierarchyIndexes, RetrievalModes.List, ContourApproximationModes.ApproxSimple);
// 1 or 2つ以外の輪郭が検出された場合
var num_obj = contours.Count();
if ((num_obj == 0) || (num_obj > 2))
{
Console.WriteLine("Segmentation result has over 1 area : extracted areas = {0}", num_obj);
try
{
RemoveSmallRegion(path_outputDir + @"\2-segmentation\2-binOutput\" + name_file + ".png", path_outputDir + @"\2-segmentation\2-binOutput(refined)\" + name_file + ".png", 500, 1, 1);
RemoveSmallRegion(path_outputDir + @"\2-segmentation\2-binOutput(refined)\" + name_file + ".png", path_outputDir + @"\2-segmentation\2-binOutput(refined)\" + name_file + ".png", 500, 0, 0);
}
catch
{
MessageBox.Show("Error: Unable to reprocess! Please check is there [RemoveSmallRegionDLL.dll] file in floder?", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
break;
}
// 再度確認
mat_binOutput = Cv2.ImRead(path_outputDir + @"\2-segmentation\2-binOutput(refined)\" + name_file + ".png", ImreadModes.Grayscale);
Cv2.ImShow("dst2", mat_binOutput);
Cv2.WaitKey();
Cv2.Canny(mat_binOutput, edges, 50, 200);
Cv2.FindContours(edges, out contours, out hierarchyIndexes, RetrievalModes.List, ContourApproximationModes.ApproxSimple);
num_obj = contours.Count();
if ((num_obj == 0) || (num_obj > 2))
{
using (var sw = new StreamWriter(path_outputCsv, true))
{
Console.WriteLine("Tongue segmentation error");
var swLine = name_file + "," + "Tongue segmentation error";
sw.WriteLine(swLine);
}
continue;
}
}
// Resize
var mat_resizedOutput = new Mat();
Cv2.Resize(mat_binOutput, mat_resizedOutput, dst_rect.Size);
mat_resizedOutput.SaveImage(path_outputDir + @"\2-segmentation\3-resized\" + name_file + ".png");
// マスクの作成
var mat_mask = new Mat(mat_input.Size(), MatType.CV_8UC1, 0);
for (int i = dst_rect.Top; i < dst_rect.Bottom; i++)
for (int j = dst_rect.Left; j < dst_rect.Right; j++)
{
var value = mat_resizedOutput.At<byte>(i - dst_rect.Top, j - dst_rect.Left);
mat_mask.Set(i, j, value);
}
mat_mask.SaveImage(path_outputDir + @"\2-segmentation\4-mask\" + name_file + ".png");
// マスク 表示
Invoke((MethodInvoker)delegate
{
label4.BackColor = Color.White;
label5.BackColor = Color.Red;
pictureBox_mask.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_mask);
pictureBox_mask.Refresh();
});
// 舌領域の抽出
var mat_extraction = new Mat();
mat_input.CopyTo(mat_extraction, mat_mask);
mat_extraction.SaveImage(path_outputDir + @"\2-segmentation\5-extraction\" + name_file + ".png");
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
pictureBox_extracted.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_extraction);
pictureBox_extracted.Refresh();
label5.BackColor = Color.White;
label6.BackColor = Color.Red;
label_state.Text = "舌特徴抽出中...";
});
// 面積，舌検出座標 保存
var areaCount = 0;
using (var mat_temp = new Mat())
{
Cv2.CvtColor(mat_extraction, mat_temp, ColorConversionCodes.BGR2GRAY);
areaCount = Cv2.CountNonZero(mat_temp);
}
using (var sw = new StreamWriter(path_outputDir + @"\4-featureExtraction\shape\" + name_file + ".csv"))
{
sw.WriteLine(",width,height,area," +
"topLeftX,topLeftY,bottomRightX,bottomRightY");
sw.Write(
"shape," + dst_rect.Width.ToString() + "," + dst_rect.Height.ToString() + "," + areaCount.ToString() + "," +
dst_rect.TopLeft.X.ToString() + "," + dst_rect.TopLeft.Y.ToString() + "," +
dst_rect.BottomRight.X.ToString() + "," + dst_rect.BottomRight.Y.ToString());
}
// 舌分画に基づく特徴量計算1: Chiu
var chiu = new ColorExtractor_Chiu();
var mat_chiu = chiu.Get3areaMask(mat_mask);
var path_chiuMask = path_outputDir + @"\3-imgProc\devideRegion\chiu\" + name_file + ".png";
mat_chiu.SaveImage(path_outputDir + @"\3-imgProc\devideRegion\chiu\" + name_file + ".png");
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
label6.BackColor = Color.White;
label7.BackColor = Color.Red;
pictureBox_regions.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_chiu);
pictureBox_regions.Refresh();
});
// 舌分画に基づく特徴量計算2: ExtendedChiu
var exChiu = new ColorExtractor_ExtendedChiu();
var mat_exChiu = exChiu.Get4areaMask(mat_mask);
mat_exChiu.SaveImage(path_outputDir + @"\3-imgProc\devideRegion\exChiu\" + name_file + ".png");
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
pictureBox_regions.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_exChiu);
pictureBox_regions.Refresh();
});
// 舌分画に基づく特徴量計算3: fiveclick [ishikawa, 2010]
var fiveClick = new ColorExtractor_fiveClick();
var mat_fiveClick = fiveClick.Get4areaMask(mat_mask);
mat_fiveClick.SaveImage(path_outputDir + @"\3-imgProc\devideRegion\fiveClick\" + name_file + ".png");
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
pictureBox_regions.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_fiveClick);
pictureBox_regions.Refresh();
label7.BackColor = Color.White;
});
var regionMethod = new List<string> { "chiu", "exChiu", "fiveClick" };
// Lab
for (int m = 0; m < regionMethod.Count(); m++)
{
// 色統計量，GLCM計算
var path_regionMask = path_outputDir + @"\3-imgProc\devideRegion\" + regionMethod[m] + "\\" + name_file + ".png";
var path_GLCM = path_outputDir + @"\4-featureExtraction\region\" + regionMethod[m] + @"\GLCM\" + name_file + ".csv";
var path_colorStats = path_outputDir + @"\4-featureExtraction\region\" + regionMethod[m] + @"\colorStats\" + name_file + ".csv";
var pInfo_py = new ProcessStartInfo(@"python\TongueFeatures.exe")
{
UseShellExecute = false,
RedirectStandardOutput = true,
CreateNoWindow = true,
Arguments = path_labCsv + " " + path_regionMask + " " + path_colorStats + " " + path_GLCM
};
using (var p = Process.Start(pInfo_py))
{
p.WaitForExit();
p.Close();
}
}
// 舌領域の抽出 表示
Invoke((MethodInvoker)delegate
{
label_state.Text = "保存中...";
});
// featuresを1つにまとめる
using (var sw = new StreamWriter(path_outputCsv, true))
{
var swLine = name_file + ",";
using (var sr = new StreamReader(path_outputDir + @"\4-featureExtraction\shape\" + name_file + ".csv"))
{
sr.ReadLine();
var l = sr.ReadLine();
var l_ = l.Substring(l.IndexOf(',') + 1) + ",";
swLine += l_;
}
for (int m = 0; m < regionMethod.Count(); m++)
{
using (var sr = new StreamReader(path_outputDir + @"\4-featureExtraction\region\" + regionMethod[m] + @"\GLCM\" + name_file + ".csv"))
{
sr.ReadLine();
var l = "";
var l_dst = "";
while ((l = sr.ReadLine()) != null)
{
var l_ = l.Substring(l.IndexOf(',') + 1);
l_dst += l_ + ",";
}
swLine += l_dst;
}
}
for (int m = 0; m < regionMethod.Count(); m++)
{
using (var sr = new StreamReader(path_outputDir + @"\4-featureExtraction\region\" + regionMethod[m] + @"\colorStats\" + name_file + ".csv"))
{
sr.ReadLine();
var l = "";
var l_dst = "";
while ((l = sr.ReadLine()) != null)
{
var l_ = l.Substring(l.IndexOf(',') + 1);
l_dst += l_ + ",";
}
swLine += l_dst;
}
}
swLine = swLine.Substring(0, swLine.LastIndexOf(','));
sw.WriteLine(swLine);
}
// 終了処理
var faster = false;
var waitTime = faster ? 0 : 1000;
Thread.Sleep(waitTime);
Invoke((MethodInvoker)delegate
{
pictureBox_input.Image = null;
pictureBox_input.Refresh();
pictureBox_calibration.Image = null;
pictureBox_calibration.Refresh();
pictureBox_detection.Image = null;
pictureBox_detection.Refresh();
pictureBox_segmentation.Image = null;
pictureBox_segmentation.Refresh();
pictureBox_mask.Image = null;
pictureBox_mask.Refresh();
pictureBox_extracted.Image = null;
pictureBox_extracted.Refresh();
pictureBox_regions.Image = null;
pictureBox_regions.Refresh();
pictureBox_last.Image = null;
pictureBox_last.Refresh();
});
GC.Collect();
}
MessageBox.Show("Finished!");
Invoke((MethodInvoker)delegate
{
button_start.Enabled = true;
button_pause.Enabled = false;
label_processingFileName.Text = "Processing File: None";
});
}
private OpenCvSharp.Rect TongueDetection(Mat mat_input)
{
var byte_input = Bitmap2Byte(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(mat_input));
var rect_dst = new OpenCvSharp.Rect();
using (var graph = new TFGraph())
{
var model = File.ReadAllBytes(path_detectionModel);
graph.Import(model, "");
using (var session = new TFSession(graph))
{
var tensor = ImageUtil.CreateTensorFromImageFile(byte_input, TFDataType.UInt8);
var runner = session.GetRunner();
runner
.AddInput(graph["image_tensor"][0], tensor)
.Fetch("detection_boxes", "detection_scores", "detection_classes", "num_detections");
var output = runner.Run();
var boxes = (float[,,])output[0].GetValue();
var scores = (float[,])output[1].GetValue();
var classes = (float[,])output[2].GetValue();
var detections = (float[])output[3].GetValue();
var check_detection = 0;
max_score = 0;
for (int i = 0; i < scores.Length; i++)
{
if ((scores[0, i] > 0.5) && (scores[0, i] > max_score))
{
max_score = scores[0, i];
float y_min = boxes[0, i, 0] * (float)mat_input.Height;
float x_min = boxes[0, i, 1] * (float)mat_input.Width;
float y_max = boxes[0, i, 2] * (float)mat_input.Height;
float x_max = boxes[0, i, 3] * (float)mat_input.Width;
rect_dst.X = (int)x_min;
rect_dst.Y = (int)y_min;
rect_dst.Width = (int)(x_max - x_min);
rect_dst.Height = (int)(y_max - y_min);
check_detection = 1;
}
}
}
}
return rect_dst;
}
// Ref:https://book2525stack.com/2021/06/03/net_hand_written_digit_recognition/
private Mat TongueSegmentation(Mat mat_input)
{
var mat_dst = new Mat(mat_input.Size(), MatType.CV_8UC1);
// 推論
using (var session = new InferenceSession(path_segmentationModel))
{
var inputMeta = session.InputMetadata;
var container = new List<NamedOnnxValue>();
/* input shape = [1, 3, 256, 256]
* [batchsize, channels, w, h]
*/
var inputData = new float[1, mat_input.Channels(), mat_input.Width, mat_input.Height];
var inputData_flat = new float[mat_input.Channels() * mat_input.Width * mat_input.Height];
for (int c = 0; c < mat_input.Channels(); c++)
for (int i = 0; i < mat_input.Height; i++)
for (int j = 0; j < mat_input.Width; j++)
inputData_flat[j + (i * mat_input.Width) + (c * (mat_input.Width * mat_input.Height))] =
(float)(mat_input.At<Vec3b>(i, j)[c] / 255.0);
var tensor = new DenseTensor<float>(inputData_flat, inputMeta[inputMeta.Keys.First()].Dimensions, false);
var namedOnnxValues = new List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor(inputMeta.Keys.First(), tensor)
};
/* output shape = [1, 2, 256, 256]
* [batchsize, class proba, w, h]
*/
using (var results = session.Run(namedOnnxValues))
{
var result = results.First().AsTensor<float>().ToArray();
var mat_result = new float[mat_input.Width, mat_input.Height];
var list_result = new List<float>();
for (int c = 0; c < 2; c++)
for (int i = 0; i < mat_input.Height; i++)
for (int j = 0; j < mat_input.Width; j++)
{
var proba = result[j + (i * mat_input.Width) + (c * (mat_input.Width * mat_input.Height))];
if (c == 0)
{
mat_result[j, i] = proba;
list_result.Add(proba);
}
}
for (int i = 0; i < mat_input.Height; i++)
for (int j = 0; j < mat_input.Width; j++)
{
// Sigmoid * 255
var value = (byte)((float)(1.0 / (1.0 + Math.Pow(Math.E, -(mat_result[j, i])))) * 255.0);
mat_dst.Set<byte>(i, j, value);
}
}
}
GC.Collect();
return mat_dst;
}
private static class ImageUtil
{
// Convert the image in filename to a Tensor suitable as input to the Inception model.
public static TFTensor CreateTensorFromImageFile(byte[] contents, TFDataType destinationDataType = TFDataType.UInt8)
{
// DecodeJpeg uses a scalar String-valued tensor as input.
var tensor = TFTensor.CreateString(contents);
TFGraph graph;
TFOutput input, output;
// Construct a graph to normalize the image
ConstructGraphToNormalizeImage(out graph, out input, out output, destinationDataType);
// Execute that graph to normalize this one image
using (var session = new TFSession(graph))
{
var normalized = session.Run(
inputs: new[] { input },
inputValues: new[] { tensor },
outputs: new[] { output });
if (session != null)
{
session.Dispose();
}
if (tensor != null)
{
tensor.Dispose();
}
if (graph != null)
{
graph.Dispose();
}
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
return normalized[0];
}
}
// The inception model takes as input the image described by a Tensor in a very
// specific normalized format (a particular image size, shape of the input tensor,
// normalized pixel values etc.).
//
// This function constructs a graph of TensorFlow operations which takes as
// input a JPEG-encoded string and returns a tensor suitable as input to the
// inception model.
public static void ConstructGraphToNormalizeImage(out TFGraph graph, out TFOutput input, out TFOutput output, TFDataType destinationDataType = TFDataType.UInt8)
{
const int W = 256;
const int H = 256;
const float Mean = 0;
const float Scale = 1;
graph = new TFGraph();
input = graph.Placeholder(TFDataType.String);
output = graph.Cast(graph.Div(
x: graph.Sub(
x: graph.ResizeBilinear(
images: graph.ExpandDims(
input: graph.Cast(
graph.DecodeJpeg(contents: input, channels: 3), DstT: destinationDataType),
dim: graph.Const(0, "make_batch")),
size: graph.Const(new int[] { W, H }, "size")),
y: graph.Const(Mean, "mean")),
y: graph.Const(Scale, "scale")), destinationDataType);
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
}
}
private static class ImageUtil2
{
// Convert the image in filename to a Tensor suitable as input to the Inception model.
public static TFTensor CreateTensorFromImageFile(byte[] contents, TFDataType destinationDataType = TFDataType.Float)
{
// DecodeJpeg uses a scalar String-valued tensor as input.
var tensor = TFTensor.CreateString(contents);
TFGraph graph;
TFOutput input, output;
// Construct a graph to normalize the image
ConstructGraphToNormalizeImage(out graph, out input, out output, destinationDataType);
// Execute that graph to normalize this one image
using (var session = new TFSession(graph))
{
var normalized = session.Run(
inputs: new[] { input },
inputValues: new[] { tensor },
outputs: new[] { output });
if (session != null)
{
session.Dispose();
}
if (tensor != null)
{
tensor.Dispose();
}
if (graph != null)
{
graph.Dispose();
}
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
return normalized[0];
}
}
// The inception model takes as input the image described by a Tensor in a very
// specific normalized format (a particular image size, shape of the input tensor,
// normalized pixel values etc.).
//
// This function constructs a graph of TensorFlow operations which takes as
// input a JPEG-encoded string and returns a tensor suitable as input to the
// inception model.
public static void ConstructGraphToNormalizeImage(out TFGraph graph, out TFOutput input, out TFOutput output, TFDataType destinationDataType = TFDataType.Float)
{
const int W = 256;
const int H = 256;
//const int W = 512;
//const int H = 512;
const float Mean = 128;
const float Scale = 128;
graph = new TFGraph();
input = graph.Placeholder(TFDataType.String);
output = graph.Cast(graph.Div(
x: graph.Sub(
x: graph.ResizeBilinear(
images: graph.ExpandDims(
input: graph.Cast(
graph.DecodeJpeg(contents: input, channels: 3), DstT: TFDataType.Float),
dim: graph.Const(0, "make_batch")),
size: graph.Const(new int[] { W, H }, "size")),
y: graph.Const(Mean, "mean")),
y: graph.Const(Scale, "scale")), destinationDataType);
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
}
}
private static byte[] Bitmap2Byte(Bitmap bitmap)
{
using (MemoryStream stream = new MemoryStream())
{
bitmap.Save(stream, ImageFormat.Jpeg);
byte[] data = new byte[stream.Length];
stream.Seek(0, SeekOrigin.Begin);
stream.Read(data, 0, Convert.ToInt32(stream.Length));
return data;
}
}
private static Bitmap ToGrayBitmap(byte[] rawValues, int width, int height)
{
//// Apply for a target bitmap variable and lock its memory area
Bitmap bmp = new Bitmap(width, height, PixelFormat.Format8bppIndexed);
BitmapData bmpData = bmp.LockBits(new Rectangle(0, 0, width, height),
ImageLockMode.WriteOnly, PixelFormat.Format8bppIndexed);
//// Get image parameters
int stride = bmpData.Stride; // Width of the scan line
int offset = stride - width; // Show gap between width and scan line width
IntPtr iptr = bmpData.Scan0; // Get the memory start position of bmpData
int scanBytes = stride * height;// Use stride width to indicate that this is the size of the memory area
//// The following is to convert the original display size byte array to the byte array actually stored in memory.
int posScan = 0, posReal = 0;// Set two position pointers respectively, pointing to the source array and the target array
byte[] pixelValues = new byte[scanBytes]; //Allocate memory for the target array
for (int x = 0; x < height; x++)
{
//// The following loop section is a simulated line scan
for (int y = 0; y < width; y++)
{
pixelValues[posScan++] = rawValues[posReal++];
}
posScan += offset; //At the end of the line scan, move the target position pointer over that "gap"
}
//// Use Marshal's Copy method to copy the just obtained memory byte array into BitmapData.
System.Runtime.InteropServices.Marshal.Copy(pixelValues, 0, iptr, scanBytes);
bmp.UnlockBits(bmpData); // Unlock the memory area
//// The following code is to modify the index table of the generated bitmap, from pseudo color to grayscale
ColorPalette tempPalette;
using (Bitmap tempBmp = new Bitmap(1, 1, PixelFormat.Format8bppIndexed))
{
tempPalette = tempBmp.Palette;
}
for (int i = 0; i < 256; i++)
{
tempPalette.Entries[i] = Color.FromArgb(i, i, i);
}
bmp.Palette = tempPalette;
//// The algorithm ends here and returns the result.
return bmp;
}
private void watcher_Renamed(System.Object source,
System.IO.RenamedEventArgs e)
{
Console.WriteLine(
"ファイル 「" + e.FullPath + "」の名前が変更されました。");
}
private void Button_pause_Click(object sender, EventArgs e)
{
if (button_pause.Text == "Pause")
{
manualReset.Reset();
button_pause.Text = "Continue";
}
else
{
manualReset.Set();
button_pause.Text = "Pause";
}
}
private void MyMakeDir(string path)
{
// ディレクトリが存在しなければ作成，存在すれば空にする
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
else
{
foreach (var dir in new DirectoryInfo(path).GetDirectories())
dir.Delete(true);
foreach (var file in new DirectoryInfo(path).GetFiles())
file.Delete();
}
}
static void FolderCopy(string src_str, string dest_str)
{
DirectoryInfo src_info = new DirectoryInfo(src_str);
// コピー元の存在チェック
if (!src_info.Exists)
{
Console.WriteLine("ERROR:" +src_str + " doesn’t exist!");
return;
}
// コピー先のフォルダ作成
Directory.CreateDirectory(dest_str);
// コピー元のフォルダとファイル情報を取得
DirectoryInfo[] src_folders = src_info.GetDirectories();
FileInfo[] src_files = src_info.GetFiles();
// コピー元のファイルを全てコピー先のフォルダに上書きコピー
foreach (FileInfo file in src_files)
{
string path = Path.Combine(dest_str, file.Name);
file.CopyTo(path, true);
}
// 再起呼び出しによりコピー元のフォルダを全てコピー先のフォルダにコピー
foreach (DirectoryInfo subfolder in src_folders)
{
string path = Path.Combine(dest_str, subfolder.Name);
FolderCopy(subfolder.FullName, path);
}
}
}
}
