diff --git a/ECTrainer2/BaseProcess.cpp b/ECTrainer2/BaseProcess.cpp new file mode 100644 index 0000000..52e924d --- /dev/null +++ b/ECTrainer2/BaseProcess.cpp @@ -0,0 +1,21 @@ +#include "BaseProcess.h" +#include "ECTrainer.h" +#include + +// �R���X�g���N�^ +BaseProcess::BaseProcess(ECTrainer* pEct) : _pEct(pEct) { + +} + +// ������ +bool BaseProcess::Init() { + return true; +} + +// ���[�v +bool BaseProcess::MainLoop() { + while (_pEct->IsRunning()) { + Sleep(1); + } + return true; +} diff --git a/ECTrainer2/BaseProcess.h b/ECTrainer2/BaseProcess.h new file mode 100644 index 0000000..93ae5c2 --- /dev/null +++ b/ECTrainer2/BaseProcess.h @@ -0,0 +1,15 @@ +#pragma once + +class ECTrainer; + +// �e�����̊�{�N���X +class BaseProcess +{ +protected: + ECTrainer* _pEct; + +public: + BaseProcess(ECTrainer* pEct); + virtual bool Init(); + virtual bool MainLoop(); +}; diff --git a/ECTrainer2/ECTrainer.cpp b/ECTrainer2/ECTrainer.cpp index b2676f6..f7eda3f 100644 --- a/ECTrainer2/ECTrainer.cpp +++ b/ECTrainer2/ECTrainer.cpp @@ -4,6 +4,7 @@ #include "SceneCamera.h" #include "Stimulus.h" #include "Marker.h" +#include "ImageProc.h" #ifdef _DEBUG #include @@ -15,11 +16,14 @@ , _pSceneCam(NULL) , _pStimulus(NULL) , _pMarker(NULL) + , _pImageProc(NULL) , _Running(true) + , _HomographyOK(false) { _pMarker = new Marker(); _pGui = new ECTrainerGUI(this); - _pSceneCam = new SceneCamera(this); + _pImageProc = new ImageProc(this, _pMarker); + _pSceneCam = new SceneCamera(this, _pImageProc); _pStimulus = new Stimulus(this, _pMarker); } @@ -29,29 +33,33 @@ if (_pSceneCam) delete _pSceneCam; if (_pStimulus) delete _pStimulus; if (_pMarker) delete _pMarker; + if (_pImageProc) delete _pImageProc; } // ������ bool ECTrainer::Process() { if (!_pGui->Init()) return false; - //if (!_pSceneCam->Init()) return false; + if (!_pSceneCam->Init()) return false; if (!_pStimulus->Init()) return false; + if (!_pImageProc->Init()) return false; - DWORD dwThreadIdSceneCam, dwThreadIdStimulus; - //HANDLE hThreadSceneCam = CreateThread(NULL, 0, SceneCamThreadEntry, this, 0, &dwThreadIdSceneCam); + DWORD dwThreadIdSceneCam, dwThreadIdStimulus, dwThreadIdImageProc; + HANDLE hThreadSceneCam = CreateThread(NULL, 0, SceneCamThreadEntry, this, 0, &dwThreadIdSceneCam); HANDLE hThreadStimulus = CreateThread(NULL, 0, StimulusThreadEntry, this, 0, &dwThreadIdStimulus); + HANDLE hThreadImageProc = CreateThread(NULL, 0, ImageProcThreadEntry, this, 0, &dwThreadIdImageProc); _pGui->MainLoop(); - //HANDLE handles[] = { hThreadSceneCam , hThreadStimulus }; -// DWORD timeOut = 1000; // �^�C���A�E�g(ms) -// if (WaitForMultipleObjects(sizeof(handles) / sizeof(HANDLE), handles, TRUE, timeOut) != WAIT_TIMEOUT) { -//#ifdef _DEBUG -// std::cout << "All threads properly ended." << std::endl; -//#endif -// } - //CloseHandle(hThreadSceneCam); - CloseHandle(hThreadStimulus); + HANDLE handles[] = { hThreadSceneCam , hThreadStimulus, hThreadImageProc }; + DWORD timeOut = 1000; // �^�C���A�E�g(ms) + if (WaitForMultipleObjects(sizeof(handles) / sizeof(HANDLE), handles, TRUE, timeOut) != WAIT_TIMEOUT) { +#ifdef _DEBUG + std::cout << "All threads properly ended." << std::endl; +#endif + } + if (hThreadSceneCam) CloseHandle(hThreadSceneCam); + if (hThreadStimulus) CloseHandle(hThreadStimulus); + if (hThreadImageProc) CloseHandle(hThreadImageProc); return true; } @@ -87,3 +95,9 @@ ((ECTrainer*)lpParameter)->_pStimulus->MainLoop(); return 0; } + +// �摜�����X���b�h�J�n�_ +DWORD WINAPI ECTrainer::ImageProcThreadEntry(LPVOID lpParameter) { + ((ECTrainer*)lpParameter)->_pImageProc->MainLoop(); + return 0; +} diff --git a/ECTrainer2/ECTrainer.h b/ECTrainer2/ECTrainer.h index d264532..5347a89 100644 --- a/ECTrainer2/ECTrainer.h +++ b/ECTrainer2/ECTrainer.h @@ -7,6 +7,7 @@ class SceneCamera; class Stimulus; class Marker; +class ImageProc; class ECTrainer { @@ -15,10 +16,13 @@ SceneCamera* _pSceneCam; Stimulus* _pStimulus; Marker* _pMarker; + ImageProc* _pImageProc; bool _Running; + bool _HomographyOK; // �X���b�h�J�n�_ static DWORD SceneCamThreadEntry(LPVOID lpParameter); static DWORD StimulusThreadEntry(LPVOID lpParameter); + static DWORD ImageProcThreadEntry(LPVOID lpParameter); public: ECTrainer(); @@ -31,5 +35,6 @@ // �C�����C���֐� bool IsRunning() { return _Running; } void Stop() { _Running = false; } + void SetHomographyStatus(bool ok) { _HomographyOK = ok; } + bool GetHomographyStatus() { return _HomographyOK; } }; - diff --git a/ECTrainer2/ECTrainer2.vcxproj b/ECTrainer2/ECTrainer2.vcxproj index d6d983d..52b89b6 100644 --- a/ECTrainer2/ECTrainer2.vcxproj +++ b/ECTrainer2/ECTrainer2.vcxproj @@ -142,17 +142,23 @@ + + + + + + diff --git a/ECTrainer2/ECTrainer2.vcxproj.filters b/ECTrainer2/ECTrainer2.vcxproj.filters index e0d22b8..882aa8c 100644 --- a/ECTrainer2/ECTrainer2.vcxproj.filters +++ b/ECTrainer2/ECTrainer2.vcxproj.filters @@ -33,6 +33,15 @@ ソース ファイル + + ソース ファイル + + + ソース ファイル + + + ソース ファイル + @@ -56,5 +65,14 @@ ヘッダー ファイル + + ヘッダー ファイル + + + ヘッダー ファイル + + + ヘッダー ファイル + \ No newline at end of file diff --git a/ECTrainer2/ECTrainerGUI.cpp b/ECTrainer2/ECTrainerGUI.cpp index 7cee977..3ccee5e 100644 --- a/ECTrainer2/ECTrainerGUI.cpp +++ b/ECTrainer2/ECTrainerGUI.cpp @@ -10,7 +10,7 @@ #endif // �R���X�g���N�^ -ECTrainerGUI::ECTrainerGUI(ECTrainer* pEct) :_pEct(pEct) { +ECTrainerGUI::ECTrainerGUI(ECTrainer* pEct) :BaseProcess(pEct) { } // ������ @@ -47,7 +47,6 @@ _SceneBuffer.create(SCENE_BUFFER_SIZE, CV_8UC3); _SceneBuffer = CV_RGB(0, 0, 0); _Logo = cv::imread("../images/ECTrainerLogo640x91.png"); - //_DispBuffer = cv::imread("../images/calib.png"); return true; } @@ -58,19 +57,22 @@ // ���C���E�C���h�E�t���[������ _MainFrame = cv::Scalar(49, 52, 49); + cv::Point mp = cvui::mouse(WIN_MAIN); cvui::image(_MainFrame, 0, 0, _Logo); cvui::beginRow(_MainFrame, 10, 100, -1, -1, 10); - cvui::beginColumn(-1, -1, 10); - if (cvui::button("Calibration")) _pEct->Calib(); - if (cvui::button("Start")) _pEct->Start(); - if (cvui::button("Quit")) _pEct->Stop(); + cvui::beginColumn(100, -1, 10); + if (cvui::button(100, 20, "Calibration")) _pEct->Calib(); + if (cvui::button(100, 20, "Start")) _pEct->Start(); + if (cvui::button(100, 20, "Quit")) _pEct->Stop(); + cvui::printf("mouse %d,%d", mp.x, mp.y); cvui::endColumn(); cvui::beginColumn(); - cvui::text("View camera"); - cvui::space(); cvui::image(_SceneBuffer); + cvui::space(); + cvui::text(_pEct->GetHomographyStatus() ? "Homography OK" : "No homography"); cvui::endColumn(); cvui::endRow(); + cvui::rect(_MainFrame, 120, 100, SCENE_BUFFER_SIZE.width, SCENE_BUFFER_SIZE.height, 0xFF0000); // ��ʕ\�� cvui::update(); diff --git a/ECTrainer2/ECTrainerGUI.h b/ECTrainer2/ECTrainerGUI.h index b6a5431..48e4677 100644 --- a/ECTrainer2/ECTrainerGUI.h +++ b/ECTrainer2/ECTrainerGUI.h @@ -4,21 +4,19 @@ #include #include "myOpenCV.h" #include +#include "BaseProcess.h" -class ECTrainer; - -class ECTrainerGUI +class ECTrainerGUI : public BaseProcess { private: const cv::String WIN_DISP = "ECTrainer Display"; const cv::String WIN_MAIN = "Eye Contact Trainer"; //const cv::Size MAIN_FRAME_SIZE = cv::Size(640, 400); //const cv::Size SCENE_BUFFER_SIZE = cv::Size(320, 180); - const cv::Size MAIN_FRAME_SIZE = cv::Size(1600, 1000); - const cv::Size SCENE_BUFFER_SIZE = cv::Size(1280, 720); + const cv::Size MAIN_FRAME_SIZE = cv::Size(1000, 650); + const cv::Size SCENE_BUFFER_SIZE = cv::Size(640, 480); const int KEY_ESC = 27; - ECTrainer* _pEct; std::vector _Displays; cv::Mat _MainFrame; // ���C���E�C���h�E�t���[�� cv::Mat _DispBuffer; // �h���摜 @@ -29,7 +27,7 @@ static BOOL CALLBACK MonitorEnumProc(HMONITOR hMon, HDC hdcMon, LPRECT lpMon, LPARAM dwDate); public: - ECTrainerGUI(ECTrainer* ect); + ECTrainerGUI(ECTrainer* pEct); bool Init(); bool MainLoop(); void SetSceneBuffer(cv::Mat &img); diff --git a/ECTrainer2/EyeTrack.cpp b/ECTrainer2/EyeTrack.cpp new file mode 100644 index 0000000..5ae170a --- /dev/null +++ b/ECTrainer2/EyeTrack.cpp @@ -0,0 +1 @@ +#include "EyeTrack.h" diff --git a/ECTrainer2/EyeTrack.h b/ECTrainer2/EyeTrack.h new file mode 100644 index 0000000..22e0a17 --- /dev/null +++ b/ECTrainer2/EyeTrack.h @@ -0,0 +1,8 @@ +#pragma once + +#include "BaseProcess.h" + +class EyeTrack : public BaseProcess +{ +public: +}; diff --git a/ECTrainer2/ImageProc.cpp b/ECTrainer2/ImageProc.cpp new file mode 100644 index 0000000..f85c4f6 --- /dev/null +++ b/ECTrainer2/ImageProc.cpp @@ -0,0 +1,31 @@ + +#include "ECTrainer.h" +#include "ImageProc.h" +#include "Marker.h" + +// �R���X�g���N�^ +ImageProc::ImageProc(ECTrainer* pEct, Marker* pMarker) + :BaseProcess(pEct), _pMarker(pMarker), _Write(0), _Read(1) { +} + +// ���[�v +bool ImageProc::MainLoop() { + int lastRead = _Read; + while (_pEct->IsRunning()) { + while (_Read == lastRead && _pEct->IsRunning()) Sleep(1); + cv::Mat img = _Frames[_Read].clone(); + lastRead = _Read; + bool hOK = _pMarker->Detect(img); + _pEct->SetHomographyStatus(hOK); + _pEct->SetSceneBuffer(img); + } + + return true; +} + +// �摜��M +void ImageProc::SetImage(cv::Mat& img) { + img.copyTo(_Frames[_Write]); + _Read = _Write; + _Write = (_Write + 1) % FRAMES; +} diff --git a/ECTrainer2/ImageProc.h b/ECTrainer2/ImageProc.h new file mode 100644 index 0000000..dd16e0d --- /dev/null +++ b/ECTrainer2/ImageProc.h @@ -0,0 +1,21 @@ +#pragma once + +#include "myOpenCV.h" +#include "BaseProcess.h" + +class Marker; + +class ImageProc : public BaseProcess +{ + static const int FRAMES = 2; + + Marker* _pMarker; + cv::Mat _Frames[FRAMES]; + int _Write; + int _Read; + +public: + ImageProc(ECTrainer* pEct, Marker* pMarker); + bool MainLoop(); + void SetImage(cv::Mat& img); +}; diff --git a/ECTrainer2/Marker.cpp b/ECTrainer2/Marker.cpp index d8670cf..8982c65 100644 --- a/ECTrainer2/Marker.cpp +++ b/ECTrainer2/Marker.cpp @@ -47,11 +47,36 @@ } cv::Mat marker; + int outMargin = (int)(img.rows * OUTER_MARGIN); for (int i = 0; i < _Corners.size(); i++) { - cv::aruco::drawMarker(_Dictionary, i, _Size, marker); + cv::aruco::drawMarker(_Dictionary, i + 1, _Size, marker, 1); cv::cvtColor(marker, marker, cv::COLOR_GRAY2BGR); cv::Point p = cv::Point((int)(_Corners[i][0].x * img.cols), (int)(_Corners[i][0].y * img.rows)); + cv::Mat roi2(img, cv::Rect(p.x - outMargin, p.y - outMargin, _Size + outMargin * 2, _Size + outMargin * 2)); + roi2 = cv::Scalar(255, 255, 255); cv::Mat roi(img, cv::Rect(p, cv::Size(_Size, _Size))); marker.copyTo(roi); } } + +// �}�[�J�[�̌��o +bool Marker::Detect(cv::Mat& img) { + // �}�[�J�[�̌��o + std::vector mids; + std::vector> corners; + cv::Ptr parameters = cv::aruco::DetectorParameters::create(); + cv::aruco::detectMarkers(img, _Dictionary, corners, mids, parameters); + cv::aruco::drawDetectedMarkers(img, corners, mids); + + // �Ή��_�̗� + std::vector cornerW, cornerC; + for (int i = 0; i < corners.size(); i++) { + cornerC.push_back(corners[i][0]); + cornerW.push_back(_Corners[mids[i] - 1][0]); + } + if (cornerC.size() < 4) return false; + + // �z���O���t�B�s��̎Z�o + _Homography = findHomography(cornerC, cornerW); + return !_Homography.empty(); +} diff --git a/ECTrainer2/Marker.h b/ECTrainer2/Marker.h index a1acf6b..f2af4f7 100644 --- a/ECTrainer2/Marker.h +++ b/ECTrainer2/Marker.h @@ -6,13 +6,17 @@ class Marker { const cv::Point2f MARGIN = cv::Size2f(0.05F, 0.05F); + const float OUTER_MARGIN = 0.01F; const float HEIGHT = 0.1F; std::vector> _Corners; cv::Ptr _Dictionary; int _Size; + cv::Mat _Homography; + public: Marker(); void Generate(cv::Size imgsize); void DrawMarker(cv::Mat& img); + bool Detect(cv::Mat& img); }; diff --git a/ECTrainer2/SceneCamera.cpp b/ECTrainer2/SceneCamera.cpp index 52f54fd..95a5ab2 100644 --- a/ECTrainer2/SceneCamera.cpp +++ b/ECTrainer2/SceneCamera.cpp @@ -4,17 +4,19 @@ #include "ECTrainer.h" #include "SceneCamera.h" +#include "ImageProc.h" // �R���X�g���N�^ -SceneCamera::SceneCamera(ECTrainer* pEct) - :_pEct(pEct) +SceneCamera::SceneCamera(ECTrainer* pEct, ImageProc* pImageProc) + :BaseProcess(pEct), _pImageProc(pImageProc) { } // ������ bool SceneCamera::Init() { - _SceneCam.open("rtsp://" ADDR ":8554/live/scene"); + //_SceneCam.open("rtsp://" ADDR ":8554/live/scene"); + _SceneCam.open(0); if (!_SceneCam.isOpened()) { MessageBox(NULL, _T("cannot open camera " ADDR), NULL, 0); return false; @@ -29,21 +31,8 @@ // �V�[���B�e cv::Mat scene; _SceneCam >> scene; - - cv::Ptr dictionary = cv::aruco::getPredefinedDictionary(cv::aruco::DICT_4X4_50); - std::vector mids; - std::vector> corners; - cv::Ptr parameters = cv::aruco::DetectorParameters::create(); - cv::aruco::detectMarkers(scene, dictionary, corners, mids, parameters); - cv::aruco::drawDetectedMarkers(scene, corners, mids); - if (corners.size() > 0) { - cv::putText(scene, "0", cv::Point((int)corners[0][0].x, (int)corners[0][0].y), cv::FONT_HERSHEY_PLAIN, 2.0, CV_RGB(255, 0, 0)); - cv::putText(scene, "1", corners[0][1], cv::FONT_HERSHEY_PLAIN, 2.0, CV_RGB(255, 0, 0)); - cv::putText(scene, "2", corners[0][2], cv::FONT_HERSHEY_PLAIN, 2.0, CV_RGB(255, 0, 0)); - cv::putText(scene, "3", corners[0][3], cv::FONT_HERSHEY_PLAIN, 2.0, CV_RGB(255, 0, 0)); - } - - _pEct->SetSceneBuffer(scene); + _pImageProc->SetImage(scene); + Sleep(1); } return true; diff --git a/ECTrainer2/SceneCamera.h b/ECTrainer2/SceneCamera.h index 583baf3..37e7afc 100644 --- a/ECTrainer2/SceneCamera.h +++ b/ECTrainer2/SceneCamera.h @@ -1,18 +1,19 @@ #pragma once #include "myOpenCV.h" +#include "BaseProcess.h" #define ADDR "192.168.71.50" -class ECTrainer; +class ImageProc; -class SceneCamera +class SceneCamera : public BaseProcess { - ECTrainer* _pEct; + ImageProc* _pImageProc; cv::VideoCapture _SceneCam; public: - SceneCamera(ECTrainer* pEct); + SceneCamera(ECTrainer* pEct, ImageProc* pImageProc); bool Init(); bool MainLoop(); }; diff --git a/ECTrainer2/Stimulus.cpp b/ECTrainer2/Stimulus.cpp index 5b4de59..918b0c9 100644 --- a/ECTrainer2/Stimulus.cpp +++ b/ECTrainer2/Stimulus.cpp @@ -6,12 +6,7 @@ // �R���X�g���N�^ Stimulus::Stimulus(ECTrainer* pEct, Marker* pMarker) - :_pEct(pEct), _pMarker(pMarker) { -} - -// ������ -bool Stimulus::Init() { - return true; + :BaseProcess(pEct), _pMarker(pMarker) { } // ���[�v @@ -33,7 +28,6 @@ // �h���摜�J�n void Stimulus::StartImage() { cv::Mat img = cv::imread("../images/lena.jpg"); - //cv::Mat img = cv::imread("../images/face_F_L_E.png"); _pMarker->Generate(img.size()); _pMarker->DrawMarker(img); _pEct->SetDispBuffer(img); diff --git a/ECTrainer2/Stimulus.h b/ECTrainer2/Stimulus.h index 551ba3c..69e427e 100644 --- a/ECTrainer2/Stimulus.h +++ b/ECTrainer2/Stimulus.h @@ -1,20 +1,17 @@ #pragma once #include "myOpenCV.h" +#include "BaseProcess.h" -class ECTrainer; class Marker; -class Stimulus +class Stimulus : public BaseProcess { const cv::String CALIB_FILE = "../images/calib.png"; - - ECTrainer* _pEct; Marker* _pMarker; public: Stimulus(ECTrainer* pEct, Marker* pMarker); - bool Init(); bool MainLoop(); void StartCalib(); void StartImage(); diff --git a/ECTrainer2/myOpenCV.h b/ECTrainer2/myOpenCV.h index e53a66e..cf688ae 100644 --- a/ECTrainer2/myOpenCV.h +++ b/ECTrainer2/myOpenCV.h @@ -38,7 +38,7 @@ #pragma comment(lib, PREHEAD "videoio" CV_VERSION_STR CV_EXT_STR) // Video I/O #pragma comment(lib, PREHEAD "highgui" CV_VERSION_STR CV_EXT_STR) // High-level GUI //#pragma comment(lib, PREHEAD "video" CV_VERSION_STR CV_EXT_STR) // Video Analysis -//#pragma comment(lib, PREHEAD "calib3d" CV_VERSION_STR CV_EXT_STR) // Camera Calibration and 3D Reconstruction +#pragma comment(lib, PREHEAD "calib3d" CV_VERSION_STR CV_EXT_STR) // Camera Calibration and 3D Reconstruction //#pragma comment(lib, PREHEAD "features2d" CV_VERSION_STR CV_EXT_STR) // 2D Features Framework //#pragma comment(lib, PREHEAD "objdetect" CV_VERSION_STR CV_EXT_STR) // Object Detection //#pragma comment(lib, PREHEAD "dnn" CV_VERSION_STR CV_EXT_STR) // Deep Neural Network module