diff --git a/.gitignore b/.gitignore index 92b0b7c..4f66632 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ log/ ECTrainer2/log.txt +movies/ +voices/ diff --git a/ECTrainer2/ECTrainer.cpp b/ECTrainer2/ECTrainer.cpp index 0a8a320..99ab15a 100644 --- a/ECTrainer2/ECTrainer.cpp +++ b/ECTrainer2/ECTrainer.cpp @@ -8,7 +8,7 @@ #include "EyeTrack.h" // �R���X�g���N�^ -ECTrainer::ECTrainer() +ECTrainer::ECTrainer(HINSTANCE hInstance) : _pGui(NULL) , _pSceneCam(NULL) , _pStimulus(NULL) @@ -17,6 +17,7 @@ , _pEyeTrack(NULL) , _Running(true) , _HomographyOK(false) + , _hInstance(hInstance) { _pMarker = new Marker(); _pGui = new ECTrainerGUI(this); @@ -38,7 +39,7 @@ // ������ bool ECTrainer::Process() { - if (!_pGui->Init()) return false; + if (!_pGui->Init(_hInstance)) return false; if (!_pSceneCam->Init()) return false; if (!_pStimulus->Init()) return false; if (!_pImageProc->Init()) return false; diff --git a/ECTrainer2/ECTrainer.h b/ECTrainer2/ECTrainer.h index 5cdad2b..6ff0cb4 100644 --- a/ECTrainer2/ECTrainer.h +++ b/ECTrainer2/ECTrainer.h @@ -16,6 +16,7 @@ class ECTrainer { private: + HINSTANCE _hInstance; ECTrainerGUI* _pGui; SceneCamera* _pSceneCam; Stimulus* _pStimulus; @@ -33,7 +34,7 @@ static DWORD KeepAliveThreadEntry(LPVOID lpParameter); public: - ECTrainer(); + ECTrainer(HINSTANCE hInstance); ~ECTrainer(); bool Process(); void Calib(); diff --git a/ECTrainer2/ECTrainerGUI.cpp b/ECTrainer2/ECTrainerGUI.cpp index fe6ffa7..20fbb70 100644 --- a/ECTrainer2/ECTrainerGUI.cpp +++ b/ECTrainer2/ECTrainerGUI.cpp @@ -4,6 +4,11 @@ #include #pragma comment(lib,"winmm.lib") #include +#pragma comment(lib, "dxguid.lib") +#pragma comment(lib, "d3d9.lib") +#pragma comment(lib, "Strmiids.lib") +#define _WIN32_DCOM // CoInitializeEx�֐��̌Ăяo���ɕK�v + #define CVUI_IMPLEMENTATION #include "cvui.h" @@ -13,12 +18,47 @@ #endif // �R���X�g���N�^ -ECTrainerGUI::ECTrainerGUI(ECTrainer* pEct) :BaseProcess(pEct) { - _SceneBufferScale = 1.F; +ECTrainerGUI::ECTrainerGUI(ECTrainer* pEct) + : BaseProcess(pEct) + , _pD3D(NULL) + , _pD3DDev(NULL) + , _pGB(NULL) + , _pVMR9(NULL) + , _pSource(NULL) + , _pCGB2(NULL) + , _pMediaCont(NULL) + , pMediaEvent(NULL) + , _hWnd(NULL) + , _SceneBufferHeight(0) + , _DispImageHeight(0) + , _SceneBufferScale(1.F) + , _PlayIndex(0) +{ + _MovieFiles.push_back(_T("../movies/instruction.avi")); + _MovieFiles.push_back(_T("../movies/test4.avi")); + _MovieFiles.push_back(_T("../movies/instruction.avi")); + _MovieFiles.push_back(_T("../movies/test1.avi")); + _MovieFiles.push_back(_T("../movies/instruction.avi")); + _MovieFiles.push_back(_T("../movies/test3.avi")); + _MovieFiles.push_back(_T("../movies/instruction.avi")); + _MovieFiles.push_back(_T("../movies/test2.avi")); +} + +// �f�X�g���N�^ +ECTrainerGUI::~ECTrainerGUI() { + SAFE_RELEASE(pMediaEvent); + SAFE_RELEASE(_pMediaCont); + SAFE_RELEASE(_pCGB2); + SAFE_RELEASE(_pSource); + SAFE_RELEASE(_pVMR9); + SAFE_RELEASE(_pGB); + SAFE_RELEASE(_pD3DDev); + SAFE_RELEASE(_pD3D); + CoUninitialize(); } // ������ -bool ECTrainerGUI::Init() { +bool ECTrainerGUI::Init(HINSTANCE hInstance) { // ���s�‹��̏��擾 GetDisplayInfo(); // GUI�E�C���h�E @@ -34,23 +74,62 @@ else subMonitor = i; } // �}���`���j�^�̏ꍇ �T�u��ʂŃt���X�N���[���� - cv::moveWindow(WIN_DISP, _Displays[subMonitor].left, _Displays[subMonitor].top); + cv::moveWindow(WIN_DISP, _Displays[mainMonitor].left, _Displays[mainMonitor].top); cv::setWindowProperty(WIN_DISP, cv::WND_PROP_FULLSCREEN, cv::WINDOW_FULLSCREEN); - _DispBuffer.create(cv::Size(_Displays[subMonitor].right - _Displays[subMonitor].left, - _Displays[subMonitor].bottom - _Displays[subMonitor].top), CV_8UC3); - cv::moveWindow(WIN_MAIN, _Displays[mainMonitor].left + 100, _Displays[mainMonitor].top + 100); + _DispBuffer.create(cv::Size(_Displays[mainMonitor].right - _Displays[mainMonitor].left, + _Displays[mainMonitor].bottom - _Displays[mainMonitor].top), CV_8UC3); + cv::moveWindow(WIN_MAIN, _Displays[subMonitor].left + 100, _Displays[mainMonitor].top + 100); + _DispRect = _Displays[mainMonitor]; } else { // �V���O�����j�^�̏ꍇ cv::moveWindow(WIN_DISP, _Displays[0].left, _Displays[0].top); _DispBuffer.create(cv::Size(640, 480), CV_8UC3); cv::moveWindow(WIN_MAIN, _Displays[0].left + 100, _Displays[0].top + 100); + _DispRect = _Displays[0]; } cv::Mat blank(cv::Size(IMAGE_WIDTH, 480), CV_8UC3, cv::Scalar(0)); blank.copyTo(_SceneBuffer); blank.copyTo(_DispImage); _Logo = cv::imread("../images/ECTrainerLogo640x91.png"); + return InitDx(hInstance); +} + +// DirectX�̏����� +bool ECTrainerGUI::InitDx(HINSTANCE hInstance) { + TCHAR gName[100] = _T("Video Window"); + WNDCLASSEX wcex = { sizeof(WNDCLASSEX), CS_HREDRAW | CS_VREDRAW, WndProc, 0, 0, hInstance, NULL, NULL, + (HBRUSH)(COLOR_WINDOW + 1), NULL, (TCHAR*)gName, NULL }; + if (!RegisterClassEx(&wcex)) return 0; + + // �t���X�N���[���p�Ƀ|�b�v�A�b�v�E�B���h�E���쐬 + if (!(_hWnd = CreateWindow(gName, gName, WS_POPUPWINDOW, _DispRect.left, _DispRect.top, + _DispRect.right - _DispRect.left, _DispRect.bottom - _DispRect.top, + NULL, NULL, hInstance, NULL))) + return false; + + // Direct3D�̏����� + if (!(_pD3D = Direct3DCreate9(D3D_SDK_VERSION))) return false; + + // �t���X�N���[���p�ɏ������p�����[�^��ݒ� + D3DPRESENT_PARAMETERS d3dpp = { (UINT)(_DispRect.right - _DispRect.left), + (UINT)(_DispRect.bottom - _DispRect.top), D3DFMT_A8R8G8B8, 0, D3DMULTISAMPLE_NONE, 0, + D3DSWAPEFFECT_DISCARD, _hWnd, FALSE, 0, D3DFMT_UNKNOWN, D3DPRESENT_RATE_DEFAULT, 0}; + + if (FAILED(_pD3D->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, _hWnd, + D3DCREATE_HARDWARE_VERTEXPROCESSING, &d3dpp, &_pD3DDev))) { + + CHECK(_pD3D->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, _hWnd, + D3DCREATE_SOFTWARE_VERTEXPROCESSING, &d3dpp, &_pD3DDev)); + printf("Software DirectX\n"); + } else { + printf("Hardware DirectX\n"); + } + + // COM�̏����� + CHECK(CoInitializeEx(NULL, COINIT_APARTMENTTHREADED)); + return true; } @@ -61,6 +140,7 @@ bool fStimulus = false; bool fContact = false; bool fKeepContact = false; + int contactLevel = 0; float targetSize = 2.0; clock_t startContact = 0; @@ -73,13 +153,13 @@ //cvui::rect(_MainFrame, SCENE_BUFFER_POS.x, SCENE_BUFFER_POS.y, // SCENE_BUFFER_SIZE.width, SCENE_BUFFER_SIZE.height, 0xFF0000); cv::Point mp = cvui::mouse(WIN_MAIN); - if (cvui::iarea(SCENE_BUFFER_POS.x, SCENE_BUFFER_POS.y, IMAGE_WIDTH, _SceneBufferHeight) == cvui::DOWN) { - _pEct->SetGazeV(cv::Point( - (int)((mp.x - SCENE_BUFFER_POS.x) / _SceneBufferScale), - (int)((mp.y - SCENE_BUFFER_POS.y) / _SceneBufferScale))); - } else { - _pEct->SetGazeV(cv::Point(-1, -1)); - } + //if (cvui::iarea(SCENE_BUFFER_POS.x, SCENE_BUFFER_POS.y, IMAGE_WIDTH, _SceneBufferHeight) == cvui::DOWN) { + // _pEct->SetGazeV(cv::Point( + // (int)((mp.x - SCENE_BUFFER_POS.x) / _SceneBufferScale), + // (int)((mp.y - SCENE_BUFFER_POS.y) / _SceneBufferScale))); + //} else { + // _pEct->SetGazeV(cv::Point(-1, -1)); + //} cv::Point2f imagePos = cv::Point2f(-1.F, -1.F); if (cvui::iarea(DISP_IMAGE_POS.x, DISP_IMAGE_POS.y, IMAGE_WIDTH, _DispImageHeight) == cvui::OVER) { imagePos.x = (float)(mp.x - DISP_IMAGE_POS.x) / IMAGE_WIDTH; @@ -87,10 +167,10 @@ } // �h���摜 - if (_pEct->GetGazeI().x >= 0) { - cv::circle(_DispImage, - cv::Point((int)(_DispImage.cols * _pEct->GetGazeI().x), (int)(_DispImage.rows * _pEct->GetGazeI().y)), - (int)(_DispImage.rows * 0.03), CV_RGB(0, 0, 255), 2); + if (_pEct->GetGazeV().x >= 0) { + cv::circle(_SceneBuffer, + cv::Point((int)(_pEct->GetGazeV().x * _SceneBufferScale), + (int)(_pEct->GetGazeV().y * _SceneBufferScale)), 10, CV_RGB(0, 0, 255), 2); } fContact = false; if (fStimulus) { @@ -123,24 +203,33 @@ fContact = true; } } + if (_pEct->GetGazeV().x > 200 && _pEct->GetGazeV().x < 1600 + && _pEct->GetGazeV().y >100 && _pEct->GetGazeV().y < 900 + && _PlayIndex % 2 == 1) { + fContact = true; + } } // �R���^�N�g���� if (fContact) { - if (!fKeepContact) { - fKeepContact = true; + if (contactLevel == 0) { + contactLevel = 1; startContact = clock(); - PlaySound(_T("../voices/voice2.wav"), NULL, SND_FILENAME | SND_ASYNC); + PlaySound(_T("../voices/Good_S.wav"), NULL, SND_FILENAME | SND_ASYNC); } else { - if ((clock() - startContact) / CLOCKS_PER_SEC >= 3) { - PlaySound(_T("../voices/voice1.wav"), NULL, SND_FILENAME | SND_SYNC); - fKeepContact = false; - targetSize -= 0.2F; - _pEct->Next(); + if ((clock() - startContact) / CLOCKS_PER_SEC >= (3 * contactLevel)) { + if (contactLevel == 1) PlaySound(_T("../voices/Nice_S.wav"), NULL, SND_FILENAME | SND_ASYNC); + if (contactLevel == 2) PlaySound(_T("../voices/Great_S.wav"), NULL, SND_FILENAME | SND_ASYNC); + if (contactLevel == 3) PlaySound(_T("../voices/Excellent_S.wav"), NULL, SND_FILENAME | SND_ASYNC); + + ++contactLevel; + if (contactLevel > 4) contactLevel = 0; + //targetSize -= 0.2F; + //_pEct->Next(); } } } else { - fKeepContact = false; + contactLevel = 0; } // UI�`�� @@ -148,12 +237,16 @@ cvui::beginColumn(_MainFrame, 10, 100, 140, -1, 10); if (fStimulus) { if (cvui::button(140, 30, "STOP")) { + if (!StopMovie()) _pEct->Stop(); fKeepContact = false; fStimulus = false; } if (cvui::button(140, 30, "NEXT")) { fKeepContact = false; - _pEct->Next(); + if (!StopMovie()) _pEct->Stop(); + _PlayIndex = (_PlayIndex + 1) % _MovieFiles.size(); + if (!PlayMovie()) _pEct->Stop(); + //_pEct->Next(); } cvui::trackbar(140, &targetSize, (float)0, (float)3.0); cvui::checkbox("Show Eyes", &fShowEyesPos); @@ -163,7 +256,8 @@ if (cvui::button(140, 30, "START")) { fStimulus = true; fKeepContact = false; - _pEct->Start(); + if (!PlayMovie()) _pEct->Stop(); + //_pEct->Start(); } } cvui::printf("Mouse W %d, %d", mp.x, mp.y); @@ -179,14 +273,119 @@ cvui::text(_MainFrame, DISP_IMAGE_POS.x, 100, "STIMULUS IMAGE"); cvui::image(_MainFrame, DISP_IMAGE_POS.x, DISP_IMAGE_POS.y, _DispImage); + if (pMediaEvent) { + long eventCode; + pMediaEvent->WaitForCompletion(1, &eventCode);//����̏�Ԃ�⑫ + if (eventCode != 0) { + if (!StopMovie()) _pEct->Stop(); + _PlayIndex = (_PlayIndex + 1) % _MovieFiles.size(); + if (!PlayMovie()) _pEct->Stop(); + } + } + // ��ʕ\�� cvui::update(); cv::imshow(WIN_MAIN, _MainFrame); cv::imshow(WIN_DISP, _DispBuffer); if (snapshot) cv::imwrite("d:/usr/dl/snapshot.jpg", _MainFrame); int key = cv::waitKey(1); + switch (key) { + case '1': + PlaySound(_T("../voices/Good_S.wav"), NULL, SND_FILENAME | SND_ASYNC); + break; + case '2': + PlaySound(_T("../voices/Nice_S.wav"), NULL, SND_FILENAME | SND_ASYNC); + break; + case '3': + PlaySound(_T("../voices/Great_S.wav"), NULL, SND_FILENAME | SND_ASYNC); + break; + case '4': + PlaySound(_T("../voices/Excellent_S.wav"), NULL, SND_FILENAME | SND_ASYNC); + break; + case '5': + PlaySound(_T("../voices/Good_L.wav"), NULL, SND_FILENAME | SND_ASYNC); + break; + case '6': + PlaySound(_T("../voices/Nice_L.wav"), NULL, SND_FILENAME | SND_ASYNC); + break; + case '7': + PlaySound(_T("../voices/Great_L.wav"), NULL, SND_FILENAME | SND_ASYNC); + break; + case '8': + PlaySound(_T("../voices/Excellent_L.wav"), NULL, SND_FILENAME | SND_ASYNC); + break; + } if (key == KEY_ESC) _pEct->Stop(); } + _pEct->Stop(); + return true; +} + +// ����Đ� +bool ECTrainerGUI::PlayMovie() { + if (_PlayIndex < 0 || _PlayIndex >= _MovieFiles.size()) return false; + + // �t�B���^�O���t�}�l�[�W���̍쐬 + CHECK(CoCreateInstance(CLSID_FilterGraph, NULL, CLSCTX_INPROC_SERVER, IID_IGraphBuilder, (void**)&_pGB)); + + // VRM9�t�B���^�̍쐬�Ɠo�^ + CHECK(CoCreateInstance(CLSID_VideoMixingRenderer9, NULL, CLSCTX_INPROC_SERVER, IID_IBaseFilter, (void**)&_pVMR9)); + CHECK(_pGB->AddFilter(_pVMR9, L"VMR9")); // �t�B���^�O���t�ɓo�^ + + // VRM9���E�B���h�E���X���[�h�ɂ��� + IVMRFilterConfig* pVMRCfg = NULL; + CHECK(_pVMR9->QueryInterface(IID_IVMRFilterConfig9, (void**)&pVMRCfg)); + CHECK(pVMRCfg->SetRenderingMode(VMRMode_Windowless)); + pVMRCfg->Release(); // IVMRFilterConfig�͂����K�v�Ȃ� + + // �`��E�B���h�E�̎w�� + IVMRWindowlessControl9* pVMRWndCont = NULL; + CHECK(_pVMR9->QueryInterface(IID_IVMRWindowlessControl9, (void**)&pVMRWndCont)); + CHECK(pVMRWndCont->SetVideoClippingWindow(_hWnd)); + + // �\�[�X�t�B���^�̐����Ɠo�^ + CHECK(_pGB->AddSourceFilter(_MovieFiles[_PlayIndex].c_str(), _MovieFiles[_PlayIndex].c_str(), &_pSource)); + + // CaptureGraphBuilder2�C���^�[�t�F�C�X�̎擾 + CHECK(CoCreateInstance(CLSID_CaptureGraphBuilder2, NULL, CLSCTX_INPROC_SERVER, IID_ICaptureGraphBuilder2, (void**)&_pCGB2)); + CHECK(_pCGB2->SetFiltergraph(_pGB)); + + // �t�B���^�̐ڑ� + CHECK(_pCGB2->RenderStream(0, 0, _pSource, 0, _pVMR9)); + CHECK(_pCGB2->RenderStream(0, &MEDIATYPE_Audio, _pSource, 0, 0)); + + // �`��̈�̐ݒ�i�ڑ���łȂ��ƃG���[�ɂȂ�j + LONG W, H; + CHECK(pVMRWndCont->GetNativeVideoSize(&W, &H, NULL, NULL)); + RECT SrcR, DestR; + SetRect(&SrcR, 0, 0, W, H); + GetClientRect(_hWnd, &DestR); + CHECK(pVMRWndCont->SetVideoPosition(&SrcR, &DestR)); + pVMRWndCont->Release(); // �E�B���h�E���X�R���g���[���͂����K�v�Ȃ� + + // ���f�B�A�R���g���[���C���^�[�t�F�C�X�̎擾 + CHECK(_pGB->QueryInterface(IID_IMediaControl, (void**)&_pMediaCont)); + + // ���f�B�A�C�x���g�C���^�[�t�F�C�X�̎擾 + CHECK(_pGB->QueryInterface(IID_IMediaEvent, (void**)&pMediaEvent)); + + ShowWindow(_hWnd, SW_SHOWNORMAL); + _pMediaCont->Run(); + + return true; +} + +// �����~ +bool ECTrainerGUI::StopMovie() { + _pMediaCont->Stop(); + SAFE_RELEASE(pMediaEvent); + SAFE_RELEASE(_pMediaCont); + SAFE_RELEASE(_pCGB2); + SAFE_RELEASE(_pSource); + SAFE_RELEASE(_pVMR9); + SAFE_RELEASE(_pGB); + //pMediaPosition->Release(); + return true; } @@ -240,3 +439,12 @@ displays->push_back(*lpMon); return TRUE; } + +// �E�B���h�E�v���V�[�W�� +LRESULT CALLBACK WndProc(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam) { + if (msg == WM_DESTROY) { PostQuitMessage(0); return 0; } + if (msg == WM_CHAR) { + printf("key=%d\n", (int)wParam); + } + return DefWindowProc(hWnd, msg, wParam, lParam); +} diff --git a/ECTrainer2/ECTrainerGUI.h b/ECTrainer2/ECTrainerGUI.h index 1e3add1..48e0b28 100644 --- a/ECTrainer2/ECTrainerGUI.h +++ b/ECTrainer2/ECTrainerGUI.h @@ -5,6 +5,13 @@ #include "myOpenCV.h" #include #include "BaseProcess.h" +#include +#include +#include + +LRESULT CALLBACK WndProc(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam); +#define SAFE_RELEASE(x) { if (x) x->Release(); x = NULL; } +#define CHECK(x) { if(((HRESULT)(x)) < 0) return false; } class ECTrainerGUI : public BaseProcess { @@ -17,6 +24,7 @@ const cv::Point DISP_IMAGE_POS = cv::Point(820, 120); const int KEY_ESC = 27; + std::vector _MovieFiles; std::vector _Displays; cv::Mat _MainFrame; // ���C���E�C���h�E�t���[�� cv::Mat _DispImage; // �h���摜 @@ -26,13 +34,29 @@ int _SceneBufferHeight; float _SceneBufferScale; int _DispImageHeight; + int _PlayIndex; + RECT _DispRect; + HWND _hWnd; + LPDIRECT3D9 _pD3D; + LPDIRECT3DDEVICE9 _pD3DDev; + IGraphBuilder* _pGB; + IBaseFilter* _pVMR9; + IBaseFilter* _pSource; + ICaptureGraphBuilder2* _pCGB2; + IMediaControl* _pMediaCont; + IMediaEvent* pMediaEvent; + + bool InitDx(HINSTANCE hInstance); + bool PlayMovie(); + bool StopMovie(); void GetDisplayInfo(); static BOOL CALLBACK MonitorEnumProc(HMONITOR hMon, HDC hdcMon, LPRECT lpMon, LPARAM dwDate); public: ECTrainerGUI(ECTrainer* pEct); - bool Init(); + ~ECTrainerGUI(); + bool Init(HINSTANCE hInstance); bool MainLoop(); void SetSceneBuffer(cv::Mat &img); void SetDispBuffer(cv::Mat& img); diff --git a/ECTrainer2/EyeTrack.cpp b/ECTrainer2/EyeTrack.cpp index 749b213..367a60b 100644 --- a/ECTrainer2/EyeTrack.cpp +++ b/ECTrainer2/EyeTrack.cpp @@ -53,8 +53,8 @@ cv::Size sceneSize = _pEct->GetSceneSize(); std::cout << "sceneSize: " << sceneSize.width << "," << sceneSize.height << std::endl; RingBuffer gpCx(DATA_MEAN_SIZE), gpCy(DATA_MEAN_SIZE); - std::ofstream ofs; - ofs.open("log.txt"); + //std::ofstream ofs; + //ofs.open("datalog.txt"); while (!_KeepAliveSignal) Sleep(1); // �ŏ���KeepAlive���M�҂� @@ -72,7 +72,7 @@ // ��M�f�[�^��� GazeData gd(buf); - ofs << buf; + //ofs << buf; if (gd.gidx > lastGidx) { //printf("gp %f,%f ", gp.x, gp.y); if (gp.x > 0 && gp.y > 0) { @@ -102,7 +102,7 @@ //} //Sleep(1); } - if (ofs) ofs.close(); + //if (ofs) ofs.close(); return true; } diff --git a/ECTrainer2/main.cpp b/ECTrainer2/main.cpp index 1149ff9..7b5f796 100644 --- a/ECTrainer2/main.cpp +++ b/ECTrainer2/main.cpp @@ -8,12 +8,12 @@ #endif #ifdef _DEBUG -int main() +int main() { + HINSTANCE hInstance = ::GetModuleHandle(NULL); #else -int APIENTRY _tWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPTSTR lpCmdLine, int nCmdShow) +int APIENTRY _tWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPTSTR lpCmdLine, int nCmdShow) { #endif -{ - ECTrainer ect; + ECTrainer ect(hInstance); if (!ect.Process()) return 1; #ifdef _DEBUG