#include "common.h"
#include <mmsystem.h>
#pragma comment(lib,"winmm.lib")
#include <time.h>
#pragma comment(lib, "dxguid.lib")
#pragma comment(lib, "d3d9.lib")
#pragma comment(lib, "Strmiids.lib")
#define _WIN32_DCOM // CoInitializeEx関数の呼び出しに必要
#define CVUI_IMPLEMENTATION
#include "cvui.h"
#ifdef _DEBUG
#include <iostream>
#endif
// コンストラクタ
ECTrainerGUI::ECTrainerGUI(ECTrainer* pEct)
: BaseProcess(pEct)
, _pD3D(NULL)
, _pD3DDev(NULL)
, _pGB(NULL)
, _pVMR9(NULL)
, _pSource(NULL)
, _pCGB2(NULL)
, _pMediaCont(NULL)
, _pMediaEvent(NULL)
, _hWnd(NULL)
, _SceneBufferHeight(0)
, _DispImageHeight(0)
, _SceneBufferScale(1.F)
, _hWndMain(NULL)
, _AppRunning(true)
{
}
// デストラクタ
ECTrainerGUI::~ECTrainerGUI() {
SAFE_RELEASE(_pMediaPosition);
SAFE_RELEASE(_pMediaEvent);
SAFE_RELEASE(_pMediaCont);
SAFE_RELEASE(_pCGB2);
SAFE_RELEASE(_pSource);
SAFE_RELEASE(_pVMR9);
SAFE_RELEASE(_pGB);
SAFE_RELEASE(_pD3DDev);
SAFE_RELEASE(_pD3D);
CoUninitialize();
}
// 初期化
bool ECTrainerGUI::Init() {
// 実行環境の情報取得
std::vector<RECT> _Displays = mwut::GetDisplayInfo();
// GUIウインドウ
cvui::init(WIN_MAIN);
_MainFrame.create(MAIN_FRAME_SIZE, CV_8UC3);
// 刺激提示ウインドウ
if (_Displays.size() > 1) {
// マルチモニタの場合
// サブ画面に操作パネル表示
cv::moveWindow(WIN_MAIN, _Displays[1].left, _Displays[1].top);
_hWndMain = ::FindWindowA(NULL, WIN_MAIN.c_str());
// メイン画面に被験者映像提示(DirectShow動画がサブ画面に出力できないため)
cv::namedWindow(WIN_DISP, cv::WINDOW_NORMAL | cv::WINDOW_OPENGL);
cv::moveWindow(WIN_DISP, _Displays[0].left, _Displays[0].top);
cv::resizeWindow(WIN_DISP, cv::Size(640, 400));
cv::setWindowProperty(WIN_DISP, cv::WND_PROP_FULLSCREEN, cv::WINDOW_FULLSCREEN);
_DispBuffer.create(cv::Size(_Displays[0].right - _Displays[0].left,
_Displays[0].bottom - _Displays[0].top), CV_8UC3);
_DispRect = _Displays[0];
}
else {
// シングルモニタの場合
cv::namedWindow(WIN_DISP, cv::WINDOW_AUTOSIZE);
cv::moveWindow(WIN_DISP, _Displays[0].left, _Displays[0].top);
_DispBuffer.create(cv::Size(640, 480), CV_8UC3);
cv::moveWindow(WIN_MAIN, _Displays[0].left + 100, _Displays[0].top + 100);
_DispRect = _Displays[0];
}
_blank = cv::Mat(DISP_SIZE, CV_8UC3, cv::Scalar(0));
//blank.copyTo(_SceneBuffer);
_blank.copyTo(_DispImage);
_Logo = cv::imread("../images/ECTLogo2_1480l.png");
return InitDx(_pEct->GetInstance());
}
// DirectXの初期化
bool ECTrainerGUI::InitDx(HINSTANCE hInstance) {
TCHAR gName[100] = _T("Video Window");
WNDCLASSEX wcex = { sizeof(WNDCLASSEX), CS_HREDRAW | CS_VREDRAW, WndProc, 0, 0, hInstance, NULL, NULL,
(HBRUSH)(COLOR_WINDOW + 1), NULL, (TCHAR*)gName, NULL };
if (!RegisterClassEx(&wcex)) return 0;
// フルスクリーン用にポップアップウィンドウを作成
if (!(_hWnd = CreateWindow(gName, gName, WS_POPUPWINDOW, _DispRect.left, _DispRect.top,
_DispRect.right - _DispRect.left, _DispRect.bottom - _DispRect.top,
NULL, NULL, hInstance, NULL)))
return false;
// Direct3Dの初期化
if (!(_pD3D = Direct3DCreate9(D3D_SDK_VERSION))) return false;
// フルスクリーン用に初期化パラメータを設定
D3DPRESENT_PARAMETERS d3dpp = { (UINT)(_DispRect.right - _DispRect.left),
(UINT)(_DispRect.bottom - _DispRect.top), D3DFMT_A8R8G8B8, 0, D3DMULTISAMPLE_NONE, 0,
D3DSWAPEFFECT_DISCARD, _hWnd, FALSE, 0, D3DFMT_UNKNOWN, D3DPRESENT_RATE_DEFAULT, 0};
if (FAILED(_pD3D->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, _hWnd,
D3DCREATE_HARDWARE_VERTEXPROCESSING, &d3dpp, &_pD3DDev))) {
CHECK(_pD3D->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, _hWnd,
D3DCREATE_SOFTWARE_VERTEXPROCESSING, &d3dpp, &_pD3DDev));
printf("Software DirectX\n");
} else {
printf("Hardware DirectX\n");
}
// COMの初期化
CHECK(CoInitializeEx(NULL, COINIT_APARTMENTTHREADED));
return true;
}
// 非メッセージ時の処理
bool ECTrainerGUI::Routine() {
// メインウインドウフレームクリア
_MainFrame = cv::Scalar(49, 52, 49);
//cv::Point2f imagePos = cv::Point2f(-1.F, -1.F);
//if (cvui::iarea(DISP_IMAGE_POS.x, DISP_IMAGE_POS.y, IMAGE_WIDTH, _DispImageHeight) == cvui::OVER) {
// imagePos.x = (float)(mp.x - DISP_IMAGE_POS.x) / IMAGE_WIDTH;
// imagePos.y = (float)(mp.y - DISP_IMAGE_POS.y) / _DispImageHeight;
//}
// 表示バッファ生成
cv::Mat sceneBuf = _pEct->PImageProc()->GetImage();
if (sceneBuf.empty()) sceneBuf = _blank;
cv::Mat CurDisplay = _DispImage.clone();
// 注視点の表示
if (_pEct->PEyeTrack()->GetGazeV().x >= 0) {
cv::circle(sceneBuf, _pEct->PEyeTrack()->GetGazeV(), 10, CV_RGB(0, 0, 255), 2);
}
// UI描画
cvui::image(_MainFrame, 0, 0, _Logo);
cvui::beginColumn(_MainFrame, 10, 100, 140, -1, 10);
switch (_pEct->PWorker()->GetAppStatus()) {
case APP_STATUS::IDLE:
if (cvui::button(140, 30, "CALIBRATION")) {
_pEct->PWorker()->Tell(ECTMSG::CALIB_START);
}
if (cvui::button(140, 30, "START")) {
//fStimulus = true;
//fKeepContact = false;
//if (!PlayMovie()) _pEct->Stop();
((BaseProcess*)_pEct->PWorker())->Tell(ECTMSG::STIM_START);
}
break;
case APP_STATUS::STIM:
if (cvui::button(140, 30, "STOP")) {
if (!StopMovie()) _pEct->StopApp();
//fKeepContact = false;
((BaseProcess*)_pEct->PWorker())->Tell(ECTMSG::STIM_STOP);
//_pEct->PWorker()->StopStim();
//fStimulus = false;
}
//cvui::trackbar(140, &targetSize, (float)0, (float)3.0);
//cvui::checkbox("Show Eyes", &fShowEyesPos);
//cvui::text(fContact ? "Eyes Contact!" : "No contact");
break;
}
//cvui::printf("Mouse W %d, %d", mp.x, mp.y);
//cvui::printf("Mouse L %.2f, %.2f", imagePos.x, imagePos.y);
cvui::printf("View Gaze %.0f, %.0f", _pEct->PEyeTrack()->GetGazeV().x, _pEct->PEyeTrack()->GetGazeV().y);
//cvui::printf("Img Gaze %.2f, %.2f", _pEct->GetGazeI().x, _pEct->GetGazeI().y);
cvui::text(_pEct->PMarker()->IsDetected() ? "AR Markers OK" : "AR Markers NG");
cvui::printf("Battery %d %%", _pEct->PTobiiREST()->GetBatteryLevel());
if (IsPlaying()) {
REFTIME tm;
_pMediaPosition->get_CurrentPosition(&tm);
cvui::printf("Movie pos: %.1f", tm);
}
bool snapshot = cvui::button(140, 30, "SNAPSHOT");
if (cvui::button(140, 30, "QUIT")) _pEct->StopApp();
if (_pEct->PBitalMonitor()->IsUseDevice()) {
cvui::printf("HeartBeat (bpm)");
cvui::text(cv::format("%3d", _pEct->PBitalMonitor()->GetHB()), 2.0);
} else {
cvui::printf("HeartBeat no device");
}
cvui::endColumn();
cvui::text(_MainFrame, SCENE_BUFFER_POS.x, 100, "SCENE CAMERA");
//cv::Mat sceneDisp;
//cv::resize(sceneBuf, sceneDisp, DISP_SIZE);
cv::Mat resized = KeepAspectResize(sceneBuf, DISP_SIZE.width);
cvui::image(_MainFrame, SCENE_BUFFER_POS.x, SCENE_BUFFER_POS.y, resized);
cvui::text(_MainFrame, DISP_IMAGE_POS.x, 100, "STIMULUS IMAGE");
cv::Mat disp;
cv::resize(CurDisplay, disp, cv::Size(IMAGE_WIDTH, _DispImageHeight));
cvui::image(_MainFrame, DISP_IMAGE_POS.x, DISP_IMAGE_POS.y, disp);
// 刺激動画
std::wstring movieToShow = _pEct->GetMovieToShow();
if (movieToShow.size() > 0) {
_pEct->SetMovieToShow(L"");
if (movieToShow == L"STOP") {
if (!StopMovie()) _pEct->StopApp();
} else {
if (!PlayMovie(movieToShow)) {
mwut::DebugPrintf(_T("Can't play movie: %s\n"), movieToShow.c_str());
_pEct->StopApp();
}
}
}
// 再生終了のチェック
if (_pMediaEvent) {
long eventCode;
_pMediaEvent->WaitForCompletion(1, &eventCode);//動画の状態を補足
if (eventCode != 0) {
if (!StopMovie()) _pEct->StopApp();
//_PlayIndex = (_PlayIndex + 1) % _MovieFiles.size();
//if (!PlayMovie()) _pEct->Stop();
}
}
// 画面表示
cvui::update();
cv::imshow(WIN_MAIN, _MainFrame);
cv::imshow(WIN_DISP, _DispBuffer);
if (snapshot) cv::imwrite("d:/usr/dl/snapshot.jpg", _MainFrame);
int key = cv::waitKey(1);
switch (key) {
case '1':
PlaySound(_T("../voices/Good_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
break;
case '2':
PlaySound(_T("../voices/Nice_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
break;
case '3':
PlaySound(_T("../voices/Great_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
break;
case '4':
PlaySound(_T("../voices/Excellent_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
break;
case '5':
PlaySound(_T("../voices/Good_L.wav"), NULL, SND_FILENAME | SND_ASYNC);
break;
case '6':
PlaySound(_T("../voices/Nice_L.wav"), NULL, SND_FILENAME | SND_ASYNC);
break;
case '7':
PlaySound(_T("../voices/Great_L.wav"), NULL, SND_FILENAME | SND_ASYNC);
break;
case '8':
PlaySound(_T("../voices/Excellent_L.wav"), NULL, SND_FILENAME | SND_ASYNC);
break;
}
if (key == KEY_ESC) _pEct->StopApp();
return true;
}
// イベントループ(GUIはメインスレッドのためメッセージ処理しない)
bool ECTrainerGUI::MainLoop() {
while (_AppRunning && cv::getWindowProperty(WIN_MAIN, 0) >= 0) {
this->Routine();
}
_pEct->StopApp();
return true;
}
// メッセージ通知(GUIはメインスレッドのため直接処理)
bool ECTrainerGUI::Tell(ECTMSG msg, WPARAM wp, LPARAM lp) {
if (msg == ECTMSG::QUIT) _AppRunning = false;
return true;
}
// 動画再生
bool ECTrainerGUI::PlayMovie(std::wstring movie) {
//if (_PlayIndex < 0 || _PlayIndex >= _MovieFiles.size()) return false;
// フィルタグラフマネージャの作成
CHECK(CoCreateInstance(CLSID_FilterGraph, NULL, CLSCTX_INPROC_SERVER, IID_IGraphBuilder, (void**)&_pGB));
// VRM9フィルタの作成と登録
CHECK(CoCreateInstance(CLSID_VideoMixingRenderer9, NULL, CLSCTX_INPROC_SERVER, IID_IBaseFilter, (void**)&_pVMR9));
CHECK(_pGB->AddFilter(_pVMR9, L"VMR9")); // フィルタグラフに登録
// VRM9をウィンドウレスモードにする
IVMRFilterConfig* pVMRCfg = NULL;
CHECK(_pVMR9->QueryInterface(IID_IVMRFilterConfig9, (void**)&pVMRCfg));
CHECK(pVMRCfg->SetRenderingMode(VMRMode_Windowless));
pVMRCfg->Release(); // IVMRFilterConfigはもう必要ない
// 描画ウィンドウの指定
IVMRWindowlessControl9* pVMRWndCont = NULL;
CHECK(_pVMR9->QueryInterface(IID_IVMRWindowlessControl9, (void**)&pVMRWndCont));
CHECK(pVMRWndCont->SetVideoClippingWindow(_hWnd));
// ソースフィルタの生成と登録
CHECK(_pGB->AddSourceFilter(movie.c_str(), movie.c_str(), &_pSource));
// CaptureGraphBuilder2インターフェイスの取得
CHECK(CoCreateInstance(CLSID_CaptureGraphBuilder2, NULL, CLSCTX_INPROC_SERVER, IID_ICaptureGraphBuilder2, (void**)&_pCGB2));
CHECK(_pCGB2->SetFiltergraph(_pGB));
// フィルタの接続
CHECK(_pCGB2->RenderStream(0, 0, _pSource, 0, _pVMR9));
CHECK(_pCGB2->RenderStream(0, &MEDIATYPE_Audio, _pSource, 0, 0));
// 描画領域の設定(接続後でないとエラーになる)
LONG W, H;
CHECK(pVMRWndCont->GetNativeVideoSize(&W, &H, NULL, NULL));
RECT SrcR, DestR;
SetRect(&SrcR, 0, 0, W, H);
GetClientRect(_hWnd, &DestR);
CHECK(pVMRWndCont->SetVideoPosition(&SrcR, &DestR));
pVMRWndCont->Release(); // ウィンドウレスコントロールはもう必要ない
// メディアコントロールインターフェイスの取得
CHECK(_pGB->QueryInterface(IID_IMediaControl, (void**)&_pMediaCont));
// メディアイベントインターフェイスの取得
CHECK(_pGB->QueryInterface(IID_IMediaEvent, (void**)&_pMediaEvent));
// メディアポジションインターフェイスの取得
CHECK(_pGB->QueryInterface(IID_IMediaPosition, (void**)&_pMediaPosition));
ShowWindow(_hWnd, SW_SHOWNORMAL);
_pMediaCont->Run();
return true;
}
// 動画停止
bool ECTrainerGUI::StopMovie() {
if (!IsPlaying()) return false;
_pMediaCont->Stop();
SAFE_RELEASE(_pMediaPosition);
SAFE_RELEASE(_pMediaEvent);
SAFE_RELEASE(_pMediaCont);
SAFE_RELEASE(_pCGB2);
SAFE_RELEASE(_pSource);
SAFE_RELEASE(_pVMR9);
SAFE_RELEASE(_pGB);
ShowWindow(_hWnd, SW_HIDE);
return true;
}
// 再生状態の確認
bool ECTrainerGUI::IsPlaying() {
if (!_pMediaCont) return false;
FILTER_STATE fs;
_pMediaCont->GetState(100, (OAFilterState*)&fs);
return (fs == State_Running);
}
// 視野画像バッファに画像を設定
//void ECTrainerGUI::SetSceneBuffer(cv::Mat& img) {
// _SceneBufferScale = (float)IMAGE_WIDTH / img.cols;
// _SceneBufferHeight = (int)(img.rows * _SceneBufferScale);
// cv::resize(img, _SceneBuffer, cv::Size(IMAGE_WIDTH, _SceneBufferHeight));
//}
// 刺激画像バッファに画像を設定
void ECTrainerGUI::SetDispBuffer(cv::Mat& img) {
img.copyTo(_DispImage);
_DispImageHeight = img.rows * IMAGE_WIDTH / img.cols;
cv::Mat buf(_DispBuffer.size(), CV_8UC3, cv::Scalar(0));
cv::Rect roiRect;
if (img.rows * buf.cols / buf.rows > img.cols) {
roiRect.width = img.cols * buf.rows / img.rows;
roiRect.height = buf.rows;
} else {
roiRect.width = buf.cols;
roiRect.height = img.rows * buf.cols / img.cols;
}
roiRect.x = (buf.cols - roiRect.width) / 2;
roiRect.y = (buf.rows - roiRect.height) / 2;
if (roiRect.x + roiRect.width > buf.cols) roiRect.width = buf.cols - roiRect.x;
if (roiRect.y + roiRect.height > buf.rows) roiRect.height = buf.rows - roiRect.y;
cv::Mat roi(buf, roiRect);
cv::resize(img, roi, roi.size());
buf.copyTo(_DispBuffer);
}
// ウィンドウプロシージャ
LRESULT CALLBACK WndProc(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam) {
if (msg == WM_DESTROY) { PostQuitMessage(0); return 0; }
if (msg == WM_CHAR) {
printf("key=%d\n", (int)wParam);
}
return DefWindowProc(hWnd, msg, wParam, lParam);
}