Newer
Older
PrismSoftware / ECTrainer2 / ECTrainerGUI.cpp
@Toshiya Nakaguchi Toshiya Nakaguchi on 6 Jul 2020 15 KB Bital "no device" 表示

#include "common.h"
#include <mmsystem.h>
#pragma comment(lib,"winmm.lib")
#include <time.h>
#pragma comment(lib, "dxguid.lib")
#pragma comment(lib, "d3d9.lib")
#pragma comment(lib, "Strmiids.lib")
#define _WIN32_DCOM     // CoInitializeEx関数の呼び出しに必要

#define CVUI_IMPLEMENTATION
#include "cvui.h"

#ifdef _DEBUG
#include <iostream>
#endif

// コンストラクタ
ECTrainerGUI::ECTrainerGUI(ECTrainer* pEct) 
	: BaseProcess(pEct)
	, _pD3D(NULL)
	, _pD3DDev(NULL)
	, _pGB(NULL)
	, _pVMR9(NULL)
	, _pSource(NULL)
	, _pCGB2(NULL)
	, _pMediaCont(NULL)
	, _pMediaEvent(NULL)
	, _hWnd(NULL)
	, _SceneBufferHeight(0)
	, _DispImageHeight(0)
	, _SceneBufferScale(1.F)
	, _hWndMain(NULL)
{
}

// デストラクタ
ECTrainerGUI::~ECTrainerGUI() {
	SAFE_RELEASE(_pMediaPosition);
	SAFE_RELEASE(_pMediaEvent);
	SAFE_RELEASE(_pMediaCont);
	SAFE_RELEASE(_pCGB2);
	SAFE_RELEASE(_pSource);
	SAFE_RELEASE(_pVMR9);
	SAFE_RELEASE(_pGB);
	SAFE_RELEASE(_pD3DDev);
	SAFE_RELEASE(_pD3D);
	CoUninitialize();
}

// 初期化
bool ECTrainerGUI::Init(HINSTANCE hInstance) {
	// 実行環境の情報取得
	std::vector<RECT> _Displays = mwut::GetDisplayInfo();
	// GUIウインドウ
	cvui::init(WIN_MAIN);
	_MainFrame.create(MAIN_FRAME_SIZE, CV_8UC3);
	// 刺激提示ウインドウ
	if (_Displays.size() > 1) {
		// マルチモニタの場合
		// サブ画面に操作パネル表示
		cv::moveWindow(WIN_MAIN, _Displays[1].left, _Displays[1].top);
		_hWndMain = ::FindWindowA(NULL, WIN_MAIN.c_str());

		// メイン画面に被験者映像提示(DirectShow動画がサブ画面に出力できないため)
		cv::namedWindow(WIN_DISP, cv::WINDOW_NORMAL | cv::WINDOW_OPENGL);
		cv::moveWindow(WIN_DISP, _Displays[0].left, _Displays[0].top);
		cv::resizeWindow(WIN_DISP, cv::Size(640, 400));
		cv::setWindowProperty(WIN_DISP, cv::WND_PROP_FULLSCREEN, cv::WINDOW_FULLSCREEN);
		_DispBuffer.create(cv::Size(_Displays[0].right - _Displays[0].left,
			_Displays[0].bottom - _Displays[0].top), CV_8UC3);
		_DispRect = _Displays[0];
	}
	else {
		// シングルモニタの場合
		cv::namedWindow(WIN_DISP, cv::WINDOW_AUTOSIZE);
		cv::moveWindow(WIN_DISP, _Displays[0].left, _Displays[0].top);
		_DispBuffer.create(cv::Size(640, 480), CV_8UC3);
		cv::moveWindow(WIN_MAIN, _Displays[0].left + 100, _Displays[0].top + 100);
		_DispRect = _Displays[0];
	}
	_blank = cv::Mat(DISP_SIZE, CV_8UC3, cv::Scalar(0));
	//blank.copyTo(_SceneBuffer);
	_blank.copyTo(_DispImage);
	_Logo = cv::imread("../images/ECTLogo2_1480l.png");

	return InitDx(hInstance);
}

// DirectXの初期化
bool ECTrainerGUI::InitDx(HINSTANCE hInstance) {
	TCHAR gName[100] = _T("Video Window");
	WNDCLASSEX wcex = { sizeof(WNDCLASSEX), CS_HREDRAW | CS_VREDRAW, WndProc, 0, 0, hInstance, NULL, NULL,
								 (HBRUSH)(COLOR_WINDOW + 1), NULL, (TCHAR*)gName, NULL };
	if (!RegisterClassEx(&wcex)) return 0;

	// フルスクリーン用にポップアップウィンドウを作成
	if (!(_hWnd = CreateWindow(gName, gName, WS_POPUPWINDOW, _DispRect.left, _DispRect.top, 
		_DispRect.right - _DispRect.left, _DispRect.bottom - _DispRect.top,
		NULL, NULL, hInstance, NULL)))
		return false;

	// Direct3Dの初期化
	if (!(_pD3D = Direct3DCreate9(D3D_SDK_VERSION))) return false;

	// フルスクリーン用に初期化パラメータを設定
	D3DPRESENT_PARAMETERS d3dpp = { (UINT)(_DispRect.right - _DispRect.left),
		(UINT)(_DispRect.bottom - _DispRect.top), D3DFMT_A8R8G8B8, 0, D3DMULTISAMPLE_NONE, 0,
		D3DSWAPEFFECT_DISCARD, _hWnd, FALSE, 0, D3DFMT_UNKNOWN, D3DPRESENT_RATE_DEFAULT, 0};

	if (FAILED(_pD3D->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, _hWnd, 
		D3DCREATE_HARDWARE_VERTEXPROCESSING, &d3dpp, &_pD3DDev))) {

		CHECK(_pD3D->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, _hWnd,
			D3DCREATE_SOFTWARE_VERTEXPROCESSING, &d3dpp, &_pD3DDev));
		printf("Software DirectX\n");
	} else {
		printf("Hardware DirectX\n");
	}

	// COMの初期化
	CHECK(CoInitializeEx(NULL, COINIT_APARTMENTTHREADED));

	return true;
}

// イベントループ
bool ECTrainerGUI::MainLoop() {

	bool fShowEyesPos = true;
	bool fStimulus = false;
	bool fContact = false;
	bool fCaliblated = true;
	//bool fKeepContact = false;
	int contactLevel = 0;
	float targetSize = 2.0;
	clock_t startContact = 0;

	while (_pEct->IsAppRun() && cv::getWindowProperty(WIN_MAIN, 0) >= 0) {

		// メインウインドウフレームクリア
		_MainFrame = cv::Scalar(49, 52, 49);

		// マウスカーソルを注視点とする処理
		//cvui::rect(_MainFrame, SCENE_BUFFER_POS.x, SCENE_BUFFER_POS.y, 
		//	SCENE_BUFFER_SIZE.width, SCENE_BUFFER_SIZE.height, 0xFF0000);
		cv::Point mp = cvui::mouse(WIN_MAIN);
		//if (cvui::iarea(SCENE_BUFFER_POS.x, SCENE_BUFFER_POS.y, IMAGE_WIDTH, _SceneBufferHeight) == cvui::DOWN) {
		//	_pEct->SetGazeV(cv::Point(
		//		(int)((mp.x - SCENE_BUFFER_POS.x) / _SceneBufferScale), 
		//		(int)((mp.y - SCENE_BUFFER_POS.y) / _SceneBufferScale)));
		//} else {
		//	_pEct->SetGazeV(cv::Point(-1, -1));
		//}
		cv::Point2f imagePos = cv::Point2f(-1.F, -1.F);
		if (cvui::iarea(DISP_IMAGE_POS.x, DISP_IMAGE_POS.y, IMAGE_WIDTH, _DispImageHeight) == cvui::OVER) {
			imagePos.x = (float)(mp.x - DISP_IMAGE_POS.x) / IMAGE_WIDTH;
			imagePos.y = (float)(mp.y - DISP_IMAGE_POS.y) / _DispImageHeight;
		}

		// 表示バッファ生成
		cv::Mat sceneBuf = _pEct->PImageProc()->GetImage();
		if (sceneBuf.empty()) sceneBuf = _blank;
		
		cv::Mat CurDisplay = _DispImage.clone();

		// 注視点の表示
		if (_pEct->GetGazeV().x >= 0) {
			cv::circle(sceneBuf, _pEct->GetGazeV(), 10, CV_RGB(0, 0, 255), 2);
		}
		//if (_pEct->GetGazeI().x >= 0) {
		//	cv::circle(CurDisplay, cv::Point((int)_pEct->GetGazeI().x/2,
		//		(int)_pEct->GetGazeI().y/2), 10, CV_RGB(0, 0, 255), 2);
		//}

		// 刺激画像
		//fContact = false;
		//if (fStimulus) {
		//	float dx = _pEct->GetEyeR().x - _pEct->GetEyeL().x;
		//	float dy = _pEct->GetEyeR().y - _pEct->GetEyeL().y;
		//	float eyesDistance = sqrtf(dx * dx + dy * dy);
		//	float areaSize = eyesDistance * targetSize / 2.0F;
		//	if (fShowEyesPos) {
		//		cv::circle(_DispImage,
		//			cv::Point((int)(_DispImage.cols * _pEct->GetEyeR().x), (int)(_DispImage.rows * _pEct->GetEyeR().y)),
		//			(int)(_DispImage.rows * 0.03), CV_RGB(0, 128, 0), 2);
		//		cv::circle(_DispImage,
		//			cv::Point((int)(_DispImage.cols * _pEct->GetEyeL().x), (int)(_DispImage.rows * _pEct->GetEyeL().y)),
		//			(int)(_DispImage.rows * 0.03), CV_RGB(0, 128, 0), 2);
		//		cv::circle(_DispImage,
		//			cv::Point((int)(_DispImage.cols * _pEct->GetEyeR().x), (int)(_DispImage.rows * _pEct->GetEyeR().y)),
		//			(int)(_DispImage.cols * areaSize), CV_RGB(200, 200, 0), 2);
		//		cv::circle(_DispImage,
		//			cv::Point((int)(_DispImage.cols * _pEct->GetEyeL().x), (int)(_DispImage.rows * _pEct->GetEyeL().y)),
		//			(int)(_DispImage.cols * areaSize), CV_RGB(200, 200, 0), 2);
		//	}
		//	if (_pEct->GetGazeI().x >= 0) {
		//		float dxR = _pEct->GetEyeR().x - _pEct->GetGazeI().x;
		//		float dyR = _pEct->GetEyeR().y - _pEct->GetGazeI().y;
		//		float edR = sqrtf(dxR * dxR + dyR * dyR);
		//		float dxL = _pEct->GetEyeL().x - _pEct->GetGazeI().x;
		//		float dyL = _pEct->GetEyeL().y - _pEct->GetGazeI().y;
		//		float edL = sqrtf(dxL * dxL + dyL * dyL);
		//		if (edR <= areaSize || edL <= areaSize) {
		//			fContact = true;
		//		}
		//	}
		//	if (_pEct->GetGazeV().x > 200 && _pEct->GetGazeV().x < 1600
		//		&& _pEct->GetGazeV().y >100 && _pEct->GetGazeV().y < 900
		//		&& _PlayIndex % 2 == 1) {
		//		fContact = true;
		//	}
		//}

		// コンタクト判定
		if (fContact) {
			if (contactLevel == 0) {
				contactLevel = 1;
				startContact = clock();
				PlaySound(_T("../voices/Good_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
			} else {
				if ((clock() - startContact) / CLOCKS_PER_SEC >= (3 * contactLevel)) {
					if (contactLevel == 1) PlaySound(_T("../voices/Nice_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
					if (contactLevel == 2) PlaySound(_T("../voices/Great_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
					if (contactLevel == 3) PlaySound(_T("../voices/Excellent_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
					
					++contactLevel;
					if (contactLevel > 4) contactLevel = 0;
					//targetSize -= 0.2F;
					//_pEct->Next();
				}
			}
		} else {
			contactLevel = 0;
		}

		// キャリブレーション判定
		//if (_pEct->CheckCalibResult() > 0) fCaliblated = true;

		// UI描画
		cvui::image(_MainFrame, 0, 0, _Logo);
		cvui::beginColumn(_MainFrame, 10, 100, 140, -1, 10);
		switch (_pEct->PWorker()->GetAppStatus()) {
		case APP_STATUS::IDLE:
			if (cvui::button(140, 30, "CALIBRATION")) {
				_pEct->PWorker()->StartCalibration();
			}
			if (fCaliblated) {
				if (cvui::button(140, 30, "START")) {
					fStimulus = true;
					//fKeepContact = false;
					//if (!PlayMovie()) _pEct->Stop();
					_pEct->StartStim();
				}
			}
			break;
		case APP_STATUS::EXEC:
			if (cvui::button(140, 30, "STOP")) {
				if (!StopMovie()) _pEct->StopApp();
				//fKeepContact = false;
				_pEct->StopStim();
				fStimulus = false;
			}
			cvui::trackbar(140, &targetSize, (float)0, (float)3.0);
			cvui::checkbox("Show Eyes", &fShowEyesPos);
			cvui::text(fContact ? "Eyes Contact!" : "No contact");
			break;
		}
		cvui::printf("Mouse W %d, %d", mp.x, mp.y);
		cvui::printf("Mouse L %.2f, %.2f", imagePos.x, imagePos.y);
		cvui::printf("View Gaze %d, %d", _pEct->GetGazeV().x, _pEct->GetGazeV().y);
		//cvui::printf("Img Gaze %.2f, %.2f", _pEct->GetGazeI().x, _pEct->GetGazeI().y);
		cvui::text(_pEct->PMarker()->IsDetected() ? "AR Markers OK" : "AR Markers NG");
		cvui::printf("Battery %d %%", _pEct->BatteryLevel());
		if (IsPlaying()) {
			REFTIME tm;
			_pMediaPosition->get_CurrentPosition(&tm);
			cvui::printf("Movie pos: %.1f", tm);
		}
		bool snapshot = cvui::button(140, 30, "SNAPSHOT");
		if (cvui::button(140, 30, "QUIT")) _pEct->StopApp();
		if (_pEct->PBitalMonitor()->IsUseDevice()) {
			cvui::printf("HeartBeat (bpm)");
			cvui::text(cv::format("%3d", _pEct->PBitalMonitor()->GetHB()), 2.0);
		} else {
			cvui::printf("HeartBeat no device");
		}
		cvui::endColumn();
		cvui::text(_MainFrame, SCENE_BUFFER_POS.x, 100, "SCENE CAMERA");
		//cv::Mat sceneDisp;
		//cv::resize(sceneBuf, sceneDisp, DISP_SIZE);
		cv::Mat resized = KeepAspectResize(sceneBuf, DISP_SIZE.width);
		cvui::image(_MainFrame, SCENE_BUFFER_POS.x, SCENE_BUFFER_POS.y, resized);
		cvui::text(_MainFrame, DISP_IMAGE_POS.x, 100, "STIMULUS IMAGE");
		cv::Mat disp;
		cv::resize(CurDisplay, disp, cv::Size(IMAGE_WIDTH, _DispImageHeight));
		cvui::image(_MainFrame, DISP_IMAGE_POS.x, DISP_IMAGE_POS.y, disp);

		// 刺激動画
		std::wstring movieToShow = _pEct->GetMovieToShow();
		if (movieToShow.size() > 0) {
			_pEct->SetMovieToShow(L"");
			if (movieToShow == L"STOP") {
				if (!StopMovie()) _pEct->StopApp();
			} else {
				if (!PlayMovie(movieToShow)) {
					mwut::DebugPrintf(_T("Can't play movie: %s\n"), movieToShow.c_str());
					_pEct->StopApp();
				}
			}
		}

		// 再生終了のチェック
		if (_pMediaEvent) {
			long eventCode;
			_pMediaEvent->WaitForCompletion(1, &eventCode);//動画の状態を補足
			if (eventCode != 0) {
				if (!StopMovie()) _pEct->StopApp();
				//_PlayIndex = (_PlayIndex + 1) % _MovieFiles.size();
				//if (!PlayMovie()) _pEct->Stop();
			}
		}

		// 画面表示
		cvui::update();
		cv::imshow(WIN_MAIN, _MainFrame);
		cv::imshow(WIN_DISP, _DispBuffer);
		if (snapshot) cv::imwrite("d:/usr/dl/snapshot.jpg", _MainFrame);
		int key = cv::waitKey(1);
		switch (key) {
		case '1':
			PlaySound(_T("../voices/Good_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
			break;
		case '2':
			PlaySound(_T("../voices/Nice_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
			break;
		case '3':
			PlaySound(_T("../voices/Great_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
			break;
		case '4':
			PlaySound(_T("../voices/Excellent_S.wav"), NULL, SND_FILENAME | SND_ASYNC);
			break;
		case '5':
			PlaySound(_T("../voices/Good_L.wav"), NULL, SND_FILENAME | SND_ASYNC);
			break;
		case '6':
			PlaySound(_T("../voices/Nice_L.wav"), NULL, SND_FILENAME | SND_ASYNC);
			break;
		case '7':
			PlaySound(_T("../voices/Great_L.wav"), NULL, SND_FILENAME | SND_ASYNC);
			break;
		case '8':
			PlaySound(_T("../voices/Excellent_L.wav"), NULL, SND_FILENAME | SND_ASYNC);
			break;
		}
		if (key == KEY_ESC) _pEct->StopApp();
	}
	_pEct->StopApp();
	return true;
}

// 動画再生
bool ECTrainerGUI::PlayMovie(std::wstring movie) {
	//if (_PlayIndex < 0 || _PlayIndex >= _MovieFiles.size()) return false;

	// フィルタグラフマネージャの作成
	CHECK(CoCreateInstance(CLSID_FilterGraph, NULL, CLSCTX_INPROC_SERVER, IID_IGraphBuilder, (void**)&_pGB));

	// VRM9フィルタの作成と登録
	CHECK(CoCreateInstance(CLSID_VideoMixingRenderer9, NULL, CLSCTX_INPROC_SERVER, IID_IBaseFilter, (void**)&_pVMR9));
	CHECK(_pGB->AddFilter(_pVMR9, L"VMR9"));       // フィルタグラフに登録

	// VRM9をウィンドウレスモードにする
	IVMRFilterConfig* pVMRCfg = NULL;
	CHECK(_pVMR9->QueryInterface(IID_IVMRFilterConfig9, (void**)&pVMRCfg));
	CHECK(pVMRCfg->SetRenderingMode(VMRMode_Windowless));
	pVMRCfg->Release();     // IVMRFilterConfigはもう必要ない

	// 描画ウィンドウの指定
	IVMRWindowlessControl9* pVMRWndCont = NULL;
	CHECK(_pVMR9->QueryInterface(IID_IVMRWindowlessControl9, (void**)&pVMRWndCont));
	CHECK(pVMRWndCont->SetVideoClippingWindow(_hWnd));

	// ソースフィルタの生成と登録
	CHECK(_pGB->AddSourceFilter(movie.c_str(), movie.c_str(), &_pSource));

	// CaptureGraphBuilder2インターフェイスの取得
	CHECK(CoCreateInstance(CLSID_CaptureGraphBuilder2, NULL, CLSCTX_INPROC_SERVER, IID_ICaptureGraphBuilder2, (void**)&_pCGB2));
	CHECK(_pCGB2->SetFiltergraph(_pGB));

	// フィルタの接続
	CHECK(_pCGB2->RenderStream(0, 0, _pSource, 0, _pVMR9));
	CHECK(_pCGB2->RenderStream(0, &MEDIATYPE_Audio, _pSource, 0, 0));

	// 描画領域の設定(接続後でないとエラーになる)
	LONG W, H;
	CHECK(pVMRWndCont->GetNativeVideoSize(&W, &H, NULL, NULL));
	RECT SrcR, DestR;
	SetRect(&SrcR, 0, 0, W, H);
	GetClientRect(_hWnd, &DestR);
	CHECK(pVMRWndCont->SetVideoPosition(&SrcR, &DestR));
	pVMRWndCont->Release();         // ウィンドウレスコントロールはもう必要ない

	// メディアコントロールインターフェイスの取得
	CHECK(_pGB->QueryInterface(IID_IMediaControl, (void**)&_pMediaCont));

	// メディアイベントインターフェイスの取得
	CHECK(_pGB->QueryInterface(IID_IMediaEvent, (void**)&_pMediaEvent));

	// メディアポジションインターフェイスの取得
	CHECK(_pGB->QueryInterface(IID_IMediaPosition, (void**)&_pMediaPosition));

	ShowWindow(_hWnd, SW_SHOWNORMAL);
	_pMediaCont->Run();

	return true;
}

// 動画停止
bool ECTrainerGUI::StopMovie() {
	if (!IsPlaying()) return false;
	_pMediaCont->Stop();
	SAFE_RELEASE(_pMediaPosition);
	SAFE_RELEASE(_pMediaEvent);
	SAFE_RELEASE(_pMediaCont);
	SAFE_RELEASE(_pCGB2);
	SAFE_RELEASE(_pSource);
	SAFE_RELEASE(_pVMR9);
	SAFE_RELEASE(_pGB);
	ShowWindow(_hWnd, SW_HIDE);

	return true;
}

// 再生状態の確認
bool ECTrainerGUI::IsPlaying() {
	if (!_pMediaCont) return false;
	FILTER_STATE fs;
	_pMediaCont->GetState(100, (OAFilterState*)&fs);
	return (fs == State_Running);
}

// 視野画像バッファに画像を設定
//void ECTrainerGUI::SetSceneBuffer(cv::Mat& img) {
//	_SceneBufferScale = (float)IMAGE_WIDTH / img.cols;
//	_SceneBufferHeight = (int)(img.rows * _SceneBufferScale);
//	cv::resize(img, _SceneBuffer, cv::Size(IMAGE_WIDTH, _SceneBufferHeight));
//}

// 刺激画像バッファに画像を設定
void ECTrainerGUI::SetDispBuffer(cv::Mat& img) {
	img.copyTo(_DispImage);
	_DispImageHeight = img.rows * IMAGE_WIDTH / img.cols;

	cv::Mat buf(_DispBuffer.size(), CV_8UC3, cv::Scalar(0));
	cv::Rect roiRect;
	if (img.rows * buf.cols / buf.rows > img.cols) {
		roiRect.width = img.cols * buf.rows / img.rows;
		roiRect.height = buf.rows;
	} else {
		roiRect.width = buf.cols;
		roiRect.height = img.rows * buf.cols / img.cols;
	}
	roiRect.x = (buf.cols - roiRect.width) / 2;
	roiRect.y = (buf.rows - roiRect.height) / 2;
	if (roiRect.x + roiRect.width > buf.cols) roiRect.width = buf.cols - roiRect.x;
	if (roiRect.y + roiRect.height > buf.rows) roiRect.height = buf.rows - roiRect.y;
	cv::Mat roi(buf, roiRect);
	cv::resize(img, roi, roi.size());
	buf.copyTo(_DispBuffer);
}

// ウィンドウプロシージャ
LRESULT CALLBACK WndProc(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam) {
	if (msg == WM_DESTROY) { PostQuitMessage(0); return 0; }
	if (msg == WM_CHAR) {
		printf("key=%d\n", (int)wParam);
	}
	return DefWindowProc(hWnd, msg, wParam, lParam);
}