diff --git a/.gitignore b/.gitignore index 80dd262..92b0b7c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ log/ +ECTrainer2/log.txt diff --git a/ECTrainer1/.vscode/c_cpp_properties.json b/ECTrainer1/.vscode/c_cpp_properties.json new file mode 100644 index 0000000..59c2f27 --- /dev/null +++ b/ECTrainer1/.vscode/c_cpp_properties.json @@ -0,0 +1,21 @@ +{ + "configurations": [ + { + "name": "Win32", + "includePath": [ + "${workspaceFolder}/**" + ], + "defines": [ + "_DEBUG", + "UNICODE", + "_UNICODE" + ], + "windowsSdkVersion": "10.0.17763.0", + "compilerPath": "C:/Program Files (x86)/Microsoft Visual Studio/2019/Professional/VC/Tools/MSVC/14.24.28314/bin/Hostx64/x64/cl.exe", + "cStandard": "c11", + "cppStandard": "c++17", + "intelliSenseMode": "msvc-x64" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/ECTrainer1/.vscode/settings.json b/ECTrainer1/.vscode/settings.json new file mode 100644 index 0000000..7337529 --- /dev/null +++ b/ECTrainer1/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "xstring": "cpp" + } +} \ No newline at end of file diff --git a/ECTrainer2/ECTrainer.cpp b/ECTrainer2/ECTrainer.cpp index 3a4995e..0a8a320 100644 --- a/ECTrainer2/ECTrainer.cpp +++ b/ECTrainer2/ECTrainer.cpp @@ -44,15 +44,16 @@ if (!_pImageProc->Init()) return false; if (!_pEyeTrack->Init()) return false; - DWORD dwThreadIdSceneCam, dwThreadIdStimulus, dwThreadIdImageProc, dwThreadIdEyeTrack; + DWORD dwThreadIdSceneCam, dwThreadIdStimulus, dwThreadIdImageProc, dwThreadIdEyeTrack, dwThreadIdKeepAlive; HANDLE hThreadSceneCam = CreateThread(NULL, 0, SceneCamThreadEntry, this, 0, &dwThreadIdSceneCam); HANDLE hThreadStimulus = CreateThread(NULL, 0, StimulusThreadEntry, this, 0, &dwThreadIdStimulus); HANDLE hThreadImageProc = CreateThread(NULL, 0, ImageProcThreadEntry, this, 0, &dwThreadIdImageProc); HANDLE hThreadEyeTrack = CreateThread(NULL, 0, EyeTrackThreadEntry, this, 0, &dwThreadIdEyeTrack); + HANDLE hThreadKeepAlive = CreateThread(NULL, 0, KeepAliveThreadEntry, this, 0, &dwThreadIdKeepAlive); _pGui->MainLoop(); - HANDLE handles[] = { hThreadSceneCam , hThreadStimulus, hThreadImageProc, hThreadEyeTrack }; + HANDLE handles[] = { hThreadSceneCam , hThreadStimulus, hThreadImageProc, hThreadEyeTrack, hThreadKeepAlive }; DWORD timeOut = 1000; // �^�C���A�E�g(ms) if (WaitForMultipleObjects(sizeof(handles) / sizeof(HANDLE), handles, TRUE, timeOut) != WAIT_TIMEOUT) { OutputDebugString(_T("All threads stopped sccessfully.\n")); @@ -63,6 +64,7 @@ if (hThreadStimulus) CloseHandle(hThreadStimulus); if (hThreadImageProc) CloseHandle(hThreadImageProc); if (hThreadEyeTrack) CloseHandle(hThreadEyeTrack); + if (hThreadKeepAlive) CloseHandle(hThreadKeepAlive); return true; } @@ -116,6 +118,12 @@ return 0; } +// KeepAlive�X���b�h�J�n�_ +DWORD WINAPI ECTrainer::KeepAliveThreadEntry(LPVOID lpParameter) { + ((ECTrainer*)lpParameter)->_pEyeTrack->KeepAliveLoop(); + return 0; +} + // ����摜���̒����_�ݒ� void ECTrainer::SetGazeV(cv::Point gazeV) { _pEyeTrack->SetGazeV(gazeV); diff --git a/ECTrainer2/ECTrainer.h b/ECTrainer2/ECTrainer.h index 2a298fc..5cdad2b 100644 --- a/ECTrainer2/ECTrainer.h +++ b/ECTrainer2/ECTrainer.h @@ -4,6 +4,8 @@ #include #include "myOpenCV.h" +#define ADDR "192.168.71.50" + class ECTrainerGUI; class SceneCamera; class Stimulus; @@ -20,13 +22,15 @@ Marker* _pMarker; ImageProc* _pImageProc; EyeTrack* _pEyeTrack; - bool _Running; - bool _HomographyOK; + bool _Running; // ���s���t���O + bool _HomographyOK; // �ϊ��s��̊l���L�� + cv::Size _SceneSize; // ���E�J�����̉摜�T�C�Y // �X���b�h�J�n�_ static DWORD SceneCamThreadEntry(LPVOID lpParameter); static DWORD StimulusThreadEntry(LPVOID lpParameter); static DWORD ImageProcThreadEntry(LPVOID lpParameter); static DWORD EyeTrackThreadEntry(LPVOID lpParameter); + static DWORD KeepAliveThreadEntry(LPVOID lpParameter); public: ECTrainer(); @@ -48,4 +52,6 @@ void Stop() { _Running = false; } void SetHomographyStatus(bool ok) { _HomographyOK = ok; } bool GetHomographyStatus() { return _HomographyOK; } + void SetSceneSize(cv::Size s) { _SceneSize = s; } + cv::Size GetSceneSize() { return _SceneSize; } }; diff --git a/ECTrainer2/ECTrainer2.vcxproj b/ECTrainer2/ECTrainer2.vcxproj index 52b89b6..f178ccc 100644 --- a/ECTrainer2/ECTrainer2.vcxproj +++ b/ECTrainer2/ECTrainer2.vcxproj @@ -161,6 +161,7 @@ + diff --git a/ECTrainer2/ECTrainer2.vcxproj.filters b/ECTrainer2/ECTrainer2.vcxproj.filters index 882aa8c..3bb2aaa 100644 --- a/ECTrainer2/ECTrainer2.vcxproj.filters +++ b/ECTrainer2/ECTrainer2.vcxproj.filters @@ -74,5 +74,8 @@ ヘッダー ファイル + + ヘッダー ファイル + \ No newline at end of file diff --git a/ECTrainer2/EyeTrack.cpp b/ECTrainer2/EyeTrack.cpp index 30ba5f4..749b213 100644 --- a/ECTrainer2/EyeTrack.cpp +++ b/ECTrainer2/EyeTrack.cpp @@ -1,6 +1,10 @@ +#include +#include #include "EyeTrack.h" #include "ECTrainer.h" #include "Marker.h" +#include "RingBuffer.h" +#pragma comment(lib, "ws2_32.lib") #include @@ -10,18 +14,114 @@ , _pMarker(pMarker) , _GazeV(-1, -1) , _GazeI(-1.F, -1.F) + , _KeepAliveSignal(false) { } -// ���[�v +// ������ +bool EyeTrack::Init() { + //Initialize winsock + WSADATA wsa; + + if (WSAStartup(MAKEWORD(2, 2), &wsa) != 0) + { + std::cerr << "Initialising Winsock Failed with error Code : " << WSAGetLastError() << std::endl; + return false; + } + if ((_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) == SOCKET_ERROR) + { + std::cerr << "Creating socket failed with error code : " << WSAGetLastError() << std::endl; + return false; + } + + //setup address structure + memset((char*)&_socketAddr, 0, sizeof(_socketAddr)); + _socketAddr.sin_family = AF_INET; + _socketAddr.sin_port = htons(PORT); + InetPton(_socketAddr.sin_family, _T(ADDR), &_socketAddr.sin_addr.S_un.S_addr); + + return true; +} + +// �f�[�^�X���b�h���[�v bool EyeTrack::MainLoop() { + + int slen = sizeof(_socketAddr); + char buf[BUFLEN]; + int lastGidx = 0; + cv::Point2f gp; + cv::Size sceneSize = _pEct->GetSceneSize(); + std::cout << "sceneSize: " << sceneSize.width << "," << sceneSize.height << std::endl; + RingBuffer gpCx(DATA_MEAN_SIZE), gpCy(DATA_MEAN_SIZE); + std::ofstream ofs; + ofs.open("log.txt"); + + while (!_KeepAliveSignal) Sleep(1); // �ŏ���KeepAlive���M�҂� + while (_pEct->IsRunning()) { - if (_GazeV.x >= 0 && _GazeV.y >= 0) { - _GazeI = _pMarker->ConvV2I(_GazeV); - } else { - _GazeI = cv::Point2f(-1.F, -1.F); + + // �f�[�^��M + memset(buf, '\0', BUFLEN); + if (recvfrom(_socket, buf, BUFLEN, 0, (struct sockaddr*)&_socketAddr, &slen) == SOCKET_ERROR) + { + std::cerr << "Data Receive Error" << std::endl; + Sleep(10); + continue; } - Sleep(1); + //std::cout << buf << std::endl; + + // ��M�f�[�^��� + GazeData gd(buf); + ofs << buf; + if (gd.gidx > lastGidx) { + //printf("gp %f,%f ", gp.x, gp.y); + if (gp.x > 0 && gp.y > 0) { + gpCx.Push(gp.x * sceneSize.width); + gpCy.Push(gp.y * sceneSize.height); + _GazeV = cv::Point((int)gpCx.Mean(), (int)gpCy.Mean()); + _GazeI = _pMarker->ConvV2I(_GazeV); + //printf("_GazeV %d,%d ", _GazeV.x, _GazeV.y); + //printf("_GazeI %f,%f ", _GazeI.x, _GazeI.y); + } + //printf("\n"); + + gp = cv::Point(0, 0); + lastGidx = gd.gidx; + } + if (gd.isGP && gd.s == 0) { + gp = gd.gp; + //_gp = gp; + //TRACE("%d, %f, %f\n", gd.gidx, gd.gpx, gd.gpy); + } + + // ���W�ϊ� + //if (_GazeV.x >= 0 && _GazeV.y >= 0) { + // _GazeI = _pMarker->ConvV2I(_GazeV); + //} else { + // _GazeI = cv::Point2f(-1.F, -1.F); + //} + //Sleep(1); + } + if (ofs) ofs.close(); + + return true; +} + +// KeepAlive�X���b�h���[�v +bool EyeTrack::KeepAliveLoop() { + const std::string KA_DATA_MSG = "{\"type\": \"live.data.unicast\", \"key\": \"some_GUID\", \"op\": \"start\"}"; + const int KEEP_ALIVE_WAIT_COUNT = 20; // x 100ms + char message[BUFLEN]; + strcpy_s(message, BUFLEN, KA_DATA_MSG.c_str()); + int slen = sizeof(_socketAddr); + + while (_pEct->IsRunning()) { + if (sendto(_socket, message, (int)strlen(message), 0, (struct sockaddr*)&_socketAddr, slen) == SOCKET_ERROR) { + std::cerr << "Keep Alive Data Sent Error" << std::endl; + } else { + _KeepAliveSignal = true; + } + for (int i = 0; i < KEEP_ALIVE_WAIT_COUNT && _pEct->IsRunning(); i++) Sleep(100); } return true; diff --git a/ECTrainer2/EyeTrack.h b/ECTrainer2/EyeTrack.h index 1666a39..834e085 100644 --- a/ECTrainer2/EyeTrack.h +++ b/ECTrainer2/EyeTrack.h @@ -7,13 +7,84 @@ class EyeTrack : public BaseProcess { + static const int PORT = 49152; // The port on which to listen for incoming data + static const int BUFLEN = 512; // Socket�ʐM�o�b�t�@�T�C�Y + static const int DATA_MEAN_SIZE = 20; // �����_�f�[�^�ړ����σT�C�Y + Marker* _pMarker; cv::Point _GazeV; // ����摜���̒����_���W cv::Point2f _GazeI; // �h���摜���̒����_���W�i���K�����W�n�j + SOCKET _socket; + struct sockaddr_in _socketAddr; + bool _KeepAliveSignal; + public: EyeTrack(ECTrainer* pEct, Marker* pMarker); + bool Init(); bool MainLoop(); + bool KeepAliveLoop(); void SetGazeV(cv::Point gazeV) { _GazeV = gazeV; } cv::Point GetGazeV() { return _GazeV; } cv::Point2f GetGazeI() { return _GazeI; } }; + +// �g���b�J�[�f�[�^��̓N���X +struct GazeData { + long ts = 0; + int s = 0; + int gidx = 0; + int l = 0; + cv::Point2f gp; + cv::Point3f gdr; + cv::Point3f gdl; + bool isGP = false; + bool isGDR = false; + bool isGDL = false; + + GazeData(char* str) { + char* pts = strstr(str, "\"ts\""); + if (pts) ts = atol(pts + 5); + else pts = str; + + char* ps = strstr(pts + 5, "\"s\""); + if (ps) s = atoi(ps + 4); + else ps = str; + + char* pgidx = strstr(ps + 4, "\"gidx\""); + if (pgidx) gidx = atoi(pgidx + 7); + else pgidx = str; + + char* pl = strstr(pgidx + 7, "\"l\""); + if (pl) { + l = atoi(pl + 4); + + char* pgp = strstr(pl + 4, "\"gp\""); + if (pgp) { + isGP = true; + gp.x = (float)atof(pgp + 6); + pgp = strchr(pgp + 6, ','); + if (pgp) gp.y = (float)atof(pgp + 1); + } + } + + char* pgd = strstr(pgidx, "\"gd\""); + if (pgd) { + if (strstr(pgd, "right") != NULL) { + isGDR = true; + gdr.x = (float)atof(pgd + 6); + pgd = strchr(pgd + 6, ','); + if (pgd) gdr.y = (float)atof(pgd + 1); + else pgd = pgidx; + pgd = strchr(pgd + 1, ','); + if (pgd) gdr.z = (float)atof(pgd + 1); + } else { + isGDL = true; + gdl.x = (float)atof(pgd + 6); + pgd = strchr(pgd + 6, ','); + if (pgd) gdl.y = (float)atof(pgd + 1); + pgd = strchr(pgd + 1, ','); + if (pgd) gdl.z = (float)atof(pgd + 1); + } + } + } +}; diff --git a/ECTrainer2/RingBuffer.h b/ECTrainer2/RingBuffer.h new file mode 100644 index 0000000..0201a42 --- /dev/null +++ b/ECTrainer2/RingBuffer.h @@ -0,0 +1,35 @@ +#pragma once + +#include + +class RingBuffer +{ + float *_buffer; + int _size; + int _next; + float _sum; + +public: + RingBuffer(int size) { + _size = size; + _next = 0; + _sum = 0; + _buffer = new float[_size]; + for (int i = 0; i < _size; i++) _buffer[i] = 0; + } + + ~RingBuffer() { + delete[] _buffer; + } + + void Push(float val) { + _sum -= _buffer[_next]; + _buffer[_next] = val; + _sum += val; + _next = (_next + 1) % _size; + } + + float Mean() { + return _sum / _size; + } +}; diff --git a/ECTrainer2/SceneCamera.cpp b/ECTrainer2/SceneCamera.cpp index c18e33e..899c9c5 100644 --- a/ECTrainer2/SceneCamera.cpp +++ b/ECTrainer2/SceneCamera.cpp @@ -15,12 +15,17 @@ // ������ bool SceneCamera::Init() { + // ���E�J�����ɐڑ� _SceneCam.open("rtsp://" ADDR ":8554/live/scene"); //_SceneCam.open(0); if (!_SceneCam.isOpened()) { MessageBox(NULL, _T("cannot open camera " ADDR), NULL, 0); return false; } + // �摜�T�C�Y��ۑ� + cv::Mat scene; + _SceneCam >> scene; + _pEct->SetSceneSize(scene.size()); return true; } diff --git a/ECTrainer2/SceneCamera.h b/ECTrainer2/SceneCamera.h index 37e7afc..3cab272 100644 --- a/ECTrainer2/SceneCamera.h +++ b/ECTrainer2/SceneCamera.h @@ -3,8 +3,6 @@ #include "myOpenCV.h" #include "BaseProcess.h" -#define ADDR "192.168.71.50" - class ImageProc; class SceneCamera : public BaseProcess