1 /**
2 * JmolMultiTouchDriver.cpp Version 0.3
3 * Author: Bob Hanson, hansonr@stolaf.edu
4 * Date: 12/10/2009
5 *
6 * based on HPTouchSmartDriver.cpp
7 * by Andrew Koehring and Jay Roltgen (July 24th, 2009)
8 *
9 * This file uses the NextWindow MultiTouch API to receive touch event
10 * info from the HP TouchSmart computer and send via a socket connection
11 * to the Sparsh-UI Gesture Server.
12 *
13 * For use in association with jmol.sourceforge.net using -Msparshui startup option
14 * (see http://jmol.svn.sourceforge.net/viewvc/jmol/trunk/Jmol/src/com/sparshui)
15 *
16 * NOTE: This driver will not work with the standard Sparsh-UI gesture server, which
17 * expects only a 17-byte message from the device. Here we send 29, which includes
18 * 4 bytes for the int message data length and 8 bytes for the ULONLONG event time.
19 *
20 * Modifications:
21 *
22 * Version 0.2
23 *
24 * -- consolidates touchInfos, lastTimes, and touchAlive as "TouchPoint" structure
25 * -- uses SystemTimeToFileTime to generate a "true" event time
26 * -- delivers event time as a 64-bit unsigned integer (ULONGLONG)
27 * -- ignores the NextWindow repetitions at a given location with slop (+/-1 pixel)
28 * -- delivers true moves as fast as they can come (about every 20 ms)
29 * -- times out only for "full death" events (all fingers lifted) after 75 ms
30 * -- automatically bails out if started with a server and later loses service
31 * -- (e.g. applet page closes)
32 * -- operates with or without server for testing
33 * -- not fully inspected for memory leaks or exceptions
34 *
35 * Version 0.3
36 *
37 * -- can only deliver ID "0" or "1"
38 * -- use of bitsets to deliver deaths, then births, then moves
39 * and proper accounting for errors either due to premature deaths or,
40 * in certain circumstances, the NextWindow driver sending a move
41 * with no previous "down" message. This amounts to:
42 * -- canceling all currently active points
43 * -- creating a BIRTH for each of these points in addition to this "moved" point
44 * -- canceling the "move" operation
45 *
46 * -- comments in Jmol script format for replaying at any speed within Jmol
47 * -- requires a script file data.spt that would, for example, include:
48 *
49 function setWidthAndHeight(w, h) {
50 screenWidth = w
51 screenHeight = h
52 }
53 function pt(id, index, state, time, x, y) {
54 var c = (id == 0 ? "blue", "red")
55 y = screenHeight - y
56 draw ID @{"p" + id} [@x, @y] color @c
57 delay 0.01
58 }
59 *
60 * thus allowing for playback of the transcript of a session.
61 *
62 *
63 * -- uses port 5947 due to nonstandard SparshUI:
64 *
65 * 4 (int) -1 (1 point; negative to indicate that per-point message length follows)
66 * 4 (int) 21 (21 bytes per point event will be sent)
67 *
68 * 4 (int) ID (0 or 1)
69 * 4 (float) x (0.0-1.0)
70 * 4 (float) y (0.0-1.0)
71 * 1 (char) state: 0 (down/BIRTH), 1(up/DEATH), 2(move)
72 * 8 (long) time (ms since this driver started)
73 *
74 * all ints, floats, and longs in network-endian order
75 *
76 */
77
78 #include "stdafx.h"
79 #include "math.h"
80 #include "NWMultiTouch.h"
81 #include "time.h"
82 #include <string>
83
84
85 #define VERSION "0.2/Jmol"
86
87 // The desired time to wait for more touch move events before sending the touch death.
88 #define TOUCH_WAIT_TIME_MS 75
89 #define TOUCH_SLOPXY 1
90
91 ULONGLONG startTime;
92 ULONGLONG timeLast; // time of last report from NWMultiTouch.dll
93 ULONGLONG timeThis; // time to match with timeLast for killing purposes.
94 // Flag to stop execution
95 bool running = true;
96 bool testing = false; // command line -test flag
97 bool exitOnDisconnect = true;
98 bool raw = false; // just report raw output
99
100 // bitsets to indicate active points
101
102 ULONGLONG bsAlive = 0;
103
104 // Socket for sending information to the gesture serve
105 SOCKET sockfd;
106 bool useSocket;
107 bool haveSocket;
108 // using 5947 because this is nonstandard
109 #define PORT 5947
110
111 // The height and width of the screen in pixels.
112 int displayWidth, displayHeight;
113
114 using namespace std;
115
116
117 // only the part being sent
118 #define TP_LENGTH 21
119
120 // Holds touch point information - this format is compatible with the Sparsh-UI
121 // gesture server format. (Expanded by Bob Hanson)
122 struct TouchPoint {
123
124 // for delivery (network endian):
125
126 int _id;
127 float _x;
128 float _y;
129 ULONGLONG _time;
130 char _state;
131
132 // for local use only
133 int _index;
134 point_t _touchPos;
135 ULONGLONG _timeReceived;
136 ULONGLONG _timeSent;
137 };
138
139 // Holds the last touch point received on each particular touch ID.
140 TouchPoint touchPoints[MAX_TOUCHES];
141 int nextID = 0;
142 int nActive = 0;
143
144 // includes number of points and point length
145 #define MSG_LENGTH 29
146
147 // The type of the touch received from the device.
148 typedef enum TouchDeviceDataType {
149 POINT_BIRTH,
150 POINT_DEATH,
151 POINT_MOVE,
152 };
153
getTimeNow()154 ULONGLONG getTimeNow() {
155
156 // thank you: http://en.allexperts.com/q/C-1040/time-milliseconds-Windows.htm
157
158 SYSTEMTIME st;
159 GetSystemTime(&st);
160 FILETIME fileTime;
161 SystemTimeToFileTime(&st, &fileTime);
162 ULARGE_INTEGER uli;
163 uli.LowPart = fileTime.dwLowDateTime;
164 uli.HighPart = fileTime.dwHighDateTime;
165 ULONGLONG systemTimeIn_ms(uli.QuadPart/10000);
166 return systemTimeIn_ms;
167 }
168
169 /**
170 * Initialize the touch points
171 */
initData()172 void initData() {
173 for (int tch = 0; tch < MAX_TOUCHES; tch++) {
174 (touchPoints + tch)->_index = tch;
175 }
176 }
177
isAlive(int tch)178 bool isAlive(int tch) {
179 return ((bsAlive & (1 << tch)) != 0);
180 }
181
isAlive(TouchPoint * tpp)182 bool isAlive(TouchPoint *tpp) {
183 return isAlive(tpp->_index);
184 }
185
dumpTouchPoint(TouchPoint * tpp)186 void dumpTouchPoint(TouchPoint *tpp) {
187 cout << "pt(" << tpp->_id << "," << tpp->_index << ","
188 << (int) tpp->_state << "," << (tpp->_time - startTime) << ","
189 << (int) tpp->_touchPos.x << "," << (int) tpp->_touchPos.y << ","
190 << tpp->_x << "," << tpp->_y << ");"
191 << "// Active touchpoints [";
192 for (int tch = 0; tch < MAX_TOUCHES; tch++)
193 if (isAlive(tch))
194 cout << " " << tch;
195 cout << " ]" << endl;
196 }
197
198 /**
199 * Swaps float byte order.
200 */
swapBytes(float x)201 char *swapBytes(float x) {
202 union u {
203 float f;
204 char temp[4];
205 } un, vn;
206
207 un.f = x;
208 vn.temp[0] = un.temp[3];
209 vn.temp[1] = un.temp[2];
210 vn.temp[2] = un.temp[1];
211 vn.temp[3] = un.temp[0];
212 return vn.temp;
213 }
214
215 /**
216 * Swaps long byte order.
217 */
swapBytes(ULONGLONG x)218 char *swapBytes(ULONGLONG x) {
219 union u {
220 ULONGLONG f;
221 char temp[8];
222 } un, vn;
223 un.f = x;
224 vn.temp[0] = un.temp[7];
225 vn.temp[1] = un.temp[6];
226 vn.temp[2] = un.temp[5];
227 vn.temp[3] = un.temp[4];
228 vn.temp[4] = un.temp[3];
229 vn.temp[5] = un.temp[2];
230 vn.temp[6] = un.temp[1];
231 vn.temp[7] = un.temp[0];
232 return vn.temp;
233 }
234
235 /**
236 * Send touch point information to the gesture server
237 *
238 * @param touchpoint
239 * The touch point we want to send to the Sparsh-UI Gesture Server.
240 */
sendPoint(TouchPoint * tpp)241 bool sendPoint(TouchPoint *tpp) {
242
243 tpp->_timeSent = tpp->_timeReceived;
244 if (testing)
245 dumpTouchPoint(tpp);
246
247 if (!haveSocket)
248 return true;
249
250 char* buffer = (char*) malloc(MSG_LENGTH);
251 char* bufferptr = buffer;
252
253 // Negative of number of touch points in this packet
254 // indicates that we will be sending length of single-touch data as well.
255 int temp = htonl(-1);
256 memcpy(bufferptr, &temp, 4);
257 bufferptr += 4;
258
259 // Length of a single touchpoint data
260 temp = htonl(TP_LENGTH);
261 memcpy(bufferptr, &temp, 4);
262 bufferptr += 4;
263
264 // TouchPoint id
265 temp = htonl(tpp->_id);
266 memcpy(bufferptr, &temp, 4);
267 bufferptr += 4;
268
269 // TouchPoint x
270 memcpy(bufferptr, swapBytes(tpp->_x), 4);
271 bufferptr += 4;
272
273 // TouchPoint y
274 memcpy(bufferptr, swapBytes(tpp->_y), 4);
275 bufferptr += 4;
276
277 // TouchPoint state
278 memcpy(bufferptr, &(tpp->_state), 1);
279 bufferptr++;
280
281 // TouchPoint time
282 memcpy(bufferptr, swapBytes(tpp->_time - startTime), 8);
283
284 // Send the touchpoint.
285 int nSent = send(sockfd, buffer, MSG_LENGTH, 0);
286 free(buffer);
287 if (nSent < 0) {
288 // Jmol should automatically restart this.
289 cout << "// send failed" << endl;
290 if (exitOnDisconnect)
291 running = false;
292 else
293 haveSocket = false;
294 return true;
295 }
296 return (nSent > 0);
297 }
298
299 /**
300 * Initialize the socket connection to the gesture server.
301 */
302
initSocket()303 bool initSocket() {
304 //Set up socket information
305 WSADATA wsaData;
306 if(WSAStartup(0x101,&wsaData) != 0) {
307 cout << "// Error initializing socket library." << endl;
308 return false;
309 }
310 sockaddr_in inetAddress;
311 sockfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
312
313 if(sockfd == INVALID_SOCKET) {
314 cout << "// invalid socket" << endl;
315 return false;
316 }
317
318 // Set up the socket information
319 inetAddress.sin_family = AF_INET;
320 inetAddress.sin_port = htons((u_short) (PORT));
321
322 unsigned long addr = inet_addr("127.0.0.1");
323 hostent* host = gethostbyaddr((char*) &addr, sizeof(addr), AF_INET);
324 inetAddress.sin_addr.s_addr = *((unsigned long*)host->h_addr);
325
326 // Connect to the gesture server.
327 if(connect(sockfd, (struct sockaddr*) &inetAddress, sizeof(sockaddr)) != 0) {
328 cout << "// Connect error -- press ESC to exit or start Jmol with the -Msparshui option" << endl;
329 return false;
330 }
331
332 // Send the device type.
333 const char one = 1;
334 int nBytes = send(sockfd, &one, sizeof(char), 0);
335 if (nBytes < 1) {
336 cout << "// Connection refused" << endl;
337 return false;
338 }
339 cout << "// Connection succeeded" << endl;
340 return true;
341 }
342
343
344 /**
345 * adds the NextWindow information to the Converts the point information from the device information to the desired
346 * Sparsh-UI information.
347 */
setTouchData(NWTouchPoint * nwtpp,TouchPoint * tpp,ULONGLONG time,char state)348 void setTouchData(NWTouchPoint* nwtpp, TouchPoint* tpp, ULONGLONG time, char state) {
349 tpp->_x = nwtpp->touchPos.x / displayWidth;
350 tpp->_y = nwtpp->touchPos.y / displayHeight;
351 tpp->_time = time;
352 tpp->_state = state;
353 tpp->_touchPos.x = nwtpp->touchPos.x;
354 tpp->_touchPos.y = nwtpp->touchPos.y;
355 }
356
357 /**
358 * Process a touch birth that was received from the device.
359 */
processBirth(NWTouchPoint * nwtpp,TouchPoint * tpp,ULONGLONG time)360 void processBirth(NWTouchPoint *nwtpp, TouchPoint *tpp, ULONGLONG time) {
361 tpp->_id = nextID;
362 nextID = (nextID ? 0 : 1);
363 setTouchData(nwtpp, tpp, time, POINT_BIRTH);
364 bsAlive |= (1<<tpp->_index);
365 sendPoint(tpp);
366 }
367
368
checkMove(NWTouchPoint * nwtpp,TouchPoint * tpp)369 bool checkMove(NWTouchPoint *nwtpp, TouchPoint *tpp) {
370 return isAlive(tpp) && (tpp->_state == POINT_BIRTH
371 || abs(tpp->_touchPos.x - nwtpp->touchPos.x) > TOUCH_SLOPXY
372 || abs(tpp->_touchPos.y - nwtpp->touchPos.y) > TOUCH_SLOPXY);
373 }
374
375 /**
376 * Process a touch move that was received from the device.
377 */
processMove(NWTouchPoint * nwtpp,TouchPoint * tpp,ULONGLONG time)378 void processMove(NWTouchPoint *nwtpp, TouchPoint *tpp, ULONGLONG time) {
379 if (!checkMove(nwtpp, tpp))
380 return;
381 setTouchData(nwtpp, tpp, time, POINT_MOVE);
382 sendPoint(tpp);
383 }
384
385 /**
386 * Process a touch death that was received from the device.
387 */
processDeath(TouchPoint * tpp)388 void processDeath(TouchPoint *tpp) {
389 if (!isAlive(tpp))
390 return;
391 tpp->_state = POINT_DEATH;
392 nextID = tpp->_id;
393 bsAlive &= ~(1<<tpp->_index);
394 sendPoint(tpp);
395 }
396
clearPoints(int bs)397 void clearPoints(int bs) {
398 bs &= bsAlive;
399 for(int tch = 0; tch < MAX_TOUCHES; tch++)
400 if (bs & (1<<tch))
401 processDeath(touchPoints + tch);
402 }
403
404 NWTouchPoint nwtps[MAX_TOUCHES];
405
406 /**
407 * Receive the multi-touch information from the device
408 */
ReceiveMultiTouchData(DWORD deviceID,DWORD deviceStatus,DWORD packetID,DWORD touches,DWORD ghostTouches)409 void __stdcall ReceiveMultiTouchData(DWORD deviceID, DWORD deviceStatus,
410 DWORD packetID, DWORD touches,
411 DWORD ghostTouches) {
412
413 timeThis = getTimeNow();
414 if (deviceStatus != DS_TOUCH_INFO) {
415 if (testing)
416 cout << "// NWMultiTouch.dll reports device status " << deviceStatus << endl;
417 } else if(touches) {
418 ULONGLONG bsBirths = 0;
419 ULONGLONG bsDeaths = 0;
420 ULONGLONG bsMoves = 0;
421 int nBirths = 0;
422 int tchMax = 0;
423 int tchMin = -1;
424 for(int tch = 0; touches && tch < MAX_TOUCHES; tch++) {
425 // "touches" contains a bitmask for present touch ids.
426 // If the bit is set, then a touch with this ID exists.
427 int bit = 1 << tch;
428 if(touches & bit){
429 touches &= ~bit;
430 tchMax = tch;
431 if (tchMin == -1)
432 tchMin = tch;
433 //Get the touch information.
434 DWORD retCode = GetTouch(deviceID, packetID, nwtps + tch, bit, 0);
435 if(retCode == SUCCESS){
436 TouchPoint *tpp = touchPoints + tch;
437 tpp->_timeReceived = timeThis;
438 int state = (nwtps + tch)->touchEventType;
439 if (raw) {
440 cout << "received: " << timeThis << " id=" << tch << " " << state
441 << " " << (nwtps + tch)->touchPos.x << " " << (nwtps + tch)->touchPos.y << endl;
442 continue;
443 }
444 switch(state){
445 case TE_TOUCH_DOWN:
446 bsBirths |= bit;
447 nBirths++;
448 break;
449 case TE_TOUCHING:
450 if (isAlive(tch)) {
451 bsMoves |= bit;
452 } else {
453 // we improperly canceled this one
454 // "reboot"
455 bsDeaths = -1;
456 bsBirths = bsAlive | bsBirths | bsMoves | bit;
457 }
458 break;
459 case TE_TOUCH_UP:
460 // When there is a full release, we have a problem, because
461 // these come only upon the next touch down or move.
462 // This could be SECONDS or MINUTES later.
463 // This, I would argue, is a bug in NWMultiTouch.dll.
464 bsDeaths |= bit;
465 break;
466 }
467 }
468 }
469 }
470
471 // BH: In my first version it was possible for this driver to have three active
472 // touches being reported. Obviously that is not acceptable. This is a 2-touch screen.
473 // The problem derives from the fact that the screen can fail to report a kill for
474 // reasons unkown to me.
475
476 // now deliver the events in proper order, clearing IDs, assigning IDs, and moving
477
478 if (bsDeaths != 0)
479 clearPoints(bsDeaths);
480
481 int nAlive = 0;
482 for(int tch = 0; tch < MAX_TOUCHES; tch++)
483 if (isAlive(tch))
484 nAlive++;
485 if (nAlive + nBirths > 2) {
486 if (testing)
487 cout << "// Too many touches!" << endl;
488 clearPoints(bsAlive);
489 }
490 if (bsBirths != 0) {
491 for(int tch = tchMin; tch <= tchMax; tch++)
492 if (bsBirths & (1 << tch)) {
493 processBirth(nwtps + tch, touchPoints + tch, timeThis);
494 processMove(nwtps + tch, touchPoints + tch, timeThis);
495 }
496 }
497 if (bsMoves != 0) {
498 for(int tch = tchMin; tch <= tchMax; tch++)
499 if (bsMoves & (1 << tch)) {
500 processMove(nwtps + tch, touchPoints + tch, timeThis);
501 }
502 }
503 }
504 timeLast = timeThis;
505 }
506
507 /**
508 * Thread responsible for killing touch points promptly after no moves are
509 * received for that touch point.
510 *
511 * Modified for Jmol to only trigger on both fingers away.
512 * and onl
513 */
TouchKiller(LPVOID lpParam)514 DWORD WINAPI TouchKiller(LPVOID lpParam) {
515 Sleep(TOUCH_WAIT_TIME_MS);
516 while(running) {
517 if (timeLast != timeThis) {
518 // free-wheel if NWMultiTouch.dll is reporting
519 Sleep(5);
520 } else {
521 ULONGLONG timeNow = getTimeNow();
522 for (int tch = 0; tch < MAX_TOUCHES; tch++) {
523 if (!isAlive(tch))
524 continue;
525 TouchPoint *tpp = touchPoints + tch;
526 ULONGLONG dt = timeNow - timeLast;//tpp->_timeReceived;
527 if (timeLast != timeThis) {
528 break;
529 } else if (dt > TOUCH_WAIT_TIME_MS) {
530 // Declare dead after about 75 ms.
531 if (timeNow - tpp->_timeSent > (TOUCH_WAIT_TIME_MS<<1)) {
532 // and update time if not just within 150 ms.
533 tpp->_time = tpp->_timeReceived;
534 }
535 if (testing)
536 cout << "// Killing point " << tch << " after " << dt << " ms" << endl;
537 processDeath(tpp);
538 }
539 }
540 Sleep(TOUCH_WAIT_TIME_MS);
541 }
542 }
543 return 0;
544 }
545
main(int argc,char ** argv)546 int main(int argc, char **argv) {
547
548 startTime = getTimeNow();
549
550 cout << "// JmolMultiTouchDriver for the HP TouchSmart Computer Version " << VERSION << endl
551 << "// Adapted by Bob Hanson, hansonr@stolaf.edu " << endl
552 << "// from HPTouchSmartSparshDriver.cpp, by Andrew Koehring and Jay Roltgen, " << endl
553 << "// Ssee http://code.google.com/p/sparsh-ui/" << endl << endl
554 << "// Command line options include -test -nosocket -exitondisconnect" << endl << endl
555 << "// Jmol script follows. Simply load this data into Jmol using SCRIPT foo.txt" << endl
556 << "script data.spt" << endl;
557
558 //Get the number of connected devices.
559 DWORD numDevices = GetConnectedDeviceCount();
560 DWORD serialNum = 0;
561 DWORD deviceID = 0;
562 initData();
563 //testComm();return 0;
564 testing = false;
565 haveSocket = false;
566 useSocket = true;
567 exitOnDisconnect = false;
568 raw = false;
569 for (int i = 1; i < argc; i++) {
570 if ((string) argv[i] == "-test") {
571 testing = true;
572 } else if ((string) argv[i] == "-nosocket") {
573 useSocket = false;
574 } else if ((string) argv[i] == "-exitondisconnect") {
575 exitOnDisconnect = true;
576 } else if ((string) argv[i] == "-raw") {
577 raw = true;
578 }
579 }
580 cout << "// testing=" << testing << " useSocket=" << useSocket << " exitOnDisconnect=" << exitOnDisconnect << endl;
581
582 bool isOK = false;
583 if(numDevices > 0) {
584 // If we have at least one connected device then try to connect to it.
585 // Get the serial number of the device which uniquely identifies the device.
586 cout << "// Getting device ID..." << endl;
587 serialNum = GetConnectedDeviceID(0);
588
589 // Initialize the device, passing in the serial number that uniquely
590 // identifies it and the event handler for processing touch packets.
591 cout << "// Opening device and registering callback function..." << endl;
592 deviceID = OpenDevice(serialNum, &ReceiveMultiTouchData);
593 if(deviceID == 1)
594 isOK = true;
595 else
596 cout << "// Failed to connect to device, ID: " << serialNum << endl;
597 //SetKalmanFilterStatus(serialNum, true);
598 } else {
599 cout << "// No valid devices are connected." << endl;
600 }
601
602 if (!isOK)
603 return 0;
604
605 // Set up the kill thread
606 CreateThread(NULL, 0, TouchKiller, NULL, 0, NULL);
607
608 // Obtain the display info and the resolution.
609 NWDisplayInfo displayInfo;
610 DWORD retcode = GetConnectedDisplayInfo(0, &displayInfo);
611 displayWidth = (int) displayInfo.displayRect.right;
612 displayHeight = (int) displayInfo.displayRect.bottom;
613 cout << "setWidthHeight(" << displayWidth << "," << displayHeight << ");" << endl;
614
615 DWORD displayMode = RM_MULTITOUCH; // same as RM_SLOPESMODE ?
616 SetReportMode(deviceID, displayMode);
617
618 if (useSocket)
619 haveSocket = initSocket();
620 if (useSocket && !haveSocket && exitOnDisconnect) {
621 cout << "// No socket - exiting. Use -nosocket to avoid exiting" << endl;
622 return 1;
623 }
624
625 cout << "// Press ESC to Quit or I to re-initialize socket" << endl;
626 while(running) {
627
628 if (useSocket && !haveSocket) {
629 cout << "// No socket and no -nosocket or -testnosocket -- waiting for server -- press ESC to exit" << endl;
630 while(!haveSocket && running) {
631 if (_kbhit()) {
632 if (_getch()==0x1B)
633 running = false;
634 }
635 if (running) {
636 WSACleanup();
637 haveSocket = initSocket();
638 Sleep(1000);
639 }
640 }
641 }
642 if (_kbhit()) {
643 if (_getch()==0x1B)
644 running = false;
645 else if (_getch()==0x49) {
646 WSACleanup();
647 haveSocket = initSocket();
648 }
649 }
650 if (running)
651 Sleep(200);
652 }
653
654 cout << "// Closing connection to device..." << endl;
655
656 // Close any open devices.
657 CloseDevice(serialNum);
658
659 //Reset the device connect/disconnect event handlers.
660 SetConnectEventHandler(NULL);
661 SetDisconnectEventHandler(NULL);
662
663 return 0;
664 }
665