1
2 package org.ximtec.igesture;
3
4 import java.util.ArrayList;
5 import java.util.Arrays;
6 import java.util.Calendar;
7 import java.util.HashMap;
8 import java.util.HashSet;
9 import java.util.Iterator;
10 import java.util.LinkedList;
11 import java.util.List;
12 import java.util.Map;
13 import java.util.Set;
14
15 import java.util.concurrent.Executor;
16 import java.util.concurrent.ExecutorService;
17 import java.util.concurrent.Executors;
18 import java.util.logging.Level;
19 import java.util.logging.Logger;
20
21 import org.ximtec.igesture.util.diff_match_patch;
22 import org.ximtec.igesture.util.diff_match_patch.Diff;
23 import org.ximtec.igesture.util.diff_match_patch.Operation;
24
25 import org.ximtec.igesture.MultimodalGestureQueue.QueueElement;
26 import org.ximtec.igesture.core.Descriptor;
27 import org.ximtec.igesture.core.Gesture;
28 import org.ximtec.igesture.core.GestureClass;
29 import org.ximtec.igesture.core.GestureSet;
30 import org.ximtec.igesture.core.composite.CompositeDescriptor;
31 import org.ximtec.igesture.core.composite.Constraint;
32 import org.ximtec.igesture.event.GestureHandler;
33 import org.ximtec.igesture.io.IDeviceManager;
34
35
36
37
38
39
40
41 public class MultimodalGestureRecogniser {
42
43 private static final Logger LOGGER = Logger.getLogger(MultimodalGestureRecogniser.class.getName());
44
45 private Set<MultimodalGestureHandler> gestureHandlers = new HashSet<MultimodalGestureHandler>();
46
47
48 private static final int START_CHAR = 35;
49 private static final int NR_THREADS = 6;
50 private static final int PATTERNS_PER_THREAD = 10;
51
52 private int patternsPerThread;
53 private int nrThreads;
54
55
56 private Map<String, String> charMapping;
57
58
59 private Map<String, Calendar> timeWindows;
60
61
62 private Map<String, String> patternsMapping;
63
64 private String[] patterns;
65
66
67 private Set<String> gestures;
68
69
70 private Map<String, Constraint> constraintsMapping;
71
72
73 private MultimodalGestureQueue queue;
74
75
76 private boolean running;
77
78
79 private ExecutorService executor;
80
81 private ExecutorService garbagecollector;
82
83 private IDeviceManager manager;
84
85
86
87
88
89 public MultimodalGestureRecogniser(GestureSet set, IDeviceManager manager)
90 {
91
92
93
94 this.manager = manager;
95 patternsPerThread = PATTERNS_PER_THREAD;
96 gestures = new HashSet<String>();
97 timeWindows = new
98 constraintsMapping = new
99 patternsMapping = new
100
101
102 List<GestureClass> classes = set.getGestureClasses();
103 for (Iterator<GestureClass> iterator = classes.iterator(); iterator.hasNext();) {
104 GestureClass gestureClass = iterator.next();
105 List<Descriptor> descriptors = gestureClass.getDescriptors();
106 for (Iterator<Descriptor> iter = descriptors.iterator(); iter.hasNext();) {
107 Descriptor descriptor = iter.next();
108 if(descriptor instanceof CompositeDescriptor)
109 {
110
111 Constraint c = ((CompositeDescriptor)descriptor).getConstraint();
112
113 constraintsMapping.put(gestureClass.getName(), c);
114
115 gestures.addAll(c.getGestureClasses());
116
117
118
119 Map<String, Calendar> map = c.determineTimeWindows();
120 Set<String> keys = map.keySet();
121
122 for (Iterator<String> itera = keys.iterator(); itera.hasNext();) {
123 String key = itera.next();
124
125 if(timeWindows.containsKey(key))
126 {
127 Calendar cal = timeWindows.get(key);
128
129 if(cal.before(map.get(key)))
130 timeWindows.put(key, map.get(key));
131 }
132 else
133 {
134 timeWindows.put(key, map.get(key));
135 }
136 }
137 break;
138 }
139 }
140 }
141
142
143 charMapping = new HashMap<String,String>();
144 int ch = START_CHAR;
145
146 for (Iterator<String> iterator = gestures.iterator(); iterator.hasNext();) {
147 String gs = iterator.next();
148
149 charMapping.put(gs, new String(new char[]{(char)ch}));
150 ch++;
151 }
152
153 queue = new MultimodalGestureQueue(charMapping, timeWindows, this);
154
155
156
157 for (Iterator<String> iterator = constraintsMapping.keySet().iterator(); iterator.hasNext();) {
158 String gesture = iterator.next();
159 Constraint constraint = constraintsMapping.get(gesture);
160
161 Set<String> patterns = constraint.generatePatterns(charMapping);
162 for (Iterator<String> iter = patterns.iterator(); iter.hasNext();) {
163 String p = iter.next();
164 patternsMapping.put(p, gesture);
165 }
166 }
167 Set<String> s = patternsMapping.keySet();
168 patterns = new String[s.size()];
169 int i = 0;
170 for (Iterator<String> iterator = s.iterator(); iterator.hasNext();i++) {
171 patterns[i] = iterator.next();
172 }
173 }
174
175
176
177
178
179
180 public MultimodalGestureRecogniser(GestureSet set, IDeviceManager manager, int ppt)
181 {
182 this(set, manager);
183 patternsPerThread = (ppt < 1) ? PATTERNS_PER_THREAD : ppt;
184 }
185
186 public void start()
187 {
188 running = true;
189
190
191
192 double size = (double)patternsMapping.keySet().size();
193 nrThreads = 1;
194 if(size % patternsPerThread == 0)
195 nrThreads = (int) size / patternsPerThread;
196 else
197 nrThreads = (int) (1 + size / patternsPerThread);
198
199 LOGGER.log(Level.INFO,"Multi-modal Recogniser will use "+nrThreads+" thread(s).");
200
201 executor = Executors.newFixedThreadPool(nrThreads);
202 LOGGER.log(Level.INFO, "Execution thread pool started.");
203
204
205 Thread t = new GarbageThread(queue);
206 t.start();
207 LOGGER.log(Level.INFO, "Garbage thread started.");
208 }
209
210 public void stop()
211 {
212 executor.shutdownNow();
213 running = false;
214 LOGGER.log(Level.INFO,"Execution thread pool stopped.");
215 }
216
217
218
219
220
221 public void recognise(QueueElement[] queueElements)
222 {
223 String text = getStringRepresentation(queueElements);
224 for(int i = 0; i < nrThreads; i++)
225 {
226 String[] patternsForTask = Arrays.copyOfRange(patterns, i*patternsPerThread, (i+1)*patternsPerThread);
227 executor.execute(new MultimodalRunnable(patternsForTask, text, queueElements, manager));
228 }
229 }
230
231
232
233
234
235 private String getStringRepresentation(QueueElement[] queueElements) {
236 StringBuilder builder = new StringBuilder();
237 for (int i = 0; i < queueElements.length; i++) {
238 builder.append(queueElements[i].getCharacterRepresentation());
239 }
240 return builder.toString();
241 }
242
243 public Set<String> getComposingGestureClasses()
244 {
245 return gestures;
246 }
247
248
249
250
251 public MultimodalGestureQueue getQueue() {
252 return queue;
253 }
254
255 public boolean isRunning()
256 {
257 return running;
258 }
259
260
261
262
263
264
265
266 protected void fireEvent(final String result) {
267
268 LOGGER.log(Level.INFO,"MMR result: "+result);
269
270 Executor executor = Executors.newFixedThreadPool(NR_THREADS);
271 for (final MultimodalGestureHandler gestureHandler : gestureHandlers) {
272 LOGGER.info("Handler: "+gestureHandler.getClass());
273 if (gestureHandler != null) {
274 executor.execute(new Runnable() {
275
276 @Override
277 public void run() {
278 gestureHandler.handle(result);
279 }
280
281 });
282 }
283 }
284 }
285
286
287
288
289
290
291
292 public void addGestureHandler(MultimodalGestureHandler gestureHandler) {
293 gestureHandlers.add(gestureHandler);
294 }
295
296
297
298
299
300
301 public void removeGestureHandler(MultimodalGestureHandler gestureHandler) {
302 gestureHandlers.remove(gestureHandler);
303 }
304
305
306
307
308 class MultimodalRunnable implements Runnable
309 {
310 private String[] patterns;
311 private String text;
312 private QueueElement[] elements;
313 private IDeviceManager manager;
314
315 public MultimodalRunnable(String[] patterns, String text, QueueElement[] queueElements, IDeviceManager manager)
316 {
317 this.manager = manager;
318 this.patterns = patterns;
319 this.text = text;
320 this.elements = queueElements;
321
322 for (int i = 0; i < queueElements.length; i++) {
323 queueElements[i].incrementWindows();
324 }
325 }
326
327
328
329
330 @Override
331 public void run() {
332
333 diff_match_patch dmp = new diff_match_patch();
334 dmp.Match_Distance = 1000;
335 dmp.Match_Threshold = 0.5f;
336
337
338 for (int i = 0; i < patterns.length; i++) {
339
340 String pattern = patterns[i];
341 ArrayList<Integer> indexes = new ArrayList<Integer>();
342
343 if(pattern == null || pattern.length() > text.length())
344 break;
345
346
347 int loc = dmp.match_main(text, pattern, 0);
348
349 if(loc > -1)
350 {
351
352
353
354 LinkedList<Diff> diffs = dmp.diff_main(text.substring(loc), pattern);
355
356
357
358 boolean skip = false;
359
360 for(Diff d : diffs)
361 {
362 if(d.operation == Operation.DELETE)
363 {
364 loc += d.text.length();
365 }
366 else if(d.operation == Operation.EQUAL)
367 {
368 int tmp = loc + d.text.length();
369 for(int j = loc; j < tmp; j++)
370 indexes.add(j);
371 loc += d.text.length();
372 }
373 else if(d.operation == Operation.INSERT)
374 {
375 skip = true;
376 break;
377 }
378 }
379
380 if(skip == false)
381 {
382
383
384 List<Gesture<?>> gestures = getGestures(indexes);
385 String gestureClass = patternsMapping.get(pattern);
386 Constraint c = constraintsMapping.get(gestureClass);
387 boolean conditionsValid = c.validateConditions(gestures, manager);
388
389
390 if(conditionsValid)
391 {
392 fireEvent(gestureClass);
393 identifyGestures(indexes);
394 }
395
396 }
397
398
399
400
401 indexes.clear();
402 }
403
404
405 }
406 for (int i = 0; i < elements.length; i++) {
407 elements[i].decrementWindows();
408 }
409 }
410
411
412
413
414
415 private List<Gesture<?>> getGestures(ArrayList<Integer> indexes) {
416 List<Gesture<?>> list = new ArrayList<Gesture<?>>();
417 for (Iterator<Integer> iterator = indexes.iterator(); iterator.hasNext();) {
418 Integer index = iterator.next();
419 list.add(elements[index].getGesture());
420 }
421 return list;
422 }
423
424
425
426
427
428
429 private void identifyGestures(ArrayList<Integer> indexes)
430 {
431 for (Iterator<Integer> iterator = indexes.iterator(); iterator.hasNext();) {
432 Integer index = iterator.next();
433 elements[index].setIdentified(true);
434 }
435 }
436
437 }
438
439 class GarbageThread extends Thread
440 {
441 private static final int SLEEP_TIME = 60000;
442 private static final int MIN_QUEUE_SIZE = 5;
443
444 private MultimodalGestureQueue queue;
445 private int sleepTime;
446 private int minQueueSize;
447
448 public GarbageThread(MultimodalGestureQueue queue)
449 {
450 this(queue, SLEEP_TIME, MIN_QUEUE_SIZE);
451 }
452
453 public GarbageThread(MultimodalGestureQueue queue, int sleep, int queueSize)
454 {
455 this.queue = queue;
456 this.sleepTime = sleep;
457 this.minQueueSize = queueSize;
458 }
459
460
461
462
463 @Override
464 public void run() {
465 while(true)
466 {
467 try
468 {
469 sleep(sleepTime);
470 }
471 catch(InterruptedException e)
472 {}
473
474 boolean next = true;
475 if(queue.size() > minQueueSize)
476 {
477 while(next)
478 {
479
480 QueueElement elem = queue.peek();
481
482 if(elem != null && elem.getWindows() == 0)
483 {
484
485 final QueueElement e = queue.poll();
486
487 System.out.println("removed element "+e);
488
489
490 if(!e.isIdentified())
491 {
492 LOGGER.log(Level.INFO,"MMR simple result: "+e.getGesture().getName());
493
494
495 Executor executor = Executors.newFixedThreadPool(NR_THREADS);
496 for (final GestureHandler gestureHandler : e.getResultSet().getSource().getGestureHandlers()) {
497
498 if (gestureHandler != null) {
499 executor.execute(new Runnable() {
500
501 @Override
502 public void run() {
503 gestureHandler.handle(e.getResultSet());
504 }
505
506 });
507 }
508 }
509 }
510 next = true;
511 }
512 else
513 next = false;
514 }
515 }
516 }
517 }
518 }
519 }