View Javadoc

1   
2   package org.ximtec.igesture;
3   
4   import java.util.ArrayList;
5   import java.util.Arrays;
6   import java.util.Calendar;
7   import java.util.HashMap;
8   import java.util.HashSet;
9   import java.util.Iterator;
10  import java.util.LinkedList;
11  import java.util.List;
12  import java.util.Map;
13  import java.util.Set;
14  //import java.util.concurrent.ConcurrentHashMap;
15  import java.util.concurrent.Executor;
16  import java.util.concurrent.ExecutorService;
17  import java.util.concurrent.Executors;
18  import java.util.logging.Level;
19  import java.util.logging.Logger;
20  
21  import org.ximtec.igesture.util.diff_match_patch;
22  import org.ximtec.igesture.util.diff_match_patch.Diff;
23  import org.ximtec.igesture.util.diff_match_patch.Operation;
24  
25  import org.ximtec.igesture.MultimodalGestureQueue.QueueElement;
26  import org.ximtec.igesture.core.Descriptor;
27  import org.ximtec.igesture.core.Gesture;
28  import org.ximtec.igesture.core.GestureClass;
29  import org.ximtec.igesture.core.GestureSet;
30  import org.ximtec.igesture.core.composite.CompositeDescriptor;
31  import org.ximtec.igesture.core.composite.Constraint;
32  import org.ximtec.igesture.event.GestureHandler;
33  import org.ximtec.igesture.io.IDeviceManager;
34  
35  /**
36   * This class recognises composite gestures.
37   * 
38   * @author Bjorn Puype, bpuype@gmail.com
39   *
40   */
41  public class MultimodalGestureRecogniser {
42  	
43  	private static final Logger LOGGER = Logger.getLogger(MultimodalGestureRecogniser.class.getName());
44  
45  	private Set<MultimodalGestureHandler> gestureHandlers = new HashSet<MultimodalGestureHandler>();
46  	
47  	/** start value for the character mapping */
48  	private static final int START_CHAR = 35; // # in ASCII table
49  	private static final int NR_THREADS = 6;
50  	private static final int PATTERNS_PER_THREAD = 10;
51  	
52  	private int patternsPerThread;
53  	private int nrThreads;
54  	
55  	/** mapping from the gesture class to a character representation */
56  	private Map<String, String> charMapping;
57  	
58  	/** mapping from a gesture class to a time window*/
59  	private Map<String, Calendar> timeWindows;
60  	
61  	/** mapping from the patterns to the composite gesture they represent */
62  	private Map<String, String> patternsMapping;
63  	/** patterns */
64  	private String[] patterns;
65  	
66  	/** composing gestures */
67  	private Set<String> gestures;
68  	
69  	/** mapping from the composite gesture classes to constraints */
70  	private Map<String, Constraint> constraintsMapping;
71  	
72  	/** queue to push composing gestures in */
73  	private MultimodalGestureQueue queue;
74  	
75  	/** status of the multi-modal recogniser*/
76  	private boolean running;
77  	
78  	/** handles threads */
79  	private ExecutorService executor;
80  	/** handles garbage collection */
81  	private ExecutorService garbagecollector;
82  
83  	private IDeviceManager manager;
84  	
85  	/**
86  	 * Constructor
87  	 * @param set	the gesture set to use.
88  	 */
89  	public MultimodalGestureRecogniser(GestureSet set, IDeviceManager manager)
90  	{
91  		//TODO fire simple gestures that were not used in a composite but are potentially part of a composite
92  		//TODO use time windows
93  		//TODO composite of composites
94  		this.manager = manager;
95  		patternsPerThread = PATTERNS_PER_THREAD;
96  		gestures = new HashSet<String>();
97  		timeWindows = new /*Concurrent*/HashMap<String,Calendar>();
98  		constraintsMapping = new /*Concurrent*/HashMap<String, Constraint>();
99  		patternsMapping = new /*Concurrent*/HashMap<String, String>();
100 		
101 		/* get composing gesture classes and time windows */
102 		List<GestureClass> classes = set.getGestureClasses();
103 		for (Iterator<GestureClass> iterator = classes.iterator(); iterator.hasNext();) {
104 			GestureClass gestureClass = iterator.next();
105 			List<Descriptor> descriptors = gestureClass.getDescriptors();
106 			for (Iterator<Descriptor> iter = descriptors.iterator(); iter.hasNext();) {
107 				Descriptor descriptor = iter.next();
108 				if(descriptor instanceof CompositeDescriptor)
109 				{
110 					// get the constraint
111 					Constraint c = ((CompositeDescriptor)descriptor).getConstraint();
112 					// map the constraint, so you know which constraint belongs with which composite gesture
113 					constraintsMapping.put(gestureClass.getName(), c);
114 					// get the composing gestures and hold them
115 					gestures.addAll(c.getGestureClasses());
116 					
117 					/* determine time windows */
118 					//get the time windows for the composing gestures
119 					Map<String, Calendar> map = c.determineTimeWindows();
120 					Set<String> keys = map.keySet();
121 					//iterate over each composing gesture and ...
122 					for (Iterator<String> itera = keys.iterator(); itera.hasNext();) {
123 						String key = itera.next();
124 						//... if it is in timeWindows ..
125 						if(timeWindows.containsKey(key))
126 						{
127 							Calendar cal = timeWindows.get(key);
128 							//.. add the largest time window as value
129 							if(cal.before(map.get(key)))
130 								timeWindows.put(key, map.get(key));
131 						}
132 						else //... if not in timeWindows, add it
133 						{
134 							timeWindows.put(key, map.get(key));
135 						}
136 					}
137 					break;
138 				}
139 			}
140 		}
141 		
142 		/* generate character representation */
143 		charMapping = new HashMap<String,String>();
144 		int ch = START_CHAR;
145 		//iterate over all composing gestures
146 		for (Iterator<String> iterator = gestures.iterator(); iterator.hasNext();) {
147 			String gs = iterator.next();
148 			//assign a character representation (unicode, starting from startChar)
149 			charMapping.put(gs, new String(new char[]{(char)ch}));
150 			ch++;
151 		}
152 		
153 		queue = new MultimodalGestureQueue(charMapping, timeWindows, this);
154 		
155 		/* generate patterns */
156 		//for each constraint (so each composite gesture)
157 		for (Iterator<String> iterator = constraintsMapping.keySet().iterator(); iterator.hasNext();) {
158 			String gesture = iterator.next();
159 			Constraint constraint = constraintsMapping.get(gesture);
160 			//generate patterns and map them to the composite gesture, so you can know which gesture is recognised
161 			Set<String> patterns = constraint.generatePatterns(charMapping); 
162 			for (Iterator<String> iter = patterns.iterator(); iter.hasNext();) {
163 				String p = iter.next();
164 				patternsMapping.put(p, gesture);				
165 			}
166 		}
167 		Set<String> s = patternsMapping.keySet();
168 		patterns = new String[s.size()];
169 		int i = 0;
170 		for (Iterator<String> iterator = s.iterator(); iterator.hasNext();i++) {
171 			patterns[i] = iterator.next();
172 		}		
173 	}
174 	
175 	/**
176 	 * Constructor
177 	 * @param set	The gesture set to use.
178 	 * @param ppt	The number of patterns per thread (default 10).
179 	 */
180 	public MultimodalGestureRecogniser(GestureSet set, IDeviceManager manager, int ppt)
181 	{
182 		this(set, manager);
183 		patternsPerThread = (ppt < 1) ? PATTERNS_PER_THREAD : ppt;
184 	}
185 	
186 	public void start()
187 	{
188 		running = true;
189 		
190 		/* start thread per ppt patterns */
191 		//determine number of threads
192 		double size = (double)patternsMapping.keySet().size();
193 		nrThreads = 1;
194 		if(size % patternsPerThread == 0)
195 			nrThreads = (int) size / patternsPerThread;
196 		else
197 			nrThreads = (int) (1 + size / patternsPerThread);
198 		
199 		LOGGER.log(Level.INFO,"Multi-modal Recogniser will use "+nrThreads+" thread(s).");
200 		
201 		executor = Executors.newFixedThreadPool(nrThreads);
202 		LOGGER.log(Level.INFO, "Execution thread pool started.");
203 		
204 		//garbage collection thread
205 		Thread t = new GarbageThread(queue);
206 		t.start();
207 		LOGGER.log(Level.INFO, "Garbage thread started.");
208 	}
209 	
210 	public void stop()
211 	{
212 		executor.shutdownNow();
213 		running = false;
214 		LOGGER.log(Level.INFO,"Execution thread pool stopped.");
215 	}
216 	
217 	/**
218 	 * @param queueElements 
219 	 * 
220 	 */
221 	public void recognise(QueueElement[] queueElements) 
222 	{
223 		String text = getStringRepresentation(queueElements);
224 		for(int i = 0; i < nrThreads; i++)
225 		{
226 			String[] patternsForTask = Arrays.copyOfRange(patterns, i*patternsPerThread, (i+1)*patternsPerThread);
227 			executor.execute(new MultimodalRunnable(patternsForTask, text, queueElements, manager));
228 		}
229 	}
230 	
231 	/**
232 	 * @param queueElements
233 	 * @return
234 	 */
235 	private String getStringRepresentation(QueueElement[] queueElements) {
236 		StringBuilder builder = new StringBuilder();
237 		for (int i = 0; i < queueElements.length; i++) {
238 			builder.append(queueElements[i].getCharacterRepresentation());
239 		}
240 		return builder.toString();
241 	}	
242 	
243 	public Set<String> getComposingGestureClasses()
244 	{
245 		return gestures;
246 	}
247 	
248 	/**
249 	 * @return the queue
250 	 */
251 	public MultimodalGestureQueue getQueue() {
252 		return queue;
253 	}
254 	
255 	public boolean isRunning()
256 	{
257 		return running;
258 	}
259 	
260 	   /**
261 	    * Fires an event and informs all registered gesture handlers.
262 	    * 
263 	    * @param resultSet the result set to be used as an argument for the fired
264 	    *            event.
265 	    */
266 	   protected void fireEvent(final String result) {
267 		   
268 		  LOGGER.log(Level.INFO,"MMR result: "+result);
269 		  
270 		  Executor executor = Executors.newFixedThreadPool(NR_THREADS);
271 		  for (final MultimodalGestureHandler gestureHandler : gestureHandlers) {
272 		     LOGGER.info("Handler: "+gestureHandler.getClass());
273 		     if (gestureHandler != null) {
274 		        executor.execute(new Runnable() {
275 		
276 		           @Override
277 		           public void run() {
278 		              gestureHandler.handle(result);
279 		           }
280 		           
281 		        });
282 		     }
283 		  }
284 	   } // fireEvent
285 	   
286 	   /**
287 	    * Adds a gesture handler to the recogniser. The gesture handler's handle()
288 	    * method will be invoked every time a new ResultSet has been created (as part
289 	    * of a recognition process).
290 	    * @param gestureHandler the gesture handler to be added.
291 	    */
292 	   public void addGestureHandler(MultimodalGestureHandler gestureHandler) {
293 	      gestureHandlers.add(gestureHandler);
294 	   } // addGestureHandler
295 	
296 	
297 	   /**
298 	    * Removes a gesture handler from the recogniser.
299 	    * @param gestureHandler the gesture handler to be removed.
300 	    */
301 	   public void removeGestureHandler(MultimodalGestureHandler gestureHandler) {
302 	      gestureHandlers.remove(gestureHandler);
303 	   } // removeGestureHandler
304 	   
305 	   
306 	   
307 
308 	class MultimodalRunnable implements Runnable
309 	{
310 		private String[] patterns;
311 		private String text;
312 		private QueueElement[] elements;
313 		private IDeviceManager manager;
314 		
315 		public MultimodalRunnable(String[] patterns, String text, QueueElement[] queueElements, IDeviceManager manager)
316 		{
317 			this.manager = manager;
318 			this.patterns = patterns;
319 			this.text = text;
320 			this.elements = queueElements;
321 			
322 			for (int i = 0; i < queueElements.length; i++) {
323 				queueElements[i].incrementWindows();
324 			}
325 		}
326 		
327 		/* (non-Javadoc)
328 		 * @see java.lang.Runnable#run()
329 		 */
330 		@Override
331 		public void run() {
332 			
333 			diff_match_patch dmp = new diff_match_patch();
334 			dmp.Match_Distance = 1000;
335 			dmp.Match_Threshold = 0.5f;
336 			
337 			//detect pattern
338 			for (int i = 0; i < patterns.length; i++) {
339 				
340 				String pattern = patterns[i];
341 				ArrayList<Integer> indexes = new ArrayList<Integer>();
342 							
343 				if(pattern == null || pattern.length() > text.length())
344 					break;
345 				
346 				//search potential match
347 				int loc = dmp.match_main(text, pattern, 0);
348 				
349 				if(loc > -1) // if potential match
350 				{
351 //					System.out.println("Found potential match at location: "+loc);
352 					
353 					//get diffences
354 					LinkedList<Diff> diffs = dmp.diff_main(text.substring(loc), pattern);
355 //					System.out.println(diffs.toString());
356 					// determine indexes of items in queue to use in contraint check
357 					// + determine if real potential match
358 					boolean skip = false;
359 					
360 					for(Diff d : diffs)
361 					{		
362 						if(d.operation == Operation.DELETE) // something extra in queue
363 						{
364 							loc += d.text.length();
365 						}
366 						else if(d.operation == Operation.EQUAL) // something similar in queue
367 						{
368 							int tmp = loc + d.text.length();
369 							for(int j = loc; j < tmp; j++)
370 								indexes.add(j);
371 							loc += d.text.length();
372 						}
373 						else if(d.operation == Operation.INSERT) // something short in queue
374 						{
375 							skip = true;
376 							break;
377 						}
378 					}
379 					
380 					if(skip == false)
381 					{
382 //						System.out.println("Indexes:" + indexes.toString());
383 						//if pattern detected, do condition check
384 						List<Gesture<?>> gestures = getGestures(indexes);
385 						String gestureClass = patternsMapping.get(pattern);
386 						Constraint c = constraintsMapping.get(gestureClass);
387 						boolean conditionsValid = c.validateConditions(gestures, manager);
388 						
389 						//fire if gesture recognised
390 						if(conditionsValid)
391 						{
392 							fireEvent(gestureClass);
393 							identifyGestures(indexes);
394 						}
395 							//TODO fire resultset instead of string
396 					}
397 //					else
398 //					{
399 //						System.out.println("no possible match, gestures short");
400 //					}
401 					indexes.clear();
402 				}
403 //				else
404 //					System.out.println("Found no potential match.");
405 			}
406 			for (int i = 0; i < elements.length; i++) {
407 				elements[i].decrementWindows();
408 			}
409 		}
410 
411 		/**
412 		 * @param indexes
413 		 * @return
414 		 */
415 		private List<Gesture<?>> getGestures(ArrayList<Integer> indexes) {
416 			List<Gesture<?>> list = new ArrayList<Gesture<?>>();
417 			for (Iterator<Integer> iterator = indexes.iterator(); iterator.hasNext();) {
418 				Integer index = iterator.next();
419 				list.add(elements[index].getGesture());			
420 			}
421 			return list;
422 		}
423 		
424 		/**
425 		 * A composite gesture has been identified and validated, mark queue elements
426 		 * garbage thread will not fire the element as a single gesture
427 		 * @param indexes
428 		 */
429 		private void identifyGestures(ArrayList<Integer> indexes)
430 		{
431 			for (Iterator<Integer> iterator = indexes.iterator(); iterator.hasNext();) {
432 				Integer index = iterator.next();
433 				elements[index].setIdentified(true);			
434 			}
435 		}
436 		
437 	}
438 	
439 	class GarbageThread extends Thread
440 	{
441 		private static final int SLEEP_TIME = 60000;
442 		private static final int MIN_QUEUE_SIZE = 5;
443 		
444 		private MultimodalGestureQueue queue;
445 		private int sleepTime;
446 		private int minQueueSize;
447 		
448 		public GarbageThread(MultimodalGestureQueue queue)
449 		{
450 			this(queue, SLEEP_TIME, MIN_QUEUE_SIZE);
451 		}
452 		
453 		public GarbageThread(MultimodalGestureQueue queue, int sleep, int queueSize)
454 		{
455 			this.queue = queue;
456 			this.sleepTime = sleep;
457 			this.minQueueSize = queueSize;
458 		}
459 		
460 		/* (non-Javadoc)
461 		 * @see java.lang.Runnable#run()
462 		 */
463 		@Override
464 		public void run() {
465 			while(true)
466 			{
467 				try
468 				{
469 					sleep(sleepTime);
470 				}
471 				catch(InterruptedException e)
472 				{}
473 				
474 				boolean next = true;
475 				if(queue.size() > minQueueSize)
476 				{	
477 					while(next)
478 					{
479 						//get element
480 						QueueElement elem = queue.peek();
481 						//if queue not empty and element is not overlapped by time windows
482 						if(elem != null && elem.getWindows() == 0)
483 						{
484 							//remove element from queue
485 							final QueueElement e = queue.poll();
486 							
487 							System.out.println("removed element "+e);
488 							
489 							//if element not part of composite
490 							if(!e.isIdentified())
491 							{
492 								LOGGER.log(Level.INFO,"MMR simple result: "+e.getGesture().getName());
493 								
494 								//notify registered GestureHandlers from source recogniser
495 								Executor executor = Executors.newFixedThreadPool(NR_THREADS);
496 								for (final GestureHandler gestureHandler : e.getResultSet().getSource().getGestureHandlers()) {
497 									
498 									if (gestureHandler != null) {
499 										executor.execute(new Runnable() {
500 								
501 											@Override
502 											public void run() {
503 												gestureHandler.handle(e.getResultSet());
504 											}
505 								           
506 										});
507 									}
508 							  	}
509 							}
510 							next = true; //continue removing from head
511 						}
512 						else // element overlapped by time windows, stop removing from head
513 							next = false;
514 					}
515 				}				
516 			}			
517 		}		
518 	}
519 }