1
2
3
4 package org.ximtec.igesture.tool.view.composite;
5
6 import java.awt.BorderLayout;
7 import java.awt.Color;
8 import java.awt.GridBagConstraints;
9 import java.awt.GridBagLayout;
10 import java.awt.event.ActionEvent;
11 import java.awt.event.ActionListener;
12 import java.util.ArrayList;
13 import java.util.Collection;
14 import java.util.HashMap;
15 import java.util.Iterator;
16 import java.util.List;
17 import java.util.Map;
18 import java.util.logging.Level;
19 import java.util.logging.Logger;
20
21 import javax.swing.Action;
22 import javax.swing.BorderFactory;
23 import javax.swing.DefaultListModel;
24 import javax.swing.Icon;
25 import javax.swing.JButton;
26 import javax.swing.JComboBox;
27 import javax.swing.JComponent;
28 import javax.swing.JLabel;
29 import javax.swing.JList;
30 import javax.swing.JOptionPane;
31 import javax.swing.JPanel;
32 import javax.swing.JScrollPane;
33 import javax.swing.event.ListSelectionEvent;
34 import javax.swing.event.ListSelectionListener;
35
36 import org.sigtec.graphix.GridBagLayouter;
37 import org.ximtec.igesture.MultimodalGestureHandler;
38 import org.ximtec.igesture.MultimodalGestureManager;
39 import org.ximtec.igesture.MultimodalGestureRecogniser;
40 import org.ximtec.igesture.Recogniser;
41 import org.ximtec.igesture.algorithm.Algorithm;
42 import org.ximtec.igesture.algorithm.AlgorithmException;
43 import org.ximtec.igesture.configuration.Configuration;
44 import org.ximtec.igesture.core.Descriptor;
45 import org.ximtec.igesture.core.Gesture;
46 import org.ximtec.igesture.core.GestureClass;
47 import org.ximtec.igesture.core.GestureSet;
48 import org.ximtec.igesture.core.ResultSet;
49 import org.ximtec.igesture.core.composite.CompositeDescriptor;
50 import org.ximtec.igesture.event.GestureHandler;
51 import org.ximtec.igesture.io.AbstractGestureDevice;
52 import org.ximtec.igesture.io.DeviceManagerListener;
53 import org.ximtec.igesture.io.GestureEventListener;
54 import org.ximtec.igesture.io.IDeviceManager;
55 import org.ximtec.igesture.tool.GestureConstants;
56 import org.ximtec.igesture.tool.core.Controller;
57 import org.ximtec.igesture.tool.core.TabbedView;
58 import org.ximtec.igesture.tool.service.DeviceManagerService;
59 import org.ximtec.igesture.tool.util.Formatter;
60 import org.ximtec.igesture.tool.util.TitleFactory;
61 import org.ximtec.igesture.tool.view.AbstractPanel;
62 import org.ximtec.igesture.tool.view.MainModel;
63 import org.ximtec.igesture.tool.view.composite.action.AddRecogniserAction;
64 import org.ximtec.igesture.tool.view.composite.action.RecogniseAction;
65 import org.ximtec.igesture.tool.view.composite.action.ResetAction;
66 import org.ximtec.igesture.tool.view.devicemanager.DeviceManagerController;
67
68
69
70
71
72 public class CompositeView extends AbstractPanel implements TabbedView, DeviceManagerListener{
73
74 private static final Logger LOGGER = Logger.getLogger(CompositeView.class.getName());
75
76 private Map<String, Class> algMapping;
77
78 private IDeviceManager deviceManager;
79 private MultimodalGestureRecogniser mmrecogniser;
80 private MultimodalGestureManager mmmanager;
81
82 private List<Recogniser> recognisers;
83 private List<Connector> connectors;
84
85 private JButton addRecogniserButton, btnRecognise, btnReset;
86 private JComboBox cmbSets;
87 private JList deviceList, resultList, algList, setList;
88
89 public CompositeView(Controller controller) {
90 super(controller);
91 deviceManager = getController().getLocator().getService(DeviceManagerService.IDENTIFIER, DeviceManagerController.class);
92 deviceManager.addDeviceManagerListener(this);
93 recognisers = new ArrayList<Recogniser>();
94 connectors = new ArrayList<Connector>();
95 init();
96 }
97
98
99
100
101 private void init() {
102 setTitle(TitleFactory.createStaticTitle(GestureConstants.COMPOSITE_TEST_BENCH_VIEW));
103 setContent(initConfigurationPanel());
104 setBottom(initMultimodalPanel());
105 }
106
107
108
109
110 private JPanel initMultimodalPanel() {
111 JPanel configPanel = new JPanel();
112
113 JLabel lblSets = getComponentFactory().createLabel(GestureConstants.COMPOSITE_GESTURE_SET);
114 cmbSets = new JComboBox(getController().getLocator().getService(MainModel.IDENTIFIER, MainModel.class)
115 .getGestureSets().toArray());
116 cmbSets.insertItemAt("Select a set", 0);
117 cmbSets.setSelectedIndex(0);
118 cmbSets.setEnabled(false);
119
120 btnRecognise = createButton(GestureConstants.COMPOSITE_RECOGNISE_ACTION,
121 new RecogniseAction(getController()), false);
122
123 btnReset = createButton(GestureConstants.COMPOSITE_RESET_ACTION,
124 new ResetAction(getController()), false);
125
126 cmbSets.addActionListener(new ActionListener(){
127
128 @Override
129 public void actionPerformed(ActionEvent e) {
130 JComboBox source = (JComboBox) e.getSource();
131
132 if(source.getSelectedIndex() == 0)
133 {
134
135 btnRecognise.setEnabled(false);
136 }
137 else
138 {
139 GestureSet set = (GestureSet) cmbSets.getSelectedItem();
140
141 boolean validConfig = initMultimodalRecogniser(set);
142 if(validConfig)
143 {
144
145 btnRecognise.setEnabled(true);
146 }
147 else
148 {
149 JOptionPane.showMessageDialog(null,"Please select a set that contains composite gestures!","Error",JOptionPane.ERROR_MESSAGE);
150 btnRecognise.setEnabled(false);
151 }
152
153 }
154 }
155
156 });
157
158 configPanel.add(lblSets);
159 configPanel.add(cmbSets);
160 configPanel.add(btnRecognise);
161 configPanel.add(btnReset);
162
163 JPanel panel = new JPanel();
164 panel.setLayout(new BorderLayout());
165 panel.add(configPanel,BorderLayout.CENTER);
166
167 resultList = createList(null);
168 resultList.setEnabled(false);
169 JScrollPane scrollResultList = new JScrollPane(resultList);
170 scrollResultList.setBorder(BorderFactory.createTitledBorder(BorderFactory.createLineBorder(Color.GRAY),"Results"));
171 panel.add(scrollResultList,BorderLayout.SOUTH);
172
173 return panel;
174 }
175
176
177
178
179 private JPanel initConfigurationPanel() {
180 JPanel panel = new JPanel();
181 panel.setLayout(new GridBagLayout());
182
183 algMapping = new HashMap<String, Class>();
184 for (Iterator iterator = getController().getLocator().getService(MainModel.IDENTIFIER, MainModel.class).getAlgorithms().iterator(); iterator.hasNext();) {
185 Class<? extends Algorithm> alg = (Class<? extends Algorithm>) iterator.next();
186 algMapping.put(alg.getSimpleName(),alg);
187 }
188
189 algList = createList(algMapping.keySet());
190 GridBagLayouter.addComponent(panel, createScrollableList("Algorithms:", algList),
191 0, 0, 1, 1, 0, 0, 0, 0, 0.5, 0.5, GridBagConstraints.CENTER, GridBagConstraints.BOTH);
192
193 setList = createList(getController().getLocator().getService(MainModel.IDENTIFIER, MainModel.class).getGestureSets());
194 GridBagLayouter.addComponent(panel, createScrollableList("Gesture Sets:", setList),
195 1, 0, 1, 1, 0, 0, 0, 0, 0.7, 0.5, GridBagConstraints.CENTER, GridBagConstraints.BOTH);
196
197 deviceList = createList(getController().getLocator().getService(DeviceManagerService.IDENTIFIER, DeviceManagerController.class).getDevices());
198 GridBagLayouter.addComponent(panel, createScrollableList("Devices:", deviceList),
199 2, 0, 1, 1, 0, 0, 0, 0, 0.2, 0.5, GridBagConstraints.CENTER, GridBagConstraints.BOTH);
200
201 addRecogniserButton = createButton(GestureConstants.COMPOSITE_ADD_RECOGNISER_ACTION,
202 new AddRecogniserAction(getController()), false);
203 GridBagLayouter.addComponent(panel, addRecogniserButton,
204 2, 2, 1, 1, 0, 0, 0, 0, 0.2, 0.0, GridBagConstraints.CENTER, GridBagConstraints.BOTH);
205
206 ListSelectionListener listener = new ListSelectionListener(){
207
208 @Override
209 public void valueChanged(ListSelectionEvent e) {
210
211 if(!e.getValueIsAdjusting())
212 {
213 if(!algList.isSelectionEmpty() && !setList.isSelectionEmpty() && !deviceList.isSelectionEmpty())
214 addRecogniserButton.setEnabled(true);
215 else
216 addRecogniserButton.setEnabled(false);
217 }
218 }
219
220 };
221
222 algList.addListSelectionListener(listener);
223 setList.addListSelectionListener(listener);
224 deviceList.addListSelectionListener(listener);
225
226 return panel;
227 }
228
229 private JScrollPane createScrollableList(String title, JList list)
230 {
231 JScrollPane scroll = new JScrollPane(list);
232 scroll.setBorder(BorderFactory.createTitledBorder(BorderFactory.createLineBorder(Color.GRAY),title));
233 return scroll;
234 }
235
236 private JList createList(Collection data)
237 {
238 JList list = new JList();
239 DefaultListModel model = new DefaultListModel();
240 list.setModel(model);
241 if(data != null)
242 {
243 for (Iterator iterator = data.iterator(); iterator.hasNext();) {
244 Object object = (Object) iterator.next();
245 model.addElement(object);
246 }
247 }
248 return list;
249 }
250
251
252
253
254
255
256
257
258 private JButton createButton(String key, Action action, boolean enabled)
259 {
260 JButton btn = getComponentFactory().createButton(key,action);
261 btn.setEnabled(enabled);
262 Formatter.formatButton(btn);
263 return btn;
264 }
265
266
267
268
269
270 @Override
271 public Icon getIcon() {
272 return null;
273 }
274
275
276
277
278 @Override
279 public JComponent getPane() {
280 return this;
281 }
282
283
284
285
286 @Override
287 public String getTabName() {
288 return getComponentFactory().getGuiBundle().getName(
289 GestureConstants.COMPOSITE_TEST_BENCH_VIEW);
290 }
291
292
293
294
295 @Override
296 public void updateDeviceManagerListener(int operation, AbstractGestureDevice<?, ?> device) {
297 if(operation == DeviceManagerListener.ADD)
298 {
299 ((DefaultListModel)deviceList.getModel()).addElement(device);
300 }
301 else if(operation == DeviceManagerListener.REMOVE)
302 {
303 ((DefaultListModel)deviceList.getModel()).removeElement(device);
304 }
305 }
306
307 private boolean initMultimodalRecogniser(GestureSet set) {
308
309 GestureSet compositeGestures = new GestureSet();
310
311 List<GestureClass> gestures = set.getGestureClasses();
312 for (Iterator iterator = gestures.iterator(); iterator.hasNext();) {
313 GestureClass gestureClass = (GestureClass) iterator.next();
314 Descriptor d = gestureClass.getDescriptor(CompositeDescriptor.class);
315 if(d != null)
316 compositeGestures.addGestureClass(gestureClass);
317 }
318
319 if(compositeGestures.size()>0)
320 {
321 mmrecogniser = new MultimodalGestureRecogniser(compositeGestures,deviceManager);
322
323 mmrecogniser.addGestureHandler(new MultimodalGestureHandler(){
324
325 @Override
326 public void handle(String gestureClass) {
327
328 DefaultListModel model = (DefaultListModel)resultList.getModel();
329 model.addElement(gestureClass);
330 }
331
332 });
333
334
335 mmmanager = new MultimodalGestureManager(mmrecogniser);
336 for (Iterator iterator = recognisers.iterator(); iterator.hasNext();) {
337 Recogniser recogniser = (Recogniser) iterator.next();
338 mmmanager.addRecogniser(recogniser, false);
339 }
340 return true;
341 }
342 else
343 return false;
344
345 }
346
347 private class Connector implements GestureEventListener
348 {
349 private Recogniser recogniser;
350
351 public Connector(Recogniser recogniser)
352 {
353 this.recogniser = recogniser;
354 }
355
356
357
358
359 @Override
360 public void handleChunks(List<?> chunks) {
361 }
362
363
364
365
366 @Override
367 public void handleGesture(Gesture<?> gesture) {
368 recogniser.recognise(gesture);
369 }
370
371 }
372
373
374
375
376 public void reset() {
377
378 LOGGER.log(Level.INFO,"Resetting GUI and multi-modal recognition architecture.");
379
380 cmbSets.setSelectedIndex(0);
381 cmbSets.setEnabled(false);
382 btnRecognise.setEnabled(false);
383
384 algList.clearSelection();
385 algList.setEnabled(true);
386 setList.clearSelection();
387 setList.setEnabled(true);
388 deviceList.clearSelection();
389 deviceList.setEnabled(true);
390
391 ((DefaultListModel)resultList.getModel()).clear();
392
393
394 recognisers.clear();
395 connectors.clear();
396 Object[] dev = deviceList.getSelectedValues();
397 for (int i = 0; i < dev.length; i++) {
398 AbstractGestureDevice<?,?> device = (AbstractGestureDevice<?,?>)dev[i];
399 device.removeAllGestureHandler();
400 }
401
402 btnReset.setEnabled(false);
403 }
404
405
406
407
408
409 public void recognise(boolean startRecognising) {
410 if(startRecognising == true)
411 {
412 LOGGER.log(Level.INFO,"Multi-modal recognition process started.");
413 btnRecognise.setText("Stop");
414 mmrecogniser.start();
415 cmbSets.setEnabled(false);
416
417 deviceList.setEnabled(false);
418 algList.setEnabled(false);
419 setList.setEnabled(false);
420 addRecogniserButton.setEnabled(false);
421 btnReset.setEnabled(false);
422 }
423 else
424 {
425 LOGGER.log(Level.INFO,"Multi-modal recognition process stopped.");
426 btnRecognise.setText("Recognise");
427 mmrecogniser.stop();
428 cmbSets.setEnabled(true);
429 btnReset.setEnabled(true);
430 }
431 }
432
433
434
435
436 public void addRecogniser() {
437
438 Configuration config = new Configuration();
439
440 Object[] sets = setList.getSelectedValues();
441 for (int i = 0; i < sets.length; i++) {
442 config.addGestureSet((GestureSet)sets[i]);
443 }
444
445 Object[] algs = algList.getSelectedValues();
446 for (int i = 0; i < algs.length; i++) {
447 config.addAlgorithm(algMapping.get(algs[i]).getName());
448 }
449
450 try {
451
452 Recogniser recogniser = new Recogniser(config);
453 recognisers.add(recogniser);
454
455 recogniser.addGestureHandler(new GestureHandler(){
456
457 @Override
458 public void handle(ResultSet resultSet) {
459
460 DefaultListModel model = (DefaultListModel) resultList.getModel();
461 model.addElement(resultSet.getGesture().getName());
462 }
463
464 });
465
466
467
468 Connector connector = new Connector(recogniser);
469 connectors.add(connector);
470
471
472 Object[] dev = deviceList.getSelectedValues();
473 for (int i = 0; i < dev.length; i++) {
474 AbstractGestureDevice<?,?> device = (AbstractGestureDevice<?,?>)dev[i];
475 device.addGestureHandler(connector);
476 }
477
478 cmbSets.setEnabled(true);
479 btnReset.setEnabled(true);
480
481 LOGGER.log(Level.INFO,"Added Recogniser to multi-modal recognition architecture.");
482 } catch (AlgorithmException e1) {
483 e1.printStackTrace();
484 }
485 }
486
487 }