// Returns the distance between the index finger and thumb of the rightmost
// hand in the field of view of the Leap Motion sensor
private Double getGripDistance(Frame frame) {
Double gripDist = -1.0;
if (frame.isValid()) {
if (!frame.hands().isEmpty() && frame.fingers().count() >= 2) {
// right hand used to control arm
Hand rightHand = frame.hands().rightmost();
// get the two fingers with smallest X co-ords
Finger thumb = null, index = null;
Float minX = 9999f, minX2 = minX;
Iterator<Finger> it = rightHand.fingers().iterator();
while (it.hasNext()) {
Finger finger = it.next();
Float tipXPos = finger.stabilizedTipPosition().getX();
if (tipXPos < minX) {
minX2 = minX;
index = thumb;
minX = tipXPos;
thumb = finger;
} else if (tipXPos < minX2) {
minX2 = tipXPos;
index = finger;
}
}
Vector thumbTip = thumb.stabilizedTipPosition();
Vector indexTip = index.stabilizedTipPosition();
gripDist = (double) thumbTip.distanceTo(indexTip);
}
}
return gripDist;
}
// Returns the orientation of the palm given a frame of data
// The double returned is a real number between 0 and 1 indicating
// the component of the normal to the palm on the plane of the
// computer screen
private Double getPalmOrientation(Frame frame) {
Double armOrt = -1.0;
if (frame.isValid()) {
if (!frame.hands().isEmpty()) {
Vector normal = frame.hands().rightmost().palmNormal();
armOrt = Math.sqrt(normal.getX() * normal.getX() + normal.getY() * normal.getY());
}
}
return armOrt;
}
// Checks if the user wants to lock the position of the gripper
// For locking the gripper, there need to be at least two hands in the
// sensor's field of view, and the leftmost hand must have no fingers
// outstretched (clenched fist with the back of the hand facing the sensor)
private boolean checkGripperLock(Frame frame) {
boolean locked = false;
if (frame.isValid()) {
if (frame.hands().count() > 1) {
Hand leftHand = frame.hands().leftmost();
if (leftHand.fingers().isEmpty()) locked = true;
}
}
return locked;
}
/*
* set the internal fingerCoor and palmCoor according the LM frame
*/
public void setCoor(Frame frame) {
HandList hands = frame.hands();
int i = 0;
// for the existing hands
for (; i < hands.count(); i++) {
for (int j = 0; j < 5; j++) {
fingerCoor[i][j][0] = rangeConvert(hands.get(i).fingers().get(j).tipPosition());
fingerCoor[i][j][1] =
rangeConvert(hands.get(i).fingers().get(j).bone(Bone.Type.TYPE_DISTAL).prevJoint());
fingerCoor[i][j][2] =
rangeConvert(
hands.get(i).fingers().get(j).bone(Bone.Type.TYPE_INTERMEDIATE).prevJoint());
fingerCoor[i][j][3] =
rangeConvert(hands.get(i).fingers().get(j).bone(Bone.Type.TYPE_PROXIMAL).prevJoint());
fingerCoor[i][j][4] =
rangeConvert(hands.get(i).fingers().get(j).bone(Bone.Type.TYPE_METACARPAL).prevJoint());
}
palmCoor[i] = rangeConvert(hands.get(i).palmPosition());
}
// for non-existing hands:
for (; i < 2; i++) {
for (int j = 0; j < 5; j++) {
for (int k = 0; k < 5; k++) {
fingerCoor[i][j][k] = new Point3D(0, 0, -100);
}
}
palmCoor[i] = new Point3D(0, 0, -100);
}
}
public static void recog(ArrayList<UserGesture> gestureList, Controller controller)
throws IOException, InterruptedException, AWTException {
System.out.println("started recog");
long[] timer = new long[gestureList.size()];
int[] matched = new int[gestureList.size()];
int[] matched_attributes = new int[100];
String[] gestureStatus = new String[100];
while (MapperGUI.statusRecognizing == true) {
Thread.sleep(5);
frame = controller.frame();
currentHand = frame.hands().get(0);
for (int gestureIterator = 0; gestureIterator < gestureList.size(); gestureIterator++) {
if (frame.hands().isEmpty()) break;
if (matched[gestureIterator] == 0) {
System.out.println("NEW STARTFRAME");
// if((startFrame.get(gestureIterator)))startFrame.remove(gestureIterator);
startFrame.add(gestureIterator, frame);
timer[gestureIterator] = System.currentTimeMillis();
}
// System.out.println("timer start"+timer);
// for (int nodeIterator = matched[gestureIterator] + 1; nodeIterator < gestureList
// .get(gestureIterator).NodeList.size();) {
if (matched[gestureIterator] == gestureList.get(gestureIterator).NodeList.size() - 1) {
Robot.execute(gestureIterator);
// System.out.println("RECOGNITION SUCCESS for gesture number "+ gestureIterator + " ,
// action goes here");
if (Recorderv2.gestureList.get(Recorderv2.gestureCount - 1).cont == true) {
matched[gestureIterator] = matched[gestureIterator] - 1;
timer[gestureIterator] = System.currentTimeMillis();
matched_attributes[gestureIterator] = 0;
} else {
matched[gestureIterator] = 0;
timer[gestureIterator] = 0;
matched_attributes[gestureIterator] = 0;
}
break;
}
currentNode =
Recorderv2.gestureList.get(gestureIterator).NodeList.get(matched[gestureIterator]);
if (System.currentTimeMillis() < (timer[gestureIterator] + 200)) {
if (gestureList.get(gestureIterator).attributes.contains("XY")) {
if ((currentHand.palmPosition().getX()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getX())
< currentNode.hand0_x_denorm + deviationX
&& (currentHand.palmPosition().getX()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getX())
> currentNode.hand0_x_denorm - deviationX
&& (currentHand.palmPosition().getY()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getY())
< currentNode.hand0_y_denorm + deviationY
&& (currentHand.palmPosition().getY()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getY())
> currentNode.hand0_y_denorm - deviationY) {
// System.out.println(frame.hands().get(0).palmPosition().normalized());
matched_attributes[gestureIterator] = matched_attributes[gestureIterator] + 1;
}
}
if (gestureList.get(gestureIterator).attributes.contains("XZ")) {
if ((currentHand.palmPosition().getX()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getX())
< currentNode.hand0_x_denorm + deviationX
&& (currentHand.palmPosition().getX()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getX())
> currentNode.hand0_x_denorm - deviationX
&& (currentHand.palmPosition().getZ()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getZ())
< currentNode.hand0_z_denorm + deviationZ
&& (currentHand.palmPosition().getZ()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getZ())
> currentNode.hand0_z_denorm - deviationZ) {
// System.out.println(frame.hands().get(0).palmPosition().normalized());
matched_attributes[gestureIterator] = matched_attributes[gestureIterator] + 1;
}
}
if (gestureList.get(gestureIterator).attributes.contains("frontmost_finger_XY")) {
if ((currentHand.fingers().frontmost().tipPosition().getX()
- startFrame
.get(gestureIterator)
.hands()
.get(0)
.fingers()
.frontmost()
.tipPosition()
.getX())
< currentNode.hand0_frontmost_x + deviationX
&& (currentHand.fingers().frontmost().tipPosition().getX()
- startFrame
.get(gestureIterator)
.hands()
.get(0)
.fingers()
.frontmost()
.tipPosition()
.getX())
> currentNode.hand0_frontmost_x - deviationX
&& (currentHand.fingers().frontmost().tipPosition().getY()
- startFrame
.get(gestureIterator)
.hands()
.get(0)
.fingers()
.frontmost()
.tipPosition()
.getY())
< currentNode.hand0_frontmost_y + deviationY
&& (currentHand.fingers().frontmost().tipPosition().getY()
- startFrame
.get(gestureIterator)
.hands()
.get(0)
.fingers()
.frontmost()
.tipPosition()
.getY())
> currentNode.hand0_frontmost_y - deviationY) {
// System.out.println(frame.hands().get(0).palmPosition().normalized());
matched_attributes[gestureIterator] = matched_attributes[gestureIterator] + 1;
}
}
if (gestureList.get(gestureIterator).attributes.contains("frontmost_finger_XZ")) {
if ((currentHand.fingers().frontmost().tipPosition().getX()
- startFrame
.get(gestureIterator)
.hands()
.get(0)
.fingers()
.frontmost()
.tipPosition()
.getX())
< currentNode.hand0_frontmost_x + deviationX
&& (currentHand.fingers().frontmost().tipPosition().getX()
- startFrame
.get(gestureIterator)
.hands()
.get(0)
.fingers()
.frontmost()
.tipPosition()
.getX())
> currentNode.hand0_frontmost_x - deviationX
&& (currentHand.fingers().frontmost().tipPosition().getZ()
- startFrame
.get(gestureIterator)
.hands()
.get(0)
.fingers()
.frontmost()
.tipPosition()
.getZ())
< currentNode.hand0_frontmost_z + deviationZ
&& (currentHand.fingers().frontmost().tipPosition().getZ()
- startFrame
.get(gestureIterator)
.hands()
.get(0)
.fingers()
.frontmost()
.tipPosition()
.getZ())
> currentNode.hand0_frontmost_z - deviationZ) {
// System.out.println(frame.hands().get(0).palmPosition().normalized());
matched_attributes[gestureIterator] = matched_attributes[gestureIterator] + 1;
}
}
if (gestureList.get(gestureIterator).attributes.contains("roll")) {
if ((currentHand.palmNormal().roll()
- startFrame.get(gestureIterator).hands().get(0).palmNormal().roll())
< currentNode.hand0_roll + deviationRoll
&& (currentHand.palmNormal().roll()
- startFrame.get(gestureIterator).hands().get(0).palmNormal().roll())
> currentNode.hand0_roll - deviationRoll) {
// System.out.println(frame.hands().get(0).palmPosition().normalized());
matched_attributes[gestureIterator] = matched_attributes[gestureIterator] + 1;
}
}
if (gestureList.get(gestureIterator).attributes.contains("yaw")) {
if ((currentHand.palmNormal().yaw()
- startFrame.get(gestureIterator).hands().get(0).palmNormal().yaw())
< currentNode.hand0_yaw + deviationYaw
&& (currentHand.palmNormal().yaw()
- startFrame.get(gestureIterator).hands().get(0).palmNormal().yaw())
> currentNode.hand0_yaw - deviationYaw) {
// System.out.println(frame.hands().get(0).palmPosition().normalized());
matched_attributes[gestureIterator] = matched_attributes[gestureIterator] + 1;
}
}
if (gestureList.get(gestureIterator).attributes.contains("pitch")) {
if ((currentHand.palmNormal().pitch()
- startFrame.get(gestureIterator).hands().get(0).palmNormal().pitch())
< currentNode.hand0_pitch + deviationPitch
&& (currentHand.palmNormal().pitch()
- startFrame.get(gestureIterator).hands().get(0).palmNormal().pitch())
> currentNode.hand0_pitch - deviationPitch) {
// System.out.println(frame.hands().get(0).palmPosition().normalized());
matched_attributes[gestureIterator] = matched_attributes[gestureIterator] + 1;
}
}
if (matched_attributes[gestureIterator]
== Recorderv2.gestureList.get(gestureIterator).attributes.size()
&& matched_attributes[gestureIterator] != 0) {
matched[gestureIterator] = matched[gestureIterator] + 1;
timer[gestureIterator] = System.currentTimeMillis() + 200;
matched_attributes[gestureIterator] = 0;
System.out.println(
"matched node number "
+ matched[gestureIterator]
+ " of gesture "
+ gestureIterator
+ " with "
+ gestureList
.get(gestureIterator)
.NodeList
.get(matched[gestureIterator])
.hand0_x_denorm
+ "with "
+ (frame.hands().get(0).palmPosition().getX()
- startFrame.get(gestureIterator).hands().get(0).palmPosition().getX()));
} else {
matched_attributes[gestureIterator] = 0;
}
} else {
matched_attributes[gestureIterator] = 0;
matched[gestureIterator] = 0;
timer[gestureIterator] = 0;
}
// }
}
if (MapperGUI.statusRecognizing == false) break;
}
System.out.println("stopped recog");
// MapperGUI.runningRecognizing = false;
return;
}
@Override
public void paintComponent(Graphics g) {
Graphics2D g2d = (Graphics2D) g;
g2d.setColor(Color.LIGHT_GRAY);
g2d.fillRect(0, 0, this.getWidth(), this.getHeight());
g2d.setColor(Color.BLACK);
if (SampleListener.frame == null) return;
Frame frame = SampleListener.frame;
/*
for(Gesture gest : frame.gestures()){
Location loc = Location.fromVector(frame.fingers().fingerType(Type.TYPE_INDEX).get(0).tipPosition());
if(OldGesture.contains(gest.id())){
OldGesture og = OldGesture.getGesture(gest.id());
if(gest.state().equals(State.STATE_STOP)){
OldGesture.remove(og.getID());
System.out.println("dddd");
}
og.addLocation(Location.fromVector(frame.fingers().fingerType(Type.TYPE_INDEX).get(0).tipPosition()));
if(og.getLocations().size() <= 1) continue;
Location oldLoc = null;
g2d.setColor(Color.BLUE);
for(Location currLoc : og.getLocations()){
if(oldLoc == null){
oldLoc = currLoc;
continue;
}
g2d.drawLine(oldLoc.getX()*3 + 800, oldLoc.getZ()*3 + 400, currLoc.getX()*3 + 800, currLoc.getZ()*3 + 400);
oldLoc = currLoc;
}
}else{
new OldGesture(loc, gest.id());
}
}*/
Image image = MainFrame.controller.images().get(0);
// System.out.println(image.data().length);
byte[] data = image.data();
// g2d.setColor(Color.BLACK);
// g2d.fillRect(0, 0, image.width(), image.height());
int i = 0;
for (int w = 0; w < image.height(); w++) {
for (int h = 0; h < image.width(); h++) {
int value = data[i];
int rgb = value;
rgb = (rgb << 8) + value;
rgb = (rgb << 8) + value;
g2d.setColor(new Color(rgb));
g2d.drawRect(h + this.getWidth() - image.width(), w, 1, 1);
i++;
}
}
g2d.setColor(Color.black);
if (frame.hands().isEmpty()) return;
for (Hand hand : frame.hands()) {
Vector v = hand.palmPosition();
Location loc = Location.fromVector(v);
// System.out.println((int) (1.0 * 20/loc.getY()*100));
g2d.fillOval(loc.getX() * 3 + 800, loc.getZ() * 3 + 400, getSize(40, loc), getSize(40, loc));
v = hand.arm().center();
loc = Location.fromVector(v);
g2d.fillOval(loc.getX() * 3 + 800, loc.getZ() * 3 + 400, getSize(40, loc), getSize(40, loc));
for (Finger finger : hand.fingers()) {
v = finger.tipPosition();
loc = Location.fromVector(v);
g2d.fillRect(
loc.getX() * 3 + 800, loc.getZ() * 3 + 400, getSize(20, loc), getSize(20, loc));
v = finger.jointPosition(Joint.JOINT_DIP);
loc = Location.fromVector(v);
g2d.fillRect(
loc.getX() * 3 + 800, loc.getZ() * 3 + 400, getSize(10, loc), getSize(10, loc));
v = finger.jointPosition(Joint.JOINT_MCP);
loc = Location.fromVector(v);
g2d.fillRect(
loc.getX() * 3 + 800, loc.getZ() * 3 + 400, getSize(10, loc), getSize(10, loc));
v = finger.jointPosition(Joint.JOINT_PIP);
loc = Location.fromVector(v);
g2d.fillRect(
loc.getX() * 3 + 800, loc.getZ() * 3 + 400, getSize(20, loc), getSize(20, loc));
}
}
g2d.dispose();
}
public void onFrame(Controller controller) {
currentFrame = controller.frame();
boolean handsFlipped = false;
for (int i = 0; i < 2; i++) {
currentHand = currentFrame.hands().get(i);
if (currentHand.isValid()) {
for (int n = 0; n < 2; n++) {
if (currentHand.id() == handIds[n] && n != i) handsFlipped = true;
}
int num = i;
if (handsFlipped) num = ((num == 0) ? 1 : 0);
hands[num] = currentHand;
handIds[num] = currentHand.id();
if (!currentHand.tools().empty()) {
tools[num] = currentHand.tools().get(0);
}
FingerList fings = currentHand.fingers();
if (fings.count() >= 3) handOpen[num] = true;
else handOpen[num] = false;
for (int f = 0; f < 5; f++) {
if (f < fings.count()) {
Finger finger = fings.get(f);
fingers[f + (5 * num)] = finger;
} else fingers[f + (5 * num)] = new Finger();
}
} else {
int num = i;
if (handsFlipped) num = ((num == 0) ? 1 : 0);
hands[num] = new Hand();
handOpen[num] = false;
}
}
if (isConnected) {
data = new StringBuilder();
for (int i = 0; i < hands.length; i++) {
if (OS.contains("Linux")) {
data.append(HAND_NAMES[i] + "-x " + Math.round(hands[i].palmPosition().getX()) + "\n");
data.append(
HAND_NAMES[i]
+ "-y "
+ Math.round((hands[i].palmPosition().getY() - 220) * 1.6)
+ "\n");
data.append(HAND_NAMES[i] + "-z " + Math.round(hands[i].palmPosition().getZ()) + "\n");
} else {
data.append(
HAND_NAMES[i] + "-x " + Math.round(hands[i].stabilizedPalmPosition().getX()) + "\n");
data.append(
HAND_NAMES[i]
+ "-y "
+ Math.round((hands[i].stabilizedPalmPosition().getY() - 220) * 1.6)
+ "\n");
data.append(
HAND_NAMES[i] + "-z " + Math.round(hands[i].stabilizedPalmPosition().getZ()) + "\n");
}
data.append(HAND_NAMES[i] + "-visible " + hands[i].isValid() + "\n");
data.append(HAND_NAMES[i] + "-open " + handOpen[i] + "\n");
}
for (int i = 0; i < fingers.length; i++) {
data.append(
FINGER_NAMES[i] + "-x " + Math.round(fingers[i].stabilizedTipPosition().getX()) + "\n");
data.append(
FINGER_NAMES[i]
+ "-y "
+ Math.round((fingers[i].stabilizedTipPosition().getY() - 220) * 1.6)
+ "\n");
data.append(
FINGER_NAMES[i] + "-z " + Math.round(fingers[i].stabilizedTipPosition().getZ()) + "\n");
data.append(FINGER_NAMES[i] + "-visible " + fingers[i].isValid() + "\n");
}
for (int i = 0; i < tools.length; i++) {
data.append(
TOOL_NAMES[i] + "-x " + Math.round(tools[i].stabilizedTipPosition().getX()) + "\n");
data.append(
TOOL_NAMES[i]
+ "-y "
+ Math.round((tools[i].stabilizedTipPosition().getY() - 220) * 1.6)
+ "\n");
data.append(
TOOL_NAMES[i] + "-z " + Math.round(tools[i].stabilizedTipPosition().getZ()) + "\n");
data.append(TOOL_NAMES[i] + "-visible " + tools[i].isValid() + "\n");
}
}
}