Add support for (basic) pose tracking in skel solver

The people need to know

ovtk_track/__init__.py

@@ -0,0 +1,42 @@
+'''
+█████████████████████████████████████████████▀▀▀▀▀▀▀▀▀▀█████████████████████████████████████▀▄██████████████████████████
+█████████████████████████████████████████▀▀«─ⁿ^^^╜╜╜╝╝╣╬æ░█████████████████████████████████▀▄███████████████████████████
+███████████████████████████████████████████████████████▄▄╙╗▀█▀▀▀▀▀▀▀█████████████████████▀▄█████████████████████████████
+███████████████████████████████████████████████▀▀¼@╣╬╢╣╣æ@@╬ÑÑÑÑD╬ÑÑ╬╣æ╗╬▀▀█████████████▀▄██████████████████████████████
+████████████████████████████████████████████▀╔╢ÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD╬╖╬▀█████████▀████████████████████████████████
+██████████████████████████████████████████▀é]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ╣φ▀█████▌▄████████████████████████████████
+████████████████████████████████████████▀╗▓▓]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ]▓▓▓Ö▀██▐█████████████████████████████████
+██████████████████████████████████████▀╓▓▓▓▓▓▓▓]]DÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD╫▓▓▓▓▓▓▓u ██████████████████████████████████
+█████████████████████████████████████╓▓▓▀ÑÑ]Ñ▓▓▀▀▀ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ]D╫▓▓▓▓▓▓▀▓▓▓▐██████████████████████████████████
+███████████████████████████████████▀╔Ñ]ÑÑÑÑÑÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD]ÑÑDÑÑÑÑ]]ÑÑÑM██████████████▀▀▀▀▀▀▀▀▀▀▀▀▀▀███████
+██████████████████████████████████▌║ÑÑÑÑÑÑÑÑ╬M░╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑh██████████████ oh, here's a ███████
+█████████████████████████████████▌╓ÑÑÑD╝╢Ñ╬½╗╬╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ╬╟DÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑh██████████████  a fun fact  ███████
+█████████████████████████████████.ÑÑÑDÑ╙▓▄╓╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑU░╟ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┐██████████████▄▄▄▄▄▄▄▄▄▄▄▄▄▄███████
+████████████████████████████████⌐╣ÑÑÑ╓µ╝M╔╣Ñ╣'╬Ñ╬╫ÑÑÑÑÑÑ╬╬ÑÑÑÑÑÑDÑÑ╙╝╜,▄╕╢ÑÑÑÑÑÑÑÑÑÑ╬▐██████████████████████████████████
+████████████████████████████████║ÑÑÑÑæÖ▀ Ñ╬Ω▄LÑDM║ÑÑÑÑÑÑM╟ÑÑÑÑÑÑÑÑφΦ▀▀╠╔╬╬ÑÑÑÑÑÑÑÑÑÑÑ ██████████████████████████████████
+██████████████████████████████▀║ÑÑÑÑÑÑ╬ ╫M▄▓▄ ║D ╚DÑÑÑÑÑu║ÑÑMÑDÑÑÑÑ╬M`▄▄φΦ║ÑÑÑÑÑÑÑÑÑÑ╖▐█████████████████████████████████
+█████████████████████████████▌╓ÑÑÑÑÑÑDM┌ ▓▓▓▓L╢╣M ╬ÑÑÑÑÑ └Ñ╬~²╣ÑÑÑÑѼ▀'╔æ╢╬ÑÑÑÑÑÑÑÑÑÑÑ┐▀████████████████████████████████
+█████████████████████████████╔ÑM╬ÑÑÑÑÑù╓▀"``  "²ⁿ▌║ÑÑÑÑÑhL╢Ñ∩µ╢m─ªª╣╣M½║DÑÑÑÑÑÑÑÑÑÑÑÑÑÑ,████████████████████████████████
+████████████████████████████ ╬^║ÑÑÑÑÑÑM▓H ,╗h    ▓ ÑÑÑÑÑh▓ ÑM           ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ ███████████████████████████████
+███████████████████████████▌╣ Γ╣ÑÑÑÑÑDM║  ╢╬     ╫▌▄▄▄▄▄▄▓▓▄▄    '╬▓w   ÑÑÑÑÑÑÑÑÑÑÑÑÑD╬`▄███████████████████████████████
+███████████████████████████  █ ╬ÑÑÑÑÑÑÑ▐▌ ╬▀▀    ║██████████▄,    ╬╬╬   ÑÑÑÑÑÑÑÑÑÑÑÑ╣┘-▀5╔╢ ▀███████████████████████████
+███████████████████████████╘▐█ ÑÑÑÑÑÑÑÑ▐█▄██     ███████████▌     ▓▓▓ ▄ DÑÑÑÑÑÑÑÑÑÑ╖╖æ╣ÑÑÑÑHß ▀█████████████████████████
+███████████████████████████ ▐█ ÑÑ╬ÑÑÑÑÑ▐█▓██▓ ¥╜▄████████████ Φæ┘▄████▌╢ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ^╞▐█▄▄▀████████████████████▀▄
+███████▀█▄▄████████▄▄▄▀▀████▐█ ╬╬░╬ÑÑÑÑ▐▓╣▓▓▓▓████████████████▄▄██████ ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┘┤|█████▄▄▄▀▀██████████▀▀▄▄███
+███▀▄▄██████████████████▄▄▀███▌║ÑM╢ÑÑÑÑJ█▓▓▓███████████████████▓▓▓▓▓▓`╣ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┘▄M,████████████▄▄▄▄▄██████████
+█▀▄████████████████████████▄▀██µ╢┼,╢ÑÑÑD╣╬▀▀███████▄╬╬D█████████████▀║DÑÑÑÑÑÑÑÑÑÑDM╫DÑÑ╬ ██▄████████████████████████████
+▄████████████████████████████▄▀█ ╬▐█╙╬ÑÑÑÑ▐▄ "▀█████▄╢M███████████▀▀║ÑÑÑÑÑÑÑÑ╬╟ÑÑ╬ⁿ╬ÑÑ╝╓████████████████████████████████
+███████████████████████████████▄▀  ██▄░╢ÑÑM██   ▄`"▀▀▀▀▀▀▀▀"` .    ▄▄╢ÑÑÑM░╣╜║╬ÑM▄█║Ñ^▄█████████████████████████████████
+████████████████████████████████▄▀▄▀████▄╙╢▐███▄▐██▄         .⌡,#╓██ ╣╣╜}-,▄▌╬M▄███w▄███████████████████████████████████
+█████████████████████████████████▌▀▌▀▀▀▀,██▄████████▀7     ╓▓▌╙▀└██▌▐███████²▄██████████████████████████████████████████
+████████████████████████████████████▀,▄█████████▀▀«]░╓▓▓▓▓▓▓▀«]░J██ ████████████████████████████████████████████████████
+███████████████████████████████████▐███████▀▀µj]░  ░ █╣▓▓▓▀ j░ ░▐██ ▄Å▀▀████████████████████████████████████████████████
+█████████▀▀▀▀▀▀▀▀▀▀▀▀▀▀████████████ ██████┌]▌║ ░   r▐█▀██▀███▄`y███▐█▀█▌▄▀██████████████████████████████████████████████
+█████████   i do not   ████████████▌████▀▄▓ ╙▄▓`    ▓▓▓▓⌐jr╙████▄▀▌██▓█▓▌▐██████████████████████████████████████████████
+█████████    think     ████████████▌███▀▄▓▓╬ ╙▄▀▄░░ ▓▓▓▀j░ H`▀▀▀▀""` ]«]«¥██████████████████████████████████████████████
+█████████▄▄▄▄▄▄▄▄▄▄▄▄▄▄████████████▌██▌▐▓▓▓▓L,╨¥"Vµ ▓╣▌j░░U    ░  ░ ░,µ⌐Ö║██████████████████████████████████████████████
+███████████████████████████████████ ██,▓▓▓▓▓ ░½`"▀▄ ▌▌/δ▀DxH*¥#==═K*"▄x╨░d██████████████████████████████████████████████
+███████████████████████████████████▐█`╣▓▓▓▓▓µ░░░^ ` ╚ "░ ,«⌂]"M╨*"""^▄▄▓╙▓▐█████████████████████████████████████████████
+██████████████████████████████████ ██▐▌║▓▓▓▓▌▄▄D,╔,ⁿ«-░░░,⌂░^╓ó],▄▓█▓▓▓▓▌Γ██████████████████████████████████████████████
+█████████████████████████████████▀██ ▓▓Ω▓▓▓▓▓▓▀]░`j ]j⌂,"]H╓⌐▓▓▓▓▓▓▓▓▓▓▓▓µ▐█████████████████████████████████████████████
+'''

ovtk_track/output/debug.py

@@ -1,4 +1,5 @@
 import cv2
+import matplotlib.pyplot as plt
 
 from . import OutputProcess
 from ovtk_track import types
@@ -14,7 +15,10 @@ class Process(OutputProcess):
         super().__init__(*args)
 
     def setup(self):
-        pass
+        self.fig = plt.figure()
+        self.axes = self.fig.add_subplot(projection='3d')
+        self.axes.view_init(15, 0, vertical_axis='y')
+        plt.show(block=False)
 
     def send(self):
         landmarks = self._inputs['landmarks'].get_nowait()
@@ -31,8 +35,15 @@ class Process(OutputProcess):
             landmarks.draw(image, frame, label=False, color=(130, 130, 130))
 
         if skeleton is not None:
-            skeleton.draw(image, frame)
+            skeleton.draw(self.axes)
 
         cv2.imshow("face", frame)
+        plt.draw()
+
+        # event loops
+        plt.pause(0.0001)
         if cv2.waitKey(1) & 0xFF == ord('q'):
             raise KeyboardInterrupt('User requested stop')
+
+        for artist in plt.gca().lines + plt.gca().collections:
+            artist.remove()

ovtk_track/transform/solve/mediapipe/__init__.py

@@ -32,6 +32,26 @@ face_mesh_map = {
     ],
 }
 
+body_map = {
+    LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.LEFT: [11],
+    LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.RIGHT: [12],
+
+    LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.LEFT: [13],
+    LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.RIGHT: [14],
+
+    LANDMARK_TYPES.HIP | LANDMARK_TYPES.LEFT: [23],
+    LANDMARK_TYPES.HIP | LANDMARK_TYPES.RIGHT: [24],
+
+    LANDMARK_TYPES.KNEE | LANDMARK_TYPES.LEFT: [25],
+    LANDMARK_TYPES.KNEE | LANDMARK_TYPES.RIGHT: [26],
+
+    LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.LEFT: [27],
+    LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.RIGHT: [28],
+
+    LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.LEFT: [27],
+    LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.RIGHT: [28],
+}
+
 # SEE YEAH THESE MAKE SENSE GOOGLE WHAT THE HELL
 hand_mesh_map = {LANDMARK_TYPES.HAND | LANDMARK_TYPES.WRIST: [0]}
 _finger_map = {

ovtk_track/transform/solve/mediapipe/holistic.py

@@ -2,7 +2,7 @@ import mediapipe
 import numpy as np
 
 from ovtk_track.transform import TransformProcess
-from ovtk_track.transform.solve.mediapipe import face_mesh_map, hand_mesh_map
+from ovtk_track.transform.solve.mediapipe import face_mesh_map, hand_mesh_map, body_map
 from ovtk_track import types
 from ovtk_track.types.Landmarks import LANDMARK_TYPES
 
@@ -83,6 +83,17 @@ class Process(TransformProcess):
 
                 available.append((right_hand_landmarks, mix_maps(hand_mesh_map, LANDMARK_TYPES.RIGHT)))
 
+        if results.pose_landmarks:
+            raw_landmarks = results.pose_landmarks.landmark
+            body_landmarks = np.empty((33, 3), dtype=np.float32)
+
+            for i in range(33):
+                body_landmarks[i][0] = raw_landmarks[i].x
+                body_landmarks[i][1] = raw_landmarks[i].y
+                body_landmarks[i][2] = raw_landmarks[i].z
+
+            available.append((body_landmarks, body_map))
+
             if available:
                 avail_landmarks, maps = zip(*available)
                 combo_map = combine_maps(zip(maps, (array.shape[0] for array in avail_landmarks)))

ovtk_track/transform/solve/skel_from_3d_landmarks.py

@@ -1,6 +1,7 @@
 import math
 
 import numpy as np
+from scipy.spatial.distance import cdist
 
 from .. import TransformProcess
 from ovtk_track.types import Quaternion, Point3d
@@ -24,9 +25,97 @@ class Process(TransformProcess):
         self.normal = np.array(normal, dtype=float)
         self.up = np.array(vec_perp(normal), dtype=float)
 
+        # REVIEW: See calc_eye. These probably need to change based on normal / up.
+        #         Or maybe they dont and we just rotate the output quaternion?
+        #         Ugh. The code works for now, but i no understand....
+        self.SIN_LEFT_THETA = 2 * np.sin(np.pi / 2)
+        self.SIN_UP_THETA = np.sin(np.pi / 6)
+
     def setup(self):
         pass
 
+    def calc_head(self, landmarks):
+        # REVIEW: This doesnt really work quite right!! look + roll arent mixing as expected
+        # Vector pointing from head center to nose
+        nose = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.NOSE | LANDMARK_TYPES.TIP]).mean(axis=0)
+        head_center = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.FACE | LANDMARK_TYPES.OUTLINE]).mean(axis=0)
+        look_vec = (nose - head_center)
+        look_vec /= np.linalg.norm(look_vec)
+        # Vector pointing left to right across the face
+        eye_center_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]).mean(axis=0)
+        eye_center_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]).mean(axis=0)
+        roll_vec = (eye_center_l - eye_center_r)
+        roll_vec /= np.linalg.norm(roll_vec)
+
+        # Quat that rotates from normal to head center -> nose vec
+        look = Quaternion(np.dot(look_vec, self.normal), *np.cross(look_vec, self.normal))
+        look.w += look.magnitude()
+        look = look.normalize()
+        # Quat that represents a rotation around the roll axis (i think??)
+        roll_angle = np.sum(roll_vec * self.up)
+        roll = Quaternion(math.cos(roll_angle), *(self.normal * math.sin(roll_angle)))
+        roll = roll.normalize()
+
+        combo = look + roll
+        combo = combo.normalize()
+
+        return combo, head_center
+
+    def calc_eye(self, landmarks):
+        # Get poi
+        corners = np.empty((2, 2, 3), dtype=np.float32)
+        centers = np.empty((2, 3), dtype=np.float32)
+        pupils = np.empty((2, 3), dtype=np.float32)
+        cross_heights = np.empty((2), dtype=np.float32)
+        for i, side in enumerate([LANDMARK_TYPES.LEFT, LANDMARK_TYPES.RIGHT]):
+            # Find corners by searching for points with the largest distance from each other
+            # REVIEW: These *should* will always be the same points in the map - make a landmark type selector?
+            eye_outline = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.OUTLINE | side])
+            hdist = cdist(eye_outline, eye_outline, metric='euclidean')
+            best_pair = np.unravel_index(hdist.argmax(), hdist.shape)
+            corners[i] = eye_outline[best_pair[0]], eye_outline[best_pair[1]]
+            # Get height of eye (relative to a line passing through each corner)
+            cross_heights[i] = np.array([
+                np.linalg.norm(np.cross(corners[i][1]-corners[i][0],
+                                        corners[i][0]-point))
+                / np.linalg.norm(corners[i][0]-corners[i][1])
+                for point in eye_outline
+            ]).max()
+
+            centers[i] = eye_outline.mean(axis=0)
+            pupils[i] = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | side]).mean(axis=0)
+        # Calculate important distances based on POI
+        eye_length = np.linalg.norm(np.diff(corners, axis=1), axis=(2, 1))
+        ic_distance = np.linalg.norm(pupils - centers, axis=1)
+        zc_distance = np.linalg.norm(pupils - corners[:, 1], axis=1)
+        aspect_ratio = 1 / (cross_heights / eye_length)
+
+        # Takes above and spits out spherical coordiates of pupil (relative to camera)
+        # Black magic as far as i can comprehend
+        # Copied in large part from https://github.com/1996scarlet/OpenVtuber/blob/970229d3a5ebe14a7519352da039d00a0b87e2d9/service/TFLiteIrisLocalization.py#L101
+        s0 = (corners[1, :, 1] - corners[0, :, 1]) * pupils[:, 0]
+        s1 = (corners[1, :, 0] - corners[0, :, 0]) * pupils[:, 1]
+        s2 = corners[1, :, 0] * corners[0, :, 1]
+        s3 = corners[1, :, 1] * corners[0, :, 0]
+
+        delta_y = (s0 - s1 + s2 - s3) / eye_length / 2
+        delta_x = np.sqrt(abs(ic_distance**2 - delta_y**2))
+        delta = np.array((delta_x * self.SIN_LEFT_THETA,
+                          delta_y * self.SIN_UP_THETA))
+        delta /= eye_length
+        theta, pha = np.arcsin(delta)
+        inv_judge = zc_distance**2 - delta_y**2 < eye_length**2 / 4
+        theta[inv_judge] *= -1
+
+        # Convert spherical coordiates to quaternions
+        # Based on https://github.com/moble/quaternion/blob/8f6fc306306c45f0bf79331a22ef3998e4d187bc/src/quaternion/__init__.py#L599
+        quats = np.array([np.cos(pha/2) * np.cos(theta/2),
+                         -np.sin(pha/2) * np.sin(theta/2),
+                         np.cos(pha/2) * np.sin(theta/2),
+                         np.sin(pha/2) * np.cos(theta/2)]).T
+        quat_arr = [Quaternion(*quat) for quat in quats]
+        return quat_arr, aspect_ratio
+
     def process(self):
         landmarks = self._inputs['landmarks'].get()
         skeleton = None
@@ -34,46 +123,42 @@ class Process(TransformProcess):
             joints = {}
             if landmarks.has(LANDMARK_TYPES.FACE):
                 # Get head look / pos
-                nose = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.NOSE | LANDMARK_TYPES.TIP]).mean(0)
-                head_center = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.FACE | LANDMARK_TYPES.OUTLINE]).mean(0)
-                look_vec = (nose - head_center)
+                look_quat, head_pos = self.calc_head(landmarks)
+                eye_quats, eye_aspect = self.calc_eye(landmarks)
 
-                eye_center_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]).mean(0)
-                eye_center_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]).mean(0)
-                roll_vec = (eye_center_l - eye_center_r)
-
-                look_vec /= np.linalg.norm(look_vec)
-                roll_vec /= np.linalg.norm(roll_vec)
-
-                roll_angle = np.sum(roll_vec * self.up)
-                roll = Quaternion(math.cos(roll_angle), * self.normal * math.sin(roll_angle))
-                roll = roll.normalize()
-
-                look = Quaternion(np.dot(look_vec, self.normal), *np.cross(look_vec, self.normal))
-                look.w += look.magnitude()
-                look = look.normalize()
-
-                combo = look + roll
-                combo = combo.normalize()
-
-                # Get eye data
-                marks_left = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT])
-                marks_right = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT])
-                range = np.array([marks_left.max(axis=0) - marks_left.min(axis=0),
-                                  marks_right.max(axis=0) - marks_right.min(axis=0)])
-                delta = np.array([eye_center_l - Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | LANDMARK_TYPES.CENTER | LANDMARK_TYPES.LEFT]).mean(0),
-                                  eye_center_r - Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | LANDMARK_TYPES.CENTER | LANDMARK_TYPES.RIGHT]).mean(0)])
-
-                delta /= range
-                try:
-                    eye_aspect_ratio = range[::, 0] / range[::, 1]
-                except ZeroDivisionError:
-                    eye_aspect_ratio = None
-
-                head_joint = Joint(Point3d(*head_center), combo, dict(look_delta=delta, eye_aspect_ratio=eye_aspect_ratio))
+                head_joint = Joint(Point3d(*head_pos), look_quat, attr=dict(eye_rot=eye_quats, eye_aspect=eye_aspect))
 
                 joints[JOINT_TYPES.HEAD] = head_joint
 
+            if landmarks.has(LANDMARK_TYPES.SHOULDER):
+                shoulder_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.LEFT]).mean(axis=0)
+                shoulder_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.RIGHT]).mean(axis=0)
+                joints[JOINT_TYPES.SHOULDER_L] = Joint(Point3d(*shoulder_l), Quaternion.identity())
+                joints[JOINT_TYPES.SHOULDER_R] = Joint(Point3d(*shoulder_r), Quaternion.identity())
+
+            if landmarks.has(LANDMARK_TYPES.ELBOW):
+                elbow_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.LEFT]).mean(axis=0)
+                elbow_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.RIGHT]).mean(axis=0)
+                joints[JOINT_TYPES.ELBOW_L] = Joint(Point3d(*elbow_l), Quaternion.identity())
+                joints[JOINT_TYPES.ELBOW_R] = Joint(Point3d(*elbow_r), Quaternion.identity())
+
+            if landmarks.has(LANDMARK_TYPES.HIP):
+                hips = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.HIP]).mean(axis=0)
+                joints[JOINT_TYPES.HIPS] = Joint(Point3d(*hips), Quaternion.identity())
+
+            # Synthizise other joints from existing data
+            if not joints.get(JOINT_TYPES.CHEST):
+                if landmarks.has(LANDMARK_TYPES.SHOULDER) and landmarks.has(LANDMARK_TYPES.HIP):
+                    chest = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER, LANDMARK_TYPES.HIP]).mean(axis=0)
+                    joints[JOINT_TYPES.CHEST] = Joint(Point3d(*chest), Quaternion.identity())
+                elif joints.get(JOINT_TYPES.HEAD):
+                    chest_center = joints[JOINT_TYPES.HEAD].pos.as_np()
+                    chest_center = np.power(chest_center, 3) / (1e3 + np.power(chest_center, 2))
+                    chest_center -= [0, 100, 0]
+                    chest_rot = Quaternion.identity().slerp(joints[JOINT_TYPES.HEAD].rot, 0.1)
+
+                    joints[JOINT_TYPES.CHEST] = Joint(Point3d(*chest_center), chest_rot)
+
             skeleton = Skeleton(joints)
 
         self._outputs['skel'].send(skeleton)

ovtk_track/types/Landmarks.py

@@ -17,6 +17,17 @@ class LANDMARK_TYPES(Flag):
     LIPS = auto()
     CHIN = auto()
 
+    # Body
+    SHOULDER = auto()
+    ELBOW = auto()
+    HIP = auto()
+    KNEE = auto()
+
+    # Feet tracking lmao
+    ANKLE = auto()
+    HEEL = auto()
+    TOE_INDEX = auto()
+
     # Hand
     HAND = auto()
     WRIST = auto()
@@ -71,8 +82,9 @@ class Landmarks(Type):
 
         return False
 
-    def draw(self, image, canvas, color=(255, 255, 255), label=True):
-        for i, (x, y, z) in enumerate(point.project_to_image(image) for point in self.points):
+    def draw(self, image, canvas, color=(255, 255, 255), label=True, filter=None):
+        points = self[filter] if filter else self.points
+        for i, (x, y, z) in enumerate(point.project_to_image(image) for point in points):
             if x > image.width or x < 0 or y > image.height or y < 0:
                 continue
 

ovtk_track/types/Quaternion.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 import numpy as np
-from scipy.spatial.transform import Rotation
+from scipy.spatial.transform import Rotation, Slerp
 from .Type import Type
 
 
@@ -12,6 +12,10 @@ class Quaternion(Type):
     y: float
     z: float
 
+    @classmethod
+    def identity(cls):
+        return cls(1, 0, 0, 0)
+
     def __mul__(self, q):
         if isinstance(q, self.__class__):
             product = self.as_np() * q.as_np()
@@ -45,6 +49,16 @@ class Quaternion(Type):
     def conjugate(self):
         return self.__class__(self.w, -self.x, -self.y, -self.z)
 
+    def slerp(self, other, t):
+        r = Rotation.from_quat([
+            [self.x, self.y, self.z, self.w],
+            [other.x, other.y, other.z, other.w],
+        ])
+
+        slerp = Slerp([0, 1], r)
+        x, y, z, w = slerp([t]).as_quat()[0]
+        return self.__class__(w, x, y, z)
+
     def draw(self, canvas, origin):
         raise NotImplementedError()
 

ovtk_track/types/Skeleton.py

@@ -2,8 +2,6 @@ from dataclasses import dataclass, field
 from enum import Enum
 import typing
 
-import cv2
-
 from .Type import Type
 from .Point3d import Point3d
 from .Quaternion import Quaternion
@@ -12,20 +10,26 @@ from .Quaternion import Quaternion
 class JOINT_TYPES(Enum):
     HEAD = 'head'
     CHEST = 'chest'
+    HIPS = 'hips'
 
     SHOULDER_L = 'shoulder_l'
     ELBOW_L = 'elbow_l'
-    WRIST_L = 'wrist_l'
-    HIP_L = 'hip_l'
     KNEE_L = 'knee_l'
     FOOT_L = 'foot_l'
+    WRIST_L = 'wrist_l'
 
     SHOULDER_R = 'shoulder_r'
     ELBOW_R = 'elbow_r'
-    WRIST_R = 'wrist_r'
-    HIP_R = 'hip_r'
     KNEE_R = 'knee_r'
     FOOT_R = 'foot_r'
+    WRIST_R = 'wrist_r'
+
+
+default_colors = [
+    [255, 0, 0], [0, 255, 0], [0, 0, 255],
+    [255, 255, 0], [255, 0, 255], [0, 255, 255],
+    [128, 255, 0], [255, 0, 128], [0, 255, 128],
+]
 
 
 @dataclass
@@ -48,14 +52,10 @@ class Skeleton(Type):
         # TODO: More intelegent merge
         return Skeleton(self.joints + other.joints)
 
-    def draw(self, image, canvas, color=(255, 255, 255)):
-        for i, joint in enumerate(self.joints.values()):
-            x, y, z = joint.pos.project_to_image(image)
-
-            if x > image.width or x < 0 or y > image.height or y < 0:
-                continue
-
-            cv2.circle(canvas, (x, y), 1, color, -1, cv2.LINE_AA)
+    def draw(self, axes, colors=default_colors):
+        xs, ys, zs = zip(*(joint.pos.as_np() for joint in self.joints.values()))
+        color = [[v / 255 for v in color] for color in colors[:len(xs)]]
+        axes.scatter(xs, ys, zs, c=color)
 
     def serialize(self):
         return {type.value: joint for type, joint in self.joints.items()}