From 92044fa866b093f17b4b2216ba9d345b471cf673 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Sun, 28 Nov 2021 13:20:04 -0700 Subject: [PATCH 1/8] Add better skeleton debug view --- ovtk_track/output/debug.py | 15 +++++++++++++-- ovtk_track/types/Skeleton.py | 26 ++++++++++++++------------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/ovtk_track/output/debug.py b/ovtk_track/output/debug.py index 10bb93c..7125f18 100644 --- a/ovtk_track/output/debug.py +++ b/ovtk_track/output/debug.py @@ -1,4 +1,5 @@ import cv2 +import matplotlib.pyplot as plt from . import OutputProcess from ovtk_track import types @@ -14,7 +15,10 @@ class Process(OutputProcess): super().__init__(*args) def setup(self): - pass + self.fig = plt.figure() + self.axes = self.fig.add_subplot(projection='3d') + self.axes.view_init(15, 0, vertical_axis='y') + plt.show(block=False) def send(self): landmarks = self._inputs['landmarks'].get_nowait() @@ -31,8 +35,15 @@ class Process(OutputProcess): landmarks.draw(image, frame, label=False, color=(130, 130, 130)) if skeleton is not None: - skeleton.draw(image, frame) + skeleton.draw(self.axes) cv2.imshow("face", frame) + plt.draw() + + # event loops + plt.pause(0.0001) if cv2.waitKey(1) & 0xFF == ord('q'): raise KeyboardInterrupt('User requested stop') + + for artist in plt.gca().lines + plt.gca().collections: + artist.remove() diff --git a/ovtk_track/types/Skeleton.py b/ovtk_track/types/Skeleton.py index f9bfe94..d4a8e0c 100644 --- a/ovtk_track/types/Skeleton.py +++ b/ovtk_track/types/Skeleton.py @@ -3,6 +3,7 @@ from enum import Enum import typing import cv2 +import matplotlib.pyplot as plt from .Type import Type from .Point3d import Point3d @@ -12,22 +13,27 @@ from .Quaternion import Quaternion class JOINT_TYPES(Enum): HEAD = 'head' CHEST = 'chest' + HIPS = 'hips' SHOULDER_L = 'shoulder_l' ELBOW_L = 'elbow_l' - WRIST_L = 'wrist_l' - HIP_L = 'hip_l' KNEE_L = 'knee_l' FOOT_L = 'foot_l' + WRIST_L = 'wrist_l' SHOULDER_R = 'shoulder_r' ELBOW_R = 'elbow_r' - WRIST_R = 'wrist_r' - HIP_R = 'hip_r' KNEE_R = 'knee_r' FOOT_R = 'foot_r' + WRIST_R = 'wrist_r' +default_colors = [ + [255, 0, 0], [0, 255, 0], [0, 0, 255], + [255, 255, 0], [255, 0, 255], [0, 255, 255], + [128, 255, 0], [255, 0, 128], [0, 255, 128], +] + @dataclass class Joint(Type): pos: Point3d @@ -48,14 +54,10 @@ class Skeleton(Type): # TODO: More intelegent merge return Skeleton(self.joints + other.joints) - def draw(self, image, canvas, color=(255, 255, 255)): - for i, joint in enumerate(self.joints.values()): - x, y, z = joint.pos.project_to_image(image) - - if x > image.width or x < 0 or y > image.height or y < 0: - continue - - cv2.circle(canvas, (x, y), 1, color, -1, cv2.LINE_AA) + def draw(self, axes, colors=default_colors): + xs, ys, zs = zip(*(joint.pos.as_np() for joint in self.joints.values())) + color = [[v / 255 for v in color] for color in colors[:len(xs)]] + axes.scatter(xs, ys, zs, c=color) def serialize(self): return {type.value: joint for type, joint in self.joints.items()} -- 2.45.1 From 4beddc5e6d605428baa3e390d52989e0199a77bf Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Sun, 28 Nov 2021 15:28:15 -0700 Subject: [PATCH 2/8] Chest default --- .../transform/solve/skel_from_3d_landmarks.py | 11 +++++++++++ ovtk_track/types/Quaternion.py | 16 +++++++++++++++- ovtk_track/types/Skeleton.py | 4 +--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/ovtk_track/transform/solve/skel_from_3d_landmarks.py b/ovtk_track/transform/solve/skel_from_3d_landmarks.py index c53afd3..67be3bd 100644 --- a/ovtk_track/transform/solve/skel_from_3d_landmarks.py +++ b/ovtk_track/transform/solve/skel_from_3d_landmarks.py @@ -74,6 +74,17 @@ class Process(TransformProcess): joints[JOINT_TYPES.HEAD] = head_joint + # Synthizise other joints from existing data + if not joints.get(JOINT_TYPES.CHEST) and joints.get(JOINT_TYPES.HEAD): + chest_center = joints[JOINT_TYPES.HEAD].pos.as_np() + chest_center = np.power(chest_center, 3) / (1e3 + np.power(chest_center, 2)) + chest_center -= [0, 100, 0] + + chest_rot = Quaternion.identity().slerp(joints[JOINT_TYPES.HEAD].rot, 0.1) + + chest_joint = Joint(Point3d(*chest_center), chest_rot) + joints[JOINT_TYPES.CHEST] = chest_joint + skeleton = Skeleton(joints) self._outputs['skel'].send(skeleton) diff --git a/ovtk_track/types/Quaternion.py b/ovtk_track/types/Quaternion.py index 3a26616..1e0f877 100644 --- a/ovtk_track/types/Quaternion.py +++ b/ovtk_track/types/Quaternion.py @@ -1,7 +1,7 @@ from dataclasses import dataclass import numpy as np -from scipy.spatial.transform import Rotation +from scipy.spatial.transform import Rotation, Slerp from .Type import Type @@ -12,6 +12,10 @@ class Quaternion(Type): y: float z: float + @classmethod + def identity(cls): + return cls(1, 0, 0, 0) + def __mul__(self, q): if isinstance(q, self.__class__): product = self.as_np() * q.as_np() @@ -45,6 +49,16 @@ class Quaternion(Type): def conjugate(self): return self.__class__(self.w, -self.x, -self.y, -self.z) + def slerp(self, other, t): + r = Rotation.from_quat([ + [self.x, self.y, self.z, self.w], + [other.x, other.y, other.z, other.w], + ]) + + slerp = Slerp([0, 1], r) + x, y, z, w = slerp([t]).as_quat()[0] + return self.__class__(w, x, y, z) + def draw(self, canvas, origin): raise NotImplementedError() diff --git a/ovtk_track/types/Skeleton.py b/ovtk_track/types/Skeleton.py index d4a8e0c..3358fe7 100644 --- a/ovtk_track/types/Skeleton.py +++ b/ovtk_track/types/Skeleton.py @@ -2,9 +2,6 @@ from dataclasses import dataclass, field from enum import Enum import typing -import cv2 -import matplotlib.pyplot as plt - from .Type import Type from .Point3d import Point3d from .Quaternion import Quaternion @@ -34,6 +31,7 @@ default_colors = [ [128, 255, 0], [255, 0, 128], [0, 255, 128], ] + @dataclass class Joint(Type): pos: Point3d -- 2.45.1 From 4ed2ac13841ee0dcad31c3c1aedca2d97c342eaf Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Thu, 30 Dec 2021 12:11:05 -0700 Subject: [PATCH 3/8] THATS RIGHT BINCH WE GOT PROPER EYE TRACKING WOOOO --- .../transform/solve/skel_from_3d_landmarks.py | 127 +++++++++++++----- 1 file changed, 91 insertions(+), 36 deletions(-) diff --git a/ovtk_track/transform/solve/skel_from_3d_landmarks.py b/ovtk_track/transform/solve/skel_from_3d_landmarks.py index 67be3bd..8eabbf0 100644 --- a/ovtk_track/transform/solve/skel_from_3d_landmarks.py +++ b/ovtk_track/transform/solve/skel_from_3d_landmarks.py @@ -1,6 +1,7 @@ import math import numpy as np +from scipy.spatial.distance import cdist from .. import TransformProcess from ovtk_track.types import Quaternion, Point3d @@ -24,9 +25,96 @@ class Process(TransformProcess): self.normal = np.array(normal, dtype=float) self.up = np.array(vec_perp(normal), dtype=float) + # REVIEW: See calc_eye. These probably need to change based on normal / up. + # Or maybe they dont and we just rotate the output quaternion? + # Ugh. The code works for now, but i no understand.... + self.SIN_LEFT_THETA = 2 * np.sin(np.pi / 2) + self.SIN_UP_THETA = np.sin(np.pi / 6) + def setup(self): pass + def calc_head(self, landmarks): + # REVIEW: This doesnt really work quite right!! look + roll arent mixing as expected + # Vector pointing from head center to nose + nose = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.NOSE | LANDMARK_TYPES.TIP]).mean(axis=0) + head_center = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.FACE | LANDMARK_TYPES.OUTLINE]).mean(axis=0) + look_vec = (nose - head_center) + look_vec /= np.linalg.norm(look_vec) + # Vector pointing left to right across the face + eye_center_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]).mean(axis=0) + eye_center_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]).mean(axis=0) + roll_vec = (eye_center_l - eye_center_r) + roll_vec /= np.linalg.norm(roll_vec) + + # Quat that rotates from normal to head center -> nose vec + look = Quaternion(np.dot(look_vec, self.normal), *np.cross(look_vec, self.normal)) + look.w += look.magnitude() + look = look.normalize() + # Quat that represents a rotation around the roll axis (i think??) + roll_angle = np.sum(roll_vec * self.up) + roll = Quaternion(math.cos(roll_angle), *(self.normal * math.sin(roll_angle))) + roll = roll.normalize() + + combo = look + roll + combo = combo.normalize() + + return combo, head_center + + def calc_eye(self, landmarks): + # Get poi + corners = np.empty((2, 2, 3), dtype=np.float32) + centers = np.empty((2, 3), dtype=np.float32) + pupils = np.empty((2, 3), dtype=np.float32) + cross_heights = np.empty((2), dtype=np.float32) + for i, side in enumerate([LANDMARK_TYPES.LEFT, LANDMARK_TYPES.RIGHT]): + # Find corners by searching for points with the largest distance from each other + # REVIEW: These *should* will always be the same points in the map - make a landmark type selector? + eye_outline = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.OUTLINE | side]) + hdist = cdist(eye_outline, eye_outline, metric='euclidean') + best_pair = np.unravel_index(hdist.argmax(), hdist.shape) + corners[i] = eye_outline[best_pair[0]], eye_outline[best_pair[1]] + # Get height of eye (relative to a line passing through each corner) + cross_heights[i] = np.array([ + np.linalg.norm(np.cross(corners[i][1]-corners[i][0], + corners[i][0]-point)) + / np.linalg.norm(corners[i][0]-corners[i][1]) + for point in eye_outline + ]).max() + + centers[i] = eye_outline.mean(axis=0) + pupils[i] = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | side]).mean(axis=0) + # Calculate important distances based on POI + eye_length = np.linalg.norm(np.diff(corners, axis=1), axis=(2, 1)) + ic_distance = np.linalg.norm(pupils - centers, axis=1) + zc_distance = np.linalg.norm(pupils - corners[:, 1], axis=1) + aspect_ratio = 1 / (cross_heights / eye_length) + + # Takes above and spits out spherical coordiates of pupil (relative to camera) + # Black magic as far as i can comprehend + # Copied in large part from https://github.com/1996scarlet/OpenVtuber/blob/970229d3a5ebe14a7519352da039d00a0b87e2d9/service/TFLiteIrisLocalization.py#L101 + s0 = (corners[1, :, 1] - corners[0, :, 1]) * pupils[:, 0] + s1 = (corners[1, :, 0] - corners[0, :, 0]) * pupils[:, 1] + s2 = corners[1, :, 0] * corners[0, :, 1] + s3 = corners[1, :, 1] * corners[0, :, 0] + + delta_y = (s0 - s1 + s2 - s3) / eye_length / 2 + delta_x = np.sqrt(abs(ic_distance**2 - delta_y**2)) + delta = np.array((delta_x * self.SIN_LEFT_THETA, + delta_y * self.SIN_UP_THETA)) + delta /= eye_length + theta, pha = np.arcsin(delta) + inv_judge = zc_distance**2 - delta_y**2 < eye_length**2 / 4 + theta[inv_judge] *= -1 + + # Convert spherical coordiates to quaternions + # Based on https://github.com/moble/quaternion/blob/8f6fc306306c45f0bf79331a22ef3998e4d187bc/src/quaternion/__init__.py#L599 + quats = np.array([np.cos(pha/2) * np.cos(theta/2), + np.sin(pha/2) * np.cos(theta/2), + np.cos(pha/2) * np.sin(theta/2), + -np.sin(pha/2) * np.sin(theta/2)]).T + return [Quaternion(*quat) for quat in quats], aspect_ratio + def process(self): landmarks = self._inputs['landmarks'].get() skeleton = None @@ -34,43 +122,10 @@ class Process(TransformProcess): joints = {} if landmarks.has(LANDMARK_TYPES.FACE): # Get head look / pos - nose = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.NOSE | LANDMARK_TYPES.TIP]).mean(0) - head_center = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.FACE | LANDMARK_TYPES.OUTLINE]).mean(0) - look_vec = (nose - head_center) + look_quat, head_pos = self.calc_head(landmarks) + eye_quats, eye_aspect = self.calc_eye(landmarks) - eye_center_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]).mean(0) - eye_center_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]).mean(0) - roll_vec = (eye_center_l - eye_center_r) - - look_vec /= np.linalg.norm(look_vec) - roll_vec /= np.linalg.norm(roll_vec) - - roll_angle = np.sum(roll_vec * self.up) - roll = Quaternion(math.cos(roll_angle), * self.normal * math.sin(roll_angle)) - roll = roll.normalize() - - look = Quaternion(np.dot(look_vec, self.normal), *np.cross(look_vec, self.normal)) - look.w += look.magnitude() - look = look.normalize() - - combo = look + roll - combo = combo.normalize() - - # Get eye data - marks_left = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]) - marks_right = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]) - range = np.array([marks_left.max(axis=0) - marks_left.min(axis=0), - marks_right.max(axis=0) - marks_right.min(axis=0)]) - delta = np.array([eye_center_l - Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | LANDMARK_TYPES.CENTER | LANDMARK_TYPES.LEFT]).mean(0), - eye_center_r - Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | LANDMARK_TYPES.CENTER | LANDMARK_TYPES.RIGHT]).mean(0)]) - - delta /= range - try: - eye_aspect_ratio = range[::, 0] / range[::, 1] - except ZeroDivisionError: - eye_aspect_ratio = None - - head_joint = Joint(Point3d(*head_center), combo, dict(look_delta=delta, eye_aspect_ratio=eye_aspect_ratio)) + head_joint = Joint(Point3d(*head_pos), look_quat, attr=dict(eye_rot=eye_quats, eye_aspect=eye_aspect)) joints[JOINT_TYPES.HEAD] = head_joint -- 2.45.1 From cb137a1a066d578ac1c98179bc8f4d04c7b0f98b Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Thu, 6 Jan 2022 08:49:20 -0700 Subject: [PATCH 4/8] Disclaimer The people need to know --- ovtk_track/__init__.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/ovtk_track/__init__.py b/ovtk_track/__init__.py index e69de29..682e375 100644 --- a/ovtk_track/__init__.py +++ b/ovtk_track/__init__.py @@ -0,0 +1,42 @@ +''' +█████████████████████████████████████████████▀▀▀▀▀▀▀▀▀▀█████████████████████████████████████▀▄██████████████████████████ +█████████████████████████████████████████▀▀«─ⁿ^^^╜╜╜╝╝╣╬æ░█████████████████████████████████▀▄███████████████████████████ +███████████████████████████████████████████████████████▄▄╙╗▀█▀▀▀▀▀▀▀█████████████████████▀▄█████████████████████████████ +███████████████████████████████████████████████▀▀¼@╣╬╢╣╣æ@@╬ÑÑÑÑD╬ÑÑ╬╣æ╗╬▀▀█████████████▀▄██████████████████████████████ +████████████████████████████████████████████▀╔╢ÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD╬╖╬▀█████████▀████████████████████████████████ +██████████████████████████████████████████▀é]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ╣φ▀█████▌▄████████████████████████████████ +████████████████████████████████████████▀╗▓▓]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ]▓▓▓Ö▀██▐█████████████████████████████████ +██████████████████████████████████████▀╓▓▓▓▓▓▓▓]]DÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD╫▓▓▓▓▓▓▓u ██████████████████████████████████ +█████████████████████████████████████╓▓▓▀ÑÑ]Ñ▓▓▀▀▀ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ]D╫▓▓▓▓▓▓▀▓▓▓▐██████████████████████████████████ +███████████████████████████████████▀╔Ñ]ÑÑÑÑÑÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD]ÑÑDÑÑÑÑ]]ÑÑÑM██████████████▀▀▀▀▀▀▀▀▀▀▀▀▀▀███████ +██████████████████████████████████▌║ÑÑÑÑÑÑÑÑ╬M░╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑh██████████████ oh, here's a ███████ +█████████████████████████████████▌╓ÑÑÑD╝╢Ñ╬½╗╬╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ╬╟DÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑh██████████████ a fun fact ███████ +█████████████████████████████████.ÑÑÑDÑ╙▓▄╓╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑU░╟ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┐██████████████▄▄▄▄▄▄▄▄▄▄▄▄▄▄███████ +████████████████████████████████⌐╣ÑÑÑ╓µ╝M╔╣Ñ╣'╬Ñ╬╫ÑÑÑÑÑÑ╬╬ÑÑÑÑÑÑDÑÑ╙╝╜,▄╕╢ÑÑÑÑÑÑÑÑÑÑ╬▐██████████████████████████████████ +████████████████████████████████║ÑÑÑÑæÖ▀ Ñ╬Ω▄LÑDM║ÑÑÑÑÑÑM╟ÑÑÑÑÑÑÑÑφΦ▀▀╠╔╬╬ÑÑÑÑÑÑÑÑÑÑÑ ██████████████████████████████████ +██████████████████████████████▀║ÑÑÑÑÑÑ╬ ╫M▄▓▄ ║D ╚DÑÑÑÑÑu║ÑÑMÑDÑÑÑÑ╬M`▄▄φΦ║ÑÑÑÑÑÑÑÑÑÑ╖▐█████████████████████████████████ +█████████████████████████████▌╓ÑÑÑÑÑÑDM┌ ▓▓▓▓L╢╣M ╬ÑÑÑÑÑ └Ñ╬~²╣ÑÑÑÑѼ▀'╔æ╢╬ÑÑÑÑÑÑÑÑÑÑÑ┐▀████████████████████████████████ +█████████████████████████████╔ÑM╬ÑÑÑÑÑù╓▀"`` "²ⁿ▌║ÑÑÑÑÑhL╢Ñ∩µ╢m─ªª╣╣M½║DÑÑÑÑÑÑÑÑÑÑÑÑÑÑ,████████████████████████████████ +████████████████████████████ ╬^║ÑÑÑÑÑÑM▓H ,╗h ▓ ÑÑÑÑÑh▓ ÑM ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ ███████████████████████████████ +███████████████████████████▌╣ Γ╣ÑÑÑÑÑDM║ ╢╬ ╫▌▄▄▄▄▄▄▓▓▄▄ '╬▓w ÑÑÑÑÑÑÑÑÑÑÑÑÑD╬`▄███████████████████████████████ +███████████████████████████ █ ╬ÑÑÑÑÑÑÑ▐▌ ╬▀▀ ║██████████▄, ╬╬╬ ÑÑÑÑÑÑÑÑÑÑÑÑ╣┘-▀5╔╢ ▀███████████████████████████ +███████████████████████████╘▐█ ÑÑÑÑÑÑÑÑ▐█▄██ ███████████▌ ▓▓▓ ▄ DÑÑÑÑÑÑÑÑÑÑ╖╖æ╣ÑÑÑÑHß ▀█████████████████████████ +███████████████████████████ ▐█ ÑÑ╬ÑÑÑÑÑ▐█▓██▓ ¥╜▄████████████ Φæ┘▄████▌╢ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ^╞▐█▄▄▀████████████████████▀▄ +███████▀█▄▄████████▄▄▄▀▀████▐█ ╬╬░╬ÑÑÑÑ▐▓╣▓▓▓▓████████████████▄▄██████ ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┘┤|█████▄▄▄▀▀██████████▀▀▄▄███ +███▀▄▄██████████████████▄▄▀███▌║ÑM╢ÑÑÑÑJ█▓▓▓███████████████████▓▓▓▓▓▓`╣ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┘▄M,████████████▄▄▄▄▄██████████ +█▀▄████████████████████████▄▀██µ╢┼,╢ÑÑÑD╣╬▀▀███████▄╬╬D█████████████▀║DÑÑÑÑÑÑÑÑÑÑDM╫DÑÑ╬ ██▄████████████████████████████ +▄████████████████████████████▄▀█ ╬▐█╙╬ÑÑÑÑ▐▄ "▀█████▄╢M███████████▀▀║ÑÑÑÑÑÑÑÑ╬╟ÑÑ╬ⁿ╬ÑÑ╝╓████████████████████████████████ +███████████████████████████████▄▀ ██▄░╢ÑÑM██ ▄`"▀▀▀▀▀▀▀▀"` . ▄▄╢ÑÑÑM░╣╜║╬ÑM▄█║Ñ^▄█████████████████████████████████ +████████████████████████████████▄▀▄▀████▄╙╢▐███▄▐██▄ .⌡,#╓██ ╣╣╜}-,▄▌╬M▄███w▄███████████████████████████████████ +█████████████████████████████████▌▀▌▀▀▀▀,██▄████████▀7 ╓▓▌╙▀└██▌▐███████²▄██████████████████████████████████████████ +████████████████████████████████████▀,▄█████████▀▀«]░╓▓▓▓▓▓▓▀«]░J██ ████████████████████████████████████████████████████ +███████████████████████████████████▐███████▀▀µj]░ ░ █╣▓▓▓▀ j░ ░▐██ ▄Å▀▀████████████████████████████████████████████████ +█████████▀▀▀▀▀▀▀▀▀▀▀▀▀▀████████████ ██████┌]▌║ ░ r▐█▀██▀███▄`y███▐█▀█▌▄▀██████████████████████████████████████████████ +█████████ i do not ████████████▌████▀▄▓ ╙▄▓` ▓▓▓▓⌐jr╙████▄▀▌██▓█▓▌▐██████████████████████████████████████████████ +█████████ think ████████████▌███▀▄▓▓╬ ╙▄▀▄░░ ▓▓▓▀j░ H`▀▀▀▀""` ]«]«¥██████████████████████████████████████████████ +█████████▄▄▄▄▄▄▄▄▄▄▄▄▄▄████████████▌██▌▐▓▓▓▓L,╨¥"Vµ ▓╣▌j░░U ░ ░ ░,µ⌐Ö║██████████████████████████████████████████████ +███████████████████████████████████ ██,▓▓▓▓▓ ░½`"▀▄ ▌▌/δ▀DxH*¥#==═K*"▄x╨░d██████████████████████████████████████████████ +███████████████████████████████████▐█`╣▓▓▓▓▓µ░░░^ ` ╚ "░ ,«⌂]"M╨*"""^▄▄▓╙▓▐█████████████████████████████████████████████ +██████████████████████████████████ ██▐▌║▓▓▓▓▌▄▄D,╔,ⁿ«-░░░,⌂░^╓ó],▄▓█▓▓▓▓▌Γ██████████████████████████████████████████████ +█████████████████████████████████▀██ ▓▓Ω▓▓▓▓▓▓▀]░`j ]j⌂,"]H╓⌐▓▓▓▓▓▓▓▓▓▓▓▓µ▐█████████████████████████████████████████████ +''' -- 2.45.1 From 66c5ec59ce6d7c51e035617f45b781615dfe4ae6 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Sat, 13 Aug 2022 14:07:01 -0400 Subject: [PATCH 5/8] Add draw filter --- ovtk_track/types/Landmarks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ovtk_track/types/Landmarks.py b/ovtk_track/types/Landmarks.py index 268ffdc..d7e3fef 100644 --- a/ovtk_track/types/Landmarks.py +++ b/ovtk_track/types/Landmarks.py @@ -71,8 +71,9 @@ class Landmarks(Type): return False - def draw(self, image, canvas, color=(255, 255, 255), label=True): - for i, (x, y, z) in enumerate(point.project_to_image(image) for point in self.points): + def draw(self, image, canvas, color=(255, 255, 255), label=True, filter=None): + points = self[filter] if filter else self.points + for i, (x, y, z) in enumerate(point.project_to_image(image) for point in points): if x > image.width or x < 0 or y > image.height or y < 0: continue -- 2.45.1 From 93b8c755ca0b4124340664309ce724fbf1d4bcfb Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 27 Feb 2023 16:25:14 -0500 Subject: [PATCH 6/8] Fix head rotation --- ovtk_track/transform/solve/skel_from_3d_landmarks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ovtk_track/transform/solve/skel_from_3d_landmarks.py b/ovtk_track/transform/solve/skel_from_3d_landmarks.py index 8eabbf0..e059289 100644 --- a/ovtk_track/transform/solve/skel_from_3d_landmarks.py +++ b/ovtk_track/transform/solve/skel_from_3d_landmarks.py @@ -110,10 +110,11 @@ class Process(TransformProcess): # Convert spherical coordiates to quaternions # Based on https://github.com/moble/quaternion/blob/8f6fc306306c45f0bf79331a22ef3998e4d187bc/src/quaternion/__init__.py#L599 quats = np.array([np.cos(pha/2) * np.cos(theta/2), - np.sin(pha/2) * np.cos(theta/2), + -np.sin(pha/2) * np.sin(theta/2), np.cos(pha/2) * np.sin(theta/2), - -np.sin(pha/2) * np.sin(theta/2)]).T - return [Quaternion(*quat) for quat in quats], aspect_ratio + np.sin(pha/2) * np.cos(theta/2)]).T + quat_arr = [Quaternion(*quat) for quat in quats] + return quat_arr, aspect_ratio def process(self): landmarks = self._inputs['landmarks'].get() -- 2.45.1 From dfc683ff8a24e451b5ce0100ab22e44aed4504c1 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 27 Feb 2023 16:24:12 -0500 Subject: [PATCH 7/8] Add basic pose tracking to holistic model --- ovtk_track/transform/solve/mediapipe/__init__.py | 14 ++++++++++++++ ovtk_track/transform/solve/mediapipe/holistic.py | 13 ++++++++++++- ovtk_track/types/Landmarks.py | 11 +++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/ovtk_track/transform/solve/mediapipe/__init__.py b/ovtk_track/transform/solve/mediapipe/__init__.py index 293a314..03e9648 100644 --- a/ovtk_track/transform/solve/mediapipe/__init__.py +++ b/ovtk_track/transform/solve/mediapipe/__init__.py @@ -32,6 +32,20 @@ face_mesh_map = { ], } +body_map = { + LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.LEFT: [11], + LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.RIGHT: [12], + + LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.LEFT: [13], + LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.RIGHT: [14], + + LANDMARK_TYPES.HIP | LANDMARK_TYPES.LEFT: [23], + LANDMARK_TYPES.HIP | LANDMARK_TYPES.RIGHT: [24], + + LANDMARK_TYPES.KNEE | LANDMARK_TYPES.LEFT: [25], + LANDMARK_TYPES.KNEE | LANDMARK_TYPES.RIGHT: [26], +} + # SEE YEAH THESE MAKE SENSE GOOGLE WHAT THE HELL hand_mesh_map = {LANDMARK_TYPES.HAND | LANDMARK_TYPES.WRIST: [0]} _finger_map = { diff --git a/ovtk_track/transform/solve/mediapipe/holistic.py b/ovtk_track/transform/solve/mediapipe/holistic.py index 7c87f85..ed88039 100644 --- a/ovtk_track/transform/solve/mediapipe/holistic.py +++ b/ovtk_track/transform/solve/mediapipe/holistic.py @@ -2,7 +2,7 @@ import mediapipe import numpy as np from ovtk_track.transform import TransformProcess -from ovtk_track.transform.solve.mediapipe import face_mesh_map, hand_mesh_map +from ovtk_track.transform.solve.mediapipe import face_mesh_map, hand_mesh_map, body_map from ovtk_track import types from ovtk_track.types.Landmarks import LANDMARK_TYPES @@ -83,6 +83,17 @@ class Process(TransformProcess): available.append((right_hand_landmarks, mix_maps(hand_mesh_map, LANDMARK_TYPES.RIGHT))) + if results.pose_landmarks: + raw_landmarks = results.pose_landmarks.landmark + body_landmarks = np.empty((33, 3), dtype=np.float32) + + for i in range(33): + body_landmarks[i][0] = raw_landmarks[i].x + body_landmarks[i][1] = raw_landmarks[i].y + body_landmarks[i][2] = raw_landmarks[i].z + + available.append((body_landmarks, body_map)) + if available: avail_landmarks, maps = zip(*available) combo_map = combine_maps(zip(maps, (array.shape[0] for array in avail_landmarks))) diff --git a/ovtk_track/types/Landmarks.py b/ovtk_track/types/Landmarks.py index d7e3fef..8e95ceb 100644 --- a/ovtk_track/types/Landmarks.py +++ b/ovtk_track/types/Landmarks.py @@ -17,6 +17,17 @@ class LANDMARK_TYPES(Flag): LIPS = auto() CHIN = auto() + # Body + SHOULDER = auto() + ELBOW = auto() + HIP = auto() + KNEE = auto() + + # Feet tracking lmao + ANKLE = auto() + HEEL = auto() + TOE_INDEX = auto() + # Hand HAND = auto() WRIST = auto() -- 2.45.1 From c0e733e69ac3bf0228b5a26615593a1038ccee40 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 27 Feb 2023 16:24:48 -0500 Subject: [PATCH 8/8] Add support for (basic) pose tracking in skel solver --- .../transform/solve/mediapipe/__init__.py | 6 ++++ .../transform/solve/skel_from_3d_landmarks.py | 34 ++++++++++++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/ovtk_track/transform/solve/mediapipe/__init__.py b/ovtk_track/transform/solve/mediapipe/__init__.py index 03e9648..ca8376c 100644 --- a/ovtk_track/transform/solve/mediapipe/__init__.py +++ b/ovtk_track/transform/solve/mediapipe/__init__.py @@ -44,6 +44,12 @@ body_map = { LANDMARK_TYPES.KNEE | LANDMARK_TYPES.LEFT: [25], LANDMARK_TYPES.KNEE | LANDMARK_TYPES.RIGHT: [26], + + LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.LEFT: [27], + LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.RIGHT: [28], + + LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.LEFT: [27], + LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.RIGHT: [28], } # SEE YEAH THESE MAKE SENSE GOOGLE WHAT THE HELL diff --git a/ovtk_track/transform/solve/skel_from_3d_landmarks.py b/ovtk_track/transform/solve/skel_from_3d_landmarks.py index e059289..61d245d 100644 --- a/ovtk_track/transform/solve/skel_from_3d_landmarks.py +++ b/ovtk_track/transform/solve/skel_from_3d_landmarks.py @@ -130,16 +130,34 @@ class Process(TransformProcess): joints[JOINT_TYPES.HEAD] = head_joint + if landmarks.has(LANDMARK_TYPES.SHOULDER): + shoulder_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.LEFT]).mean(axis=0) + shoulder_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.RIGHT]).mean(axis=0) + joints[JOINT_TYPES.SHOULDER_L] = Joint(Point3d(*shoulder_l), Quaternion.identity()) + joints[JOINT_TYPES.SHOULDER_R] = Joint(Point3d(*shoulder_r), Quaternion.identity()) + + if landmarks.has(LANDMARK_TYPES.ELBOW): + elbow_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.LEFT]).mean(axis=0) + elbow_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.RIGHT]).mean(axis=0) + joints[JOINT_TYPES.ELBOW_L] = Joint(Point3d(*elbow_l), Quaternion.identity()) + joints[JOINT_TYPES.ELBOW_R] = Joint(Point3d(*elbow_r), Quaternion.identity()) + + if landmarks.has(LANDMARK_TYPES.HIP): + hips = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.HIP]).mean(axis=0) + joints[JOINT_TYPES.HIPS] = Joint(Point3d(*hips), Quaternion.identity()) + # Synthizise other joints from existing data - if not joints.get(JOINT_TYPES.CHEST) and joints.get(JOINT_TYPES.HEAD): - chest_center = joints[JOINT_TYPES.HEAD].pos.as_np() - chest_center = np.power(chest_center, 3) / (1e3 + np.power(chest_center, 2)) - chest_center -= [0, 100, 0] + if not joints.get(JOINT_TYPES.CHEST): + if landmarks.has(LANDMARK_TYPES.SHOULDER) and landmarks.has(LANDMARK_TYPES.HIP): + chest = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER, LANDMARK_TYPES.HIP]).mean(axis=0) + joints[JOINT_TYPES.CHEST] = Joint(Point3d(*chest), Quaternion.identity()) + elif joints.get(JOINT_TYPES.HEAD): + chest_center = joints[JOINT_TYPES.HEAD].pos.as_np() + chest_center = np.power(chest_center, 3) / (1e3 + np.power(chest_center, 2)) + chest_center -= [0, 100, 0] + chest_rot = Quaternion.identity().slerp(joints[JOINT_TYPES.HEAD].rot, 0.1) - chest_rot = Quaternion.identity().slerp(joints[JOINT_TYPES.HEAD].rot, 0.1) - - chest_joint = Joint(Point3d(*chest_center), chest_rot) - joints[JOINT_TYPES.CHEST] = chest_joint + joints[JOINT_TYPES.CHEST] = Joint(Point3d(*chest_center), chest_rot) skeleton = Skeleton(joints) -- 2.45.1