Compare commits

..

8 Commits

Author SHA1 Message Date
Derek c0e733e69a Add support for (basic) pose tracking in skel solver 2023-02-27 16:27:32 -05:00
Derek dfc683ff8a Add basic pose tracking to holistic model 2023-02-27 16:27:32 -05:00
Derek 93b8c755ca Fix head rotation 2023-02-27 16:27:32 -05:00
Derek 66c5ec59ce Add draw filter 2022-08-13 14:07:38 -04:00
Derek cb137a1a06 Disclaimer
The people need to know
2022-01-06 08:49:20 -07:00
Derek 4ed2ac1384 THATS RIGHT BINCH WE GOT PROPER EYE TRACKING WOOOO 2021-12-30 12:11:05 -07:00
Derek 4beddc5e6d Chest default 2021-11-28 15:28:15 -07:00
Derek 92044fa866 Add better skeleton debug view 2021-11-28 13:20:04 -07:00
8 changed files with 251 additions and 56 deletions

View File

@ -0,0 +1,42 @@
'''
«^^^æ
¼@æ@@ÑÑÑÑDÑÑæ
ÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD
é]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑφ
]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ]Ö
]]DÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑDu
ÑÑ]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ]D
Ñ]ÑÑÑÑÑÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD]ÑÑDÑÑÑÑ]]ÑÑÑM
ÑÑÑÑÑÑÑÑMÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑh oh, here's a ███████
ÑÑÑDѽÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑh a fun fact
.ÑÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑUÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ
ÑÑѵMÑ'╬Ñ╬╫ÑÑÑÑÑÑ╬╬ÑÑÑÑÑÑDÑÑ╙╝╜,▄╕╢ÑÑÑÑÑÑÑÑÑÑ╬▐██████████████████████████████████
ÑÑÑÑæÖ ÑΩLÑDMÑÑÑÑÑÑMÑÑÑÑÑÑÑÑφΦÑÑÑÑÑÑÑÑÑÑÑ
ÑÑÑÑÑÑ M D DÑÑÑÑÑuÑÑMÑDÑÑÑÑM`φΦÑÑÑÑÑÑÑÑÑÑ
ÑÑÑÑÑÑDM LM ÑÑÑÑÑ Ñ~²ÑÑÑÑѼ'╔æ╢╬ÑÑÑÑÑÑÑÑÑÑÑ┐▀████████████████████████████████
ÑMÑÑÑÑÑù"`` "²ÑÑÑÑÑhLѵmªªDÑÑÑÑÑÑÑÑÑÑÑÑÑÑ,
^ÑÑÑÑÑÑMH ,h ÑÑÑÑÑh ÑM ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ
ΓÑÑÑÑÑDM '╬▓w ÑÑÑÑÑÑÑÑÑÑÑÑÑD╬`▄███████████████████████████████
ÑÑÑÑÑÑÑ , ÑÑÑÑÑÑÑÑÑÑÑÑ-5
ÑÑÑÑÑÑÑÑ DÑÑÑÑÑÑÑÑÑÑæÑÑÑÑHß
ÑÑÑÑÑÑÑ ¥ ΦæÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ^
ÑÑÑÑ ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ|
ÑMÑÑÑÑJ`ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑM,
µ,ÑÑÑDDDÑÑÑÑÑÑÑÑÑÑDMDÑÑ
ÑÑÑÑ "▀█████▄╢M███████████▀▀║ÑÑÑÑÑÑÑÑ╬╟ÑÑ╬ⁿ╬ÑÑ╝╓████████████████████████████████
ÑÑM `"▀▀▀▀▀▀▀▀"` . ÑÑÑMÑMÑ^
.,#╓██ ╣╣╜}-,▄▌╬M▄███w▄███████████████████████████████████
,7 ²
,«]«]J
µj] j Å
] r`y
i do not ` jr
think j H`""` ]«]«¥
L,¥"Vµ ▓╣▌j░░U ░ ░ ░,µ⌐Ö║██████████████████████████████████████████████
, ½`"▀▄ ▌▌/δ▀DxH*¥#==═K*"xd
`µ^ ` "░ ,«⌂]"M*"""^▄▄▓╙▓▐█████████████████████████████████████████████
D,,«-,^ó],Γ
Ω]`j ]j,"]H╓⌐▓▓▓▓▓▓▓▓▓▓▓▓µ▐█████████████████████████████████████████████
'''

View File

@ -1,4 +1,5 @@
import cv2
import matplotlib.pyplot as plt
from . import OutputProcess
from ovtk_track import types
@ -14,7 +15,10 @@ class Process(OutputProcess):
super().__init__(*args)
def setup(self):
pass
self.fig = plt.figure()
self.axes = self.fig.add_subplot(projection='3d')
self.axes.view_init(15, 0, vertical_axis='y')
plt.show(block=False)
def send(self):
landmarks = self._inputs['landmarks'].get_nowait()
@ -31,8 +35,15 @@ class Process(OutputProcess):
landmarks.draw(image, frame, label=False, color=(130, 130, 130))
if skeleton is not None:
skeleton.draw(image, frame)
skeleton.draw(self.axes)
cv2.imshow("face", frame)
plt.draw()
# event loops
plt.pause(0.0001)
if cv2.waitKey(1) & 0xFF == ord('q'):
raise KeyboardInterrupt('User requested stop')
for artist in plt.gca().lines + plt.gca().collections:
artist.remove()

View File

@ -32,6 +32,26 @@ face_mesh_map = {
],
}
body_map = {
LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.LEFT: [11],
LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.RIGHT: [12],
LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.LEFT: [13],
LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.RIGHT: [14],
LANDMARK_TYPES.HIP | LANDMARK_TYPES.LEFT: [23],
LANDMARK_TYPES.HIP | LANDMARK_TYPES.RIGHT: [24],
LANDMARK_TYPES.KNEE | LANDMARK_TYPES.LEFT: [25],
LANDMARK_TYPES.KNEE | LANDMARK_TYPES.RIGHT: [26],
LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.LEFT: [27],
LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.RIGHT: [28],
LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.LEFT: [27],
LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.RIGHT: [28],
}
# SEE YEAH THESE MAKE SENSE GOOGLE WHAT THE HELL
hand_mesh_map = {LANDMARK_TYPES.HAND | LANDMARK_TYPES.WRIST: [0]}
_finger_map = {

View File

@ -2,7 +2,7 @@ import mediapipe
import numpy as np
from ovtk_track.transform import TransformProcess
from ovtk_track.transform.solve.mediapipe import face_mesh_map, hand_mesh_map
from ovtk_track.transform.solve.mediapipe import face_mesh_map, hand_mesh_map, body_map
from ovtk_track import types
from ovtk_track.types.Landmarks import LANDMARK_TYPES
@ -83,6 +83,17 @@ class Process(TransformProcess):
available.append((right_hand_landmarks, mix_maps(hand_mesh_map, LANDMARK_TYPES.RIGHT)))
if results.pose_landmarks:
raw_landmarks = results.pose_landmarks.landmark
body_landmarks = np.empty((33, 3), dtype=np.float32)
for i in range(33):
body_landmarks[i][0] = raw_landmarks[i].x
body_landmarks[i][1] = raw_landmarks[i].y
body_landmarks[i][2] = raw_landmarks[i].z
available.append((body_landmarks, body_map))
if available:
avail_landmarks, maps = zip(*available)
combo_map = combine_maps(zip(maps, (array.shape[0] for array in avail_landmarks)))

View File

@ -1,6 +1,7 @@
import math
import numpy as np
from scipy.spatial.distance import cdist
from .. import TransformProcess
from ovtk_track.types import Quaternion, Point3d
@ -24,9 +25,97 @@ class Process(TransformProcess):
self.normal = np.array(normal, dtype=float)
self.up = np.array(vec_perp(normal), dtype=float)
# REVIEW: See calc_eye. These probably need to change based on normal / up.
# Or maybe they dont and we just rotate the output quaternion?
# Ugh. The code works for now, but i no understand....
self.SIN_LEFT_THETA = 2 * np.sin(np.pi / 2)
self.SIN_UP_THETA = np.sin(np.pi / 6)
def setup(self):
pass
def calc_head(self, landmarks):
# REVIEW: This doesnt really work quite right!! look + roll arent mixing as expected
# Vector pointing from head center to nose
nose = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.NOSE | LANDMARK_TYPES.TIP]).mean(axis=0)
head_center = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.FACE | LANDMARK_TYPES.OUTLINE]).mean(axis=0)
look_vec = (nose - head_center)
look_vec /= np.linalg.norm(look_vec)
# Vector pointing left to right across the face
eye_center_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]).mean(axis=0)
eye_center_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]).mean(axis=0)
roll_vec = (eye_center_l - eye_center_r)
roll_vec /= np.linalg.norm(roll_vec)
# Quat that rotates from normal to head center -> nose vec
look = Quaternion(np.dot(look_vec, self.normal), *np.cross(look_vec, self.normal))
look.w += look.magnitude()
look = look.normalize()
# Quat that represents a rotation around the roll axis (i think??)
roll_angle = np.sum(roll_vec * self.up)
roll = Quaternion(math.cos(roll_angle), *(self.normal * math.sin(roll_angle)))
roll = roll.normalize()
combo = look + roll
combo = combo.normalize()
return combo, head_center
def calc_eye(self, landmarks):
# Get poi
corners = np.empty((2, 2, 3), dtype=np.float32)
centers = np.empty((2, 3), dtype=np.float32)
pupils = np.empty((2, 3), dtype=np.float32)
cross_heights = np.empty((2), dtype=np.float32)
for i, side in enumerate([LANDMARK_TYPES.LEFT, LANDMARK_TYPES.RIGHT]):
# Find corners by searching for points with the largest distance from each other
# REVIEW: These *should* will always be the same points in the map - make a landmark type selector?
eye_outline = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.OUTLINE | side])
hdist = cdist(eye_outline, eye_outline, metric='euclidean')
best_pair = np.unravel_index(hdist.argmax(), hdist.shape)
corners[i] = eye_outline[best_pair[0]], eye_outline[best_pair[1]]
# Get height of eye (relative to a line passing through each corner)
cross_heights[i] = np.array([
np.linalg.norm(np.cross(corners[i][1]-corners[i][0],
corners[i][0]-point))
/ np.linalg.norm(corners[i][0]-corners[i][1])
for point in eye_outline
]).max()
centers[i] = eye_outline.mean(axis=0)
pupils[i] = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | side]).mean(axis=0)
# Calculate important distances based on POI
eye_length = np.linalg.norm(np.diff(corners, axis=1), axis=(2, 1))
ic_distance = np.linalg.norm(pupils - centers, axis=1)
zc_distance = np.linalg.norm(pupils - corners[:, 1], axis=1)
aspect_ratio = 1 / (cross_heights / eye_length)
# Takes above and spits out spherical coordiates of pupil (relative to camera)
# Black magic as far as i can comprehend
# Copied in large part from https://github.com/1996scarlet/OpenVtuber/blob/970229d3a5ebe14a7519352da039d00a0b87e2d9/service/TFLiteIrisLocalization.py#L101
s0 = (corners[1, :, 1] - corners[0, :, 1]) * pupils[:, 0]
s1 = (corners[1, :, 0] - corners[0, :, 0]) * pupils[:, 1]
s2 = corners[1, :, 0] * corners[0, :, 1]
s3 = corners[1, :, 1] * corners[0, :, 0]
delta_y = (s0 - s1 + s2 - s3) / eye_length / 2
delta_x = np.sqrt(abs(ic_distance**2 - delta_y**2))
delta = np.array((delta_x * self.SIN_LEFT_THETA,
delta_y * self.SIN_UP_THETA))
delta /= eye_length
theta, pha = np.arcsin(delta)
inv_judge = zc_distance**2 - delta_y**2 < eye_length**2 / 4
theta[inv_judge] *= -1
# Convert spherical coordiates to quaternions
# Based on https://github.com/moble/quaternion/blob/8f6fc306306c45f0bf79331a22ef3998e4d187bc/src/quaternion/__init__.py#L599
quats = np.array([np.cos(pha/2) * np.cos(theta/2),
-np.sin(pha/2) * np.sin(theta/2),
np.cos(pha/2) * np.sin(theta/2),
np.sin(pha/2) * np.cos(theta/2)]).T
quat_arr = [Quaternion(*quat) for quat in quats]
return quat_arr, aspect_ratio
def process(self):
landmarks = self._inputs['landmarks'].get()
skeleton = None
@ -34,46 +123,42 @@ class Process(TransformProcess):
joints = {}
if landmarks.has(LANDMARK_TYPES.FACE):
# Get head look / pos
nose = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.NOSE | LANDMARK_TYPES.TIP]).mean(0)
head_center = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.FACE | LANDMARK_TYPES.OUTLINE]).mean(0)
look_vec = (nose - head_center)
look_quat, head_pos = self.calc_head(landmarks)
eye_quats, eye_aspect = self.calc_eye(landmarks)
eye_center_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]).mean(0)
eye_center_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]).mean(0)
roll_vec = (eye_center_l - eye_center_r)
look_vec /= np.linalg.norm(look_vec)
roll_vec /= np.linalg.norm(roll_vec)
roll_angle = np.sum(roll_vec * self.up)
roll = Quaternion(math.cos(roll_angle), * self.normal * math.sin(roll_angle))
roll = roll.normalize()
look = Quaternion(np.dot(look_vec, self.normal), *np.cross(look_vec, self.normal))
look.w += look.magnitude()
look = look.normalize()
combo = look + roll
combo = combo.normalize()
# Get eye data
marks_left = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT])
marks_right = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT])
range = np.array([marks_left.max(axis=0) - marks_left.min(axis=0),
marks_right.max(axis=0) - marks_right.min(axis=0)])
delta = np.array([eye_center_l - Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | LANDMARK_TYPES.CENTER | LANDMARK_TYPES.LEFT]).mean(0),
eye_center_r - Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | LANDMARK_TYPES.CENTER | LANDMARK_TYPES.RIGHT]).mean(0)])
delta /= range
try:
eye_aspect_ratio = range[::, 0] / range[::, 1]
except ZeroDivisionError:
eye_aspect_ratio = None
head_joint = Joint(Point3d(*head_center), combo, dict(look_delta=delta, eye_aspect_ratio=eye_aspect_ratio))
head_joint = Joint(Point3d(*head_pos), look_quat, attr=dict(eye_rot=eye_quats, eye_aspect=eye_aspect))
joints[JOINT_TYPES.HEAD] = head_joint
if landmarks.has(LANDMARK_TYPES.SHOULDER):
shoulder_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.LEFT]).mean(axis=0)
shoulder_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.RIGHT]).mean(axis=0)
joints[JOINT_TYPES.SHOULDER_L] = Joint(Point3d(*shoulder_l), Quaternion.identity())
joints[JOINT_TYPES.SHOULDER_R] = Joint(Point3d(*shoulder_r), Quaternion.identity())
if landmarks.has(LANDMARK_TYPES.ELBOW):
elbow_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.LEFT]).mean(axis=0)
elbow_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.RIGHT]).mean(axis=0)
joints[JOINT_TYPES.ELBOW_L] = Joint(Point3d(*elbow_l), Quaternion.identity())
joints[JOINT_TYPES.ELBOW_R] = Joint(Point3d(*elbow_r), Quaternion.identity())
if landmarks.has(LANDMARK_TYPES.HIP):
hips = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.HIP]).mean(axis=0)
joints[JOINT_TYPES.HIPS] = Joint(Point3d(*hips), Quaternion.identity())
# Synthizise other joints from existing data
if not joints.get(JOINT_TYPES.CHEST):
if landmarks.has(LANDMARK_TYPES.SHOULDER) and landmarks.has(LANDMARK_TYPES.HIP):
chest = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER, LANDMARK_TYPES.HIP]).mean(axis=0)
joints[JOINT_TYPES.CHEST] = Joint(Point3d(*chest), Quaternion.identity())
elif joints.get(JOINT_TYPES.HEAD):
chest_center = joints[JOINT_TYPES.HEAD].pos.as_np()
chest_center = np.power(chest_center, 3) / (1e3 + np.power(chest_center, 2))
chest_center -= [0, 100, 0]
chest_rot = Quaternion.identity().slerp(joints[JOINT_TYPES.HEAD].rot, 0.1)
joints[JOINT_TYPES.CHEST] = Joint(Point3d(*chest_center), chest_rot)
skeleton = Skeleton(joints)
self._outputs['skel'].send(skeleton)

View File

@ -17,6 +17,17 @@ class LANDMARK_TYPES(Flag):
LIPS = auto()
CHIN = auto()
# Body
SHOULDER = auto()
ELBOW = auto()
HIP = auto()
KNEE = auto()
# Feet tracking lmao
ANKLE = auto()
HEEL = auto()
TOE_INDEX = auto()
# Hand
HAND = auto()
WRIST = auto()
@ -71,8 +82,9 @@ class Landmarks(Type):
return False
def draw(self, image, canvas, color=(255, 255, 255), label=True):
for i, (x, y, z) in enumerate(point.project_to_image(image) for point in self.points):
def draw(self, image, canvas, color=(255, 255, 255), label=True, filter=None):
points = self[filter] if filter else self.points
for i, (x, y, z) in enumerate(point.project_to_image(image) for point in points):
if x > image.width or x < 0 or y > image.height or y < 0:
continue

View File

@ -1,7 +1,7 @@
from dataclasses import dataclass
import numpy as np
from scipy.spatial.transform import Rotation
from scipy.spatial.transform import Rotation, Slerp
from .Type import Type
@ -12,6 +12,10 @@ class Quaternion(Type):
y: float
z: float
@classmethod
def identity(cls):
return cls(1, 0, 0, 0)
def __mul__(self, q):
if isinstance(q, self.__class__):
product = self.as_np() * q.as_np()
@ -45,6 +49,16 @@ class Quaternion(Type):
def conjugate(self):
return self.__class__(self.w, -self.x, -self.y, -self.z)
def slerp(self, other, t):
r = Rotation.from_quat([
[self.x, self.y, self.z, self.w],
[other.x, other.y, other.z, other.w],
])
slerp = Slerp([0, 1], r)
x, y, z, w = slerp([t]).as_quat()[0]
return self.__class__(w, x, y, z)
def draw(self, canvas, origin):
raise NotImplementedError()

View File

@ -2,8 +2,6 @@ from dataclasses import dataclass, field
from enum import Enum
import typing
import cv2
from .Type import Type
from .Point3d import Point3d
from .Quaternion import Quaternion
@ -12,20 +10,26 @@ from .Quaternion import Quaternion
class JOINT_TYPES(Enum):
HEAD = 'head'
CHEST = 'chest'
HIPS = 'hips'
SHOULDER_L = 'shoulder_l'
ELBOW_L = 'elbow_l'
WRIST_L = 'wrist_l'
HIP_L = 'hip_l'
KNEE_L = 'knee_l'
FOOT_L = 'foot_l'
WRIST_L = 'wrist_l'
SHOULDER_R = 'shoulder_r'
ELBOW_R = 'elbow_r'
WRIST_R = 'wrist_r'
HIP_R = 'hip_r'
KNEE_R = 'knee_r'
FOOT_R = 'foot_r'
WRIST_R = 'wrist_r'
default_colors = [
[255, 0, 0], [0, 255, 0], [0, 0, 255],
[255, 255, 0], [255, 0, 255], [0, 255, 255],
[128, 255, 0], [255, 0, 128], [0, 255, 128],
]
@dataclass
@ -48,14 +52,10 @@ class Skeleton(Type):
# TODO: More intelegent merge
return Skeleton(self.joints + other.joints)
def draw(self, image, canvas, color=(255, 255, 255)):
for i, joint in enumerate(self.joints.values()):
x, y, z = joint.pos.project_to_image(image)
if x > image.width or x < 0 or y > image.height or y < 0:
continue
cv2.circle(canvas, (x, y), 1, color, -1, cv2.LINE_AA)
def draw(self, axes, colors=default_colors):
xs, ys, zs = zip(*(joint.pos.as_np() for joint in self.joints.values()))
color = [[v / 255 for v in color] for color in colors[:len(xs)]]
axes.scatter(xs, ys, zs, c=color)
def serialize(self):
return {type.value: joint for type, joint in self.joints.items()}