Compare commits
No commits in common. "main" and "4beddc5e6d605428baa3e390d52989e0199a77bf" have entirely different histories.
main
...
4beddc5e6d
5 changed files with 47 additions and 206 deletions
|
@ -1,42 +0,0 @@
|
||||||
'''
|
|
||||||
█████████████████████████████████████████████▀▀▀▀▀▀▀▀▀▀█████████████████████████████████████▀▄██████████████████████████
|
|
||||||
█████████████████████████████████████████▀▀«─ⁿ^^^╜╜╜╝╝╣╬æ░█████████████████████████████████▀▄███████████████████████████
|
|
||||||
███████████████████████████████████████████████████████▄▄╙╗▀█▀▀▀▀▀▀▀█████████████████████▀▄█████████████████████████████
|
|
||||||
███████████████████████████████████████████████▀▀¼@╣╬╢╣╣æ@@╬ÑÑÑÑD╬ÑÑ╬╣æ╗╬▀▀█████████████▀▄██████████████████████████████
|
|
||||||
████████████████████████████████████████████▀╔╢ÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD╬╖╬▀█████████▀████████████████████████████████
|
|
||||||
██████████████████████████████████████████▀é]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ╣φ▀█████▌▄████████████████████████████████
|
|
||||||
████████████████████████████████████████▀╗▓▓]ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ]▓▓▓Ö▀██▐█████████████████████████████████
|
|
||||||
██████████████████████████████████████▀╓▓▓▓▓▓▓▓]]DÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD╫▓▓▓▓▓▓▓u ██████████████████████████████████
|
|
||||||
█████████████████████████████████████╓▓▓▀ÑÑ]Ñ▓▓▀▀▀ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ]D╫▓▓▓▓▓▓▀▓▓▓▐██████████████████████████████████
|
|
||||||
███████████████████████████████████▀╔Ñ]ÑÑÑÑÑÑÑDÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑD]ÑÑDÑÑÑÑ]]ÑÑÑM██████████████▀▀▀▀▀▀▀▀▀▀▀▀▀▀███████
|
|
||||||
██████████████████████████████████▌║ÑÑÑÑÑÑÑÑ╬M░╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑh██████████████ oh, here's a ███████
|
|
||||||
█████████████████████████████████▌╓ÑÑÑD╝╢Ñ╬½╗╬╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ╬╟DÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑh██████████████ a fun fact ███████
|
|
||||||
█████████████████████████████████.ÑÑÑDÑ╙▓▄╓╬ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑU░╟ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┐██████████████▄▄▄▄▄▄▄▄▄▄▄▄▄▄███████
|
|
||||||
████████████████████████████████⌐╣ÑÑÑ╓µ╝M╔╣Ñ╣'╬Ñ╬╫ÑÑÑÑÑÑ╬╬ÑÑÑÑÑÑDÑÑ╙╝╜,▄╕╢ÑÑÑÑÑÑÑÑÑÑ╬▐██████████████████████████████████
|
|
||||||
████████████████████████████████║ÑÑÑÑæÖ▀ Ñ╬Ω▄LÑDM║ÑÑÑÑÑÑM╟ÑÑÑÑÑÑÑÑφΦ▀▀╠╔╬╬ÑÑÑÑÑÑÑÑÑÑÑ ██████████████████████████████████
|
|
||||||
██████████████████████████████▀║ÑÑÑÑÑÑ╬ ╫M▄▓▄ ║D ╚DÑÑÑÑÑu║ÑÑMÑDÑÑÑÑ╬M`▄▄φΦ║ÑÑÑÑÑÑÑÑÑÑ╖▐█████████████████████████████████
|
|
||||||
█████████████████████████████▌╓ÑÑÑÑÑÑDM┌ ▓▓▓▓L╢╣M ╬ÑÑÑÑÑ └Ñ╬~²╣ÑÑÑÑѼ▀'╔æ╢╬ÑÑÑÑÑÑÑÑÑÑÑ┐▀████████████████████████████████
|
|
||||||
█████████████████████████████╔ÑM╬ÑÑÑÑÑù╓▀"`` "²ⁿ▌║ÑÑÑÑÑhL╢Ñ∩µ╢m─ªª╣╣M½║DÑÑÑÑÑÑÑÑÑÑÑÑÑÑ,████████████████████████████████
|
|
||||||
████████████████████████████ ╬^║ÑÑÑÑÑÑM▓H ,╗h ▓ ÑÑÑÑÑh▓ ÑM ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ ███████████████████████████████
|
|
||||||
███████████████████████████▌╣ Γ╣ÑÑÑÑÑDM║ ╢╬ ╫▌▄▄▄▄▄▄▓▓▄▄ '╬▓w ÑÑÑÑÑÑÑÑÑÑÑÑÑD╬`▄███████████████████████████████
|
|
||||||
███████████████████████████ █ ╬ÑÑÑÑÑÑÑ▐▌ ╬▀▀ ║██████████▄, ╬╬╬ ÑÑÑÑÑÑÑÑÑÑÑÑ╣┘-▀5╔╢ ▀███████████████████████████
|
|
||||||
███████████████████████████╘▐█ ÑÑÑÑÑÑÑÑ▐█▄██ ███████████▌ ▓▓▓ ▄ DÑÑÑÑÑÑÑÑÑÑ╖╖æ╣ÑÑÑÑHß ▀█████████████████████████
|
|
||||||
███████████████████████████ ▐█ ÑÑ╬ÑÑÑÑÑ▐█▓██▓ ¥╜▄████████████ Φæ┘▄████▌╢ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ^╞▐█▄▄▀████████████████████▀▄
|
|
||||||
███████▀█▄▄████████▄▄▄▀▀████▐█ ╬╬░╬ÑÑÑÑ▐▓╣▓▓▓▓████████████████▄▄██████ ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┘┤|█████▄▄▄▀▀██████████▀▀▄▄███
|
|
||||||
███▀▄▄██████████████████▄▄▀███▌║ÑM╢ÑÑÑÑJ█▓▓▓███████████████████▓▓▓▓▓▓`╣ÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑÑ┘▄M,████████████▄▄▄▄▄██████████
|
|
||||||
█▀▄████████████████████████▄▀██µ╢┼,╢ÑÑÑD╣╬▀▀███████▄╬╬D█████████████▀║DÑÑÑÑÑÑÑÑÑÑDM╫DÑÑ╬ ██▄████████████████████████████
|
|
||||||
▄████████████████████████████▄▀█ ╬▐█╙╬ÑÑÑÑ▐▄ "▀█████▄╢M███████████▀▀║ÑÑÑÑÑÑÑÑ╬╟ÑÑ╬ⁿ╬ÑÑ╝╓████████████████████████████████
|
|
||||||
███████████████████████████████▄▀ ██▄░╢ÑÑM██ ▄`"▀▀▀▀▀▀▀▀"` . ▄▄╢ÑÑÑM░╣╜║╬ÑM▄█║Ñ^▄█████████████████████████████████
|
|
||||||
████████████████████████████████▄▀▄▀████▄╙╢▐███▄▐██▄ .⌡,#╓██ ╣╣╜}-,▄▌╬M▄███w▄███████████████████████████████████
|
|
||||||
█████████████████████████████████▌▀▌▀▀▀▀,██▄████████▀7 ╓▓▌╙▀└██▌▐███████²▄██████████████████████████████████████████
|
|
||||||
████████████████████████████████████▀,▄█████████▀▀«]░╓▓▓▓▓▓▓▀«]░J██ ████████████████████████████████████████████████████
|
|
||||||
███████████████████████████████████▐███████▀▀µj]░ ░ █╣▓▓▓▀ j░ ░▐██ ▄Å▀▀████████████████████████████████████████████████
|
|
||||||
█████████▀▀▀▀▀▀▀▀▀▀▀▀▀▀████████████ ██████┌]▌║ ░ r▐█▀██▀███▄`y███▐█▀█▌▄▀██████████████████████████████████████████████
|
|
||||||
█████████ i do not ████████████▌████▀▄▓ ╙▄▓` ▓▓▓▓⌐jr╙████▄▀▌██▓█▓▌▐██████████████████████████████████████████████
|
|
||||||
█████████ think ████████████▌███▀▄▓▓╬ ╙▄▀▄░░ ▓▓▓▀j░ H`▀▀▀▀""` ]«]«¥██████████████████████████████████████████████
|
|
||||||
█████████▄▄▄▄▄▄▄▄▄▄▄▄▄▄████████████▌██▌▐▓▓▓▓L,╨¥"Vµ ▓╣▌j░░U ░ ░ ░,µ⌐Ö║██████████████████████████████████████████████
|
|
||||||
███████████████████████████████████ ██,▓▓▓▓▓ ░½`"▀▄ ▌▌/δ▀DxH*¥#==═K*"▄x╨░d██████████████████████████████████████████████
|
|
||||||
███████████████████████████████████▐█`╣▓▓▓▓▓µ░░░^ ` ╚ "░ ,«⌂]"M╨*"""^▄▄▓╙▓▐█████████████████████████████████████████████
|
|
||||||
██████████████████████████████████ ██▐▌║▓▓▓▓▌▄▄D,╔,ⁿ«-░░░,⌂░^╓ó],▄▓█▓▓▓▓▌Γ██████████████████████████████████████████████
|
|
||||||
█████████████████████████████████▀██ ▓▓Ω▓▓▓▓▓▓▀]░`j ]j⌂,"]H╓⌐▓▓▓▓▓▓▓▓▓▓▓▓µ▐█████████████████████████████████████████████
|
|
||||||
'''
|
|
|
@ -32,26 +32,6 @@ face_mesh_map = {
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
body_map = {
|
|
||||||
LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.LEFT: [11],
|
|
||||||
LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.RIGHT: [12],
|
|
||||||
|
|
||||||
LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.LEFT: [13],
|
|
||||||
LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.RIGHT: [14],
|
|
||||||
|
|
||||||
LANDMARK_TYPES.HIP | LANDMARK_TYPES.LEFT: [23],
|
|
||||||
LANDMARK_TYPES.HIP | LANDMARK_TYPES.RIGHT: [24],
|
|
||||||
|
|
||||||
LANDMARK_TYPES.KNEE | LANDMARK_TYPES.LEFT: [25],
|
|
||||||
LANDMARK_TYPES.KNEE | LANDMARK_TYPES.RIGHT: [26],
|
|
||||||
|
|
||||||
LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.LEFT: [27],
|
|
||||||
LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.RIGHT: [28],
|
|
||||||
|
|
||||||
LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.LEFT: [27],
|
|
||||||
LANDMARK_TYPES.ANKLE | LANDMARK_TYPES.RIGHT: [28],
|
|
||||||
}
|
|
||||||
|
|
||||||
# SEE YEAH THESE MAKE SENSE GOOGLE WHAT THE HELL
|
# SEE YEAH THESE MAKE SENSE GOOGLE WHAT THE HELL
|
||||||
hand_mesh_map = {LANDMARK_TYPES.HAND | LANDMARK_TYPES.WRIST: [0]}
|
hand_mesh_map = {LANDMARK_TYPES.HAND | LANDMARK_TYPES.WRIST: [0]}
|
||||||
_finger_map = {
|
_finger_map = {
|
||||||
|
|
|
@ -2,7 +2,7 @@ import mediapipe
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
from ovtk_track.transform import TransformProcess
|
from ovtk_track.transform import TransformProcess
|
||||||
from ovtk_track.transform.solve.mediapipe import face_mesh_map, hand_mesh_map, body_map
|
from ovtk_track.transform.solve.mediapipe import face_mesh_map, hand_mesh_map
|
||||||
from ovtk_track import types
|
from ovtk_track import types
|
||||||
from ovtk_track.types.Landmarks import LANDMARK_TYPES
|
from ovtk_track.types.Landmarks import LANDMARK_TYPES
|
||||||
|
|
||||||
|
@ -83,17 +83,6 @@ class Process(TransformProcess):
|
||||||
|
|
||||||
available.append((right_hand_landmarks, mix_maps(hand_mesh_map, LANDMARK_TYPES.RIGHT)))
|
available.append((right_hand_landmarks, mix_maps(hand_mesh_map, LANDMARK_TYPES.RIGHT)))
|
||||||
|
|
||||||
if results.pose_landmarks:
|
|
||||||
raw_landmarks = results.pose_landmarks.landmark
|
|
||||||
body_landmarks = np.empty((33, 3), dtype=np.float32)
|
|
||||||
|
|
||||||
for i in range(33):
|
|
||||||
body_landmarks[i][0] = raw_landmarks[i].x
|
|
||||||
body_landmarks[i][1] = raw_landmarks[i].y
|
|
||||||
body_landmarks[i][2] = raw_landmarks[i].z
|
|
||||||
|
|
||||||
available.append((body_landmarks, body_map))
|
|
||||||
|
|
||||||
if available:
|
if available:
|
||||||
avail_landmarks, maps = zip(*available)
|
avail_landmarks, maps = zip(*available)
|
||||||
combo_map = combine_maps(zip(maps, (array.shape[0] for array in avail_landmarks)))
|
combo_map = combine_maps(zip(maps, (array.shape[0] for array in avail_landmarks)))
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
import math
|
import math
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from scipy.spatial.distance import cdist
|
|
||||||
|
|
||||||
from .. import TransformProcess
|
from .. import TransformProcess
|
||||||
from ovtk_track.types import Quaternion, Point3d
|
from ovtk_track.types import Quaternion, Point3d
|
||||||
|
@ -25,97 +24,9 @@ class Process(TransformProcess):
|
||||||
self.normal = np.array(normal, dtype=float)
|
self.normal = np.array(normal, dtype=float)
|
||||||
self.up = np.array(vec_perp(normal), dtype=float)
|
self.up = np.array(vec_perp(normal), dtype=float)
|
||||||
|
|
||||||
# REVIEW: See calc_eye. These probably need to change based on normal / up.
|
|
||||||
# Or maybe they dont and we just rotate the output quaternion?
|
|
||||||
# Ugh. The code works for now, but i no understand....
|
|
||||||
self.SIN_LEFT_THETA = 2 * np.sin(np.pi / 2)
|
|
||||||
self.SIN_UP_THETA = np.sin(np.pi / 6)
|
|
||||||
|
|
||||||
def setup(self):
|
def setup(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def calc_head(self, landmarks):
|
|
||||||
# REVIEW: This doesnt really work quite right!! look + roll arent mixing as expected
|
|
||||||
# Vector pointing from head center to nose
|
|
||||||
nose = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.NOSE | LANDMARK_TYPES.TIP]).mean(axis=0)
|
|
||||||
head_center = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.FACE | LANDMARK_TYPES.OUTLINE]).mean(axis=0)
|
|
||||||
look_vec = (nose - head_center)
|
|
||||||
look_vec /= np.linalg.norm(look_vec)
|
|
||||||
# Vector pointing left to right across the face
|
|
||||||
eye_center_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]).mean(axis=0)
|
|
||||||
eye_center_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]).mean(axis=0)
|
|
||||||
roll_vec = (eye_center_l - eye_center_r)
|
|
||||||
roll_vec /= np.linalg.norm(roll_vec)
|
|
||||||
|
|
||||||
# Quat that rotates from normal to head center -> nose vec
|
|
||||||
look = Quaternion(np.dot(look_vec, self.normal), *np.cross(look_vec, self.normal))
|
|
||||||
look.w += look.magnitude()
|
|
||||||
look = look.normalize()
|
|
||||||
# Quat that represents a rotation around the roll axis (i think??)
|
|
||||||
roll_angle = np.sum(roll_vec * self.up)
|
|
||||||
roll = Quaternion(math.cos(roll_angle), *(self.normal * math.sin(roll_angle)))
|
|
||||||
roll = roll.normalize()
|
|
||||||
|
|
||||||
combo = look + roll
|
|
||||||
combo = combo.normalize()
|
|
||||||
|
|
||||||
return combo, head_center
|
|
||||||
|
|
||||||
def calc_eye(self, landmarks):
|
|
||||||
# Get poi
|
|
||||||
corners = np.empty((2, 2, 3), dtype=np.float32)
|
|
||||||
centers = np.empty((2, 3), dtype=np.float32)
|
|
||||||
pupils = np.empty((2, 3), dtype=np.float32)
|
|
||||||
cross_heights = np.empty((2), dtype=np.float32)
|
|
||||||
for i, side in enumerate([LANDMARK_TYPES.LEFT, LANDMARK_TYPES.RIGHT]):
|
|
||||||
# Find corners by searching for points with the largest distance from each other
|
|
||||||
# REVIEW: These *should* will always be the same points in the map - make a landmark type selector?
|
|
||||||
eye_outline = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.OUTLINE | side])
|
|
||||||
hdist = cdist(eye_outline, eye_outline, metric='euclidean')
|
|
||||||
best_pair = np.unravel_index(hdist.argmax(), hdist.shape)
|
|
||||||
corners[i] = eye_outline[best_pair[0]], eye_outline[best_pair[1]]
|
|
||||||
# Get height of eye (relative to a line passing through each corner)
|
|
||||||
cross_heights[i] = np.array([
|
|
||||||
np.linalg.norm(np.cross(corners[i][1]-corners[i][0],
|
|
||||||
corners[i][0]-point))
|
|
||||||
/ np.linalg.norm(corners[i][0]-corners[i][1])
|
|
||||||
for point in eye_outline
|
|
||||||
]).max()
|
|
||||||
|
|
||||||
centers[i] = eye_outline.mean(axis=0)
|
|
||||||
pupils[i] = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | side]).mean(axis=0)
|
|
||||||
# Calculate important distances based on POI
|
|
||||||
eye_length = np.linalg.norm(np.diff(corners, axis=1), axis=(2, 1))
|
|
||||||
ic_distance = np.linalg.norm(pupils - centers, axis=1)
|
|
||||||
zc_distance = np.linalg.norm(pupils - corners[:, 1], axis=1)
|
|
||||||
aspect_ratio = 1 / (cross_heights / eye_length)
|
|
||||||
|
|
||||||
# Takes above and spits out spherical coordiates of pupil (relative to camera)
|
|
||||||
# Black magic as far as i can comprehend
|
|
||||||
# Copied in large part from https://github.com/1996scarlet/OpenVtuber/blob/970229d3a5ebe14a7519352da039d00a0b87e2d9/service/TFLiteIrisLocalization.py#L101
|
|
||||||
s0 = (corners[1, :, 1] - corners[0, :, 1]) * pupils[:, 0]
|
|
||||||
s1 = (corners[1, :, 0] - corners[0, :, 0]) * pupils[:, 1]
|
|
||||||
s2 = corners[1, :, 0] * corners[0, :, 1]
|
|
||||||
s3 = corners[1, :, 1] * corners[0, :, 0]
|
|
||||||
|
|
||||||
delta_y = (s0 - s1 + s2 - s3) / eye_length / 2
|
|
||||||
delta_x = np.sqrt(abs(ic_distance**2 - delta_y**2))
|
|
||||||
delta = np.array((delta_x * self.SIN_LEFT_THETA,
|
|
||||||
delta_y * self.SIN_UP_THETA))
|
|
||||||
delta /= eye_length
|
|
||||||
theta, pha = np.arcsin(delta)
|
|
||||||
inv_judge = zc_distance**2 - delta_y**2 < eye_length**2 / 4
|
|
||||||
theta[inv_judge] *= -1
|
|
||||||
|
|
||||||
# Convert spherical coordiates to quaternions
|
|
||||||
# Based on https://github.com/moble/quaternion/blob/8f6fc306306c45f0bf79331a22ef3998e4d187bc/src/quaternion/__init__.py#L599
|
|
||||||
quats = np.array([np.cos(pha/2) * np.cos(theta/2),
|
|
||||||
-np.sin(pha/2) * np.sin(theta/2),
|
|
||||||
np.cos(pha/2) * np.sin(theta/2),
|
|
||||||
np.sin(pha/2) * np.cos(theta/2)]).T
|
|
||||||
quat_arr = [Quaternion(*quat) for quat in quats]
|
|
||||||
return quat_arr, aspect_ratio
|
|
||||||
|
|
||||||
def process(self):
|
def process(self):
|
||||||
landmarks = self._inputs['landmarks'].get()
|
landmarks = self._inputs['landmarks'].get()
|
||||||
skeleton = None
|
skeleton = None
|
||||||
|
@ -123,41 +34,56 @@ class Process(TransformProcess):
|
||||||
joints = {}
|
joints = {}
|
||||||
if landmarks.has(LANDMARK_TYPES.FACE):
|
if landmarks.has(LANDMARK_TYPES.FACE):
|
||||||
# Get head look / pos
|
# Get head look / pos
|
||||||
look_quat, head_pos = self.calc_head(landmarks)
|
nose = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.NOSE | LANDMARK_TYPES.TIP]).mean(0)
|
||||||
eye_quats, eye_aspect = self.calc_eye(landmarks)
|
head_center = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.FACE | LANDMARK_TYPES.OUTLINE]).mean(0)
|
||||||
|
look_vec = (nose - head_center)
|
||||||
|
|
||||||
head_joint = Joint(Point3d(*head_pos), look_quat, attr=dict(eye_rot=eye_quats, eye_aspect=eye_aspect))
|
eye_center_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT]).mean(0)
|
||||||
|
eye_center_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT]).mean(0)
|
||||||
|
roll_vec = (eye_center_l - eye_center_r)
|
||||||
|
|
||||||
|
look_vec /= np.linalg.norm(look_vec)
|
||||||
|
roll_vec /= np.linalg.norm(roll_vec)
|
||||||
|
|
||||||
|
roll_angle = np.sum(roll_vec * self.up)
|
||||||
|
roll = Quaternion(math.cos(roll_angle), * self.normal * math.sin(roll_angle))
|
||||||
|
roll = roll.normalize()
|
||||||
|
|
||||||
|
look = Quaternion(np.dot(look_vec, self.normal), *np.cross(look_vec, self.normal))
|
||||||
|
look.w += look.magnitude()
|
||||||
|
look = look.normalize()
|
||||||
|
|
||||||
|
combo = look + roll
|
||||||
|
combo = combo.normalize()
|
||||||
|
|
||||||
|
# Get eye data
|
||||||
|
marks_left = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.LEFT])
|
||||||
|
marks_right = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.EYE | LANDMARK_TYPES.RIGHT])
|
||||||
|
range = np.array([marks_left.max(axis=0) - marks_left.min(axis=0),
|
||||||
|
marks_right.max(axis=0) - marks_right.min(axis=0)])
|
||||||
|
delta = np.array([eye_center_l - Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | LANDMARK_TYPES.CENTER | LANDMARK_TYPES.LEFT]).mean(0),
|
||||||
|
eye_center_r - Landmarks.to_numpy(landmarks[LANDMARK_TYPES.IRIS | LANDMARK_TYPES.CENTER | LANDMARK_TYPES.RIGHT]).mean(0)])
|
||||||
|
|
||||||
|
delta /= range
|
||||||
|
try:
|
||||||
|
eye_aspect_ratio = range[::, 0] / range[::, 1]
|
||||||
|
except ZeroDivisionError:
|
||||||
|
eye_aspect_ratio = None
|
||||||
|
|
||||||
|
head_joint = Joint(Point3d(*head_center), combo, dict(look_delta=delta, eye_aspect_ratio=eye_aspect_ratio))
|
||||||
|
|
||||||
joints[JOINT_TYPES.HEAD] = head_joint
|
joints[JOINT_TYPES.HEAD] = head_joint
|
||||||
|
|
||||||
if landmarks.has(LANDMARK_TYPES.SHOULDER):
|
|
||||||
shoulder_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.LEFT]).mean(axis=0)
|
|
||||||
shoulder_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER | LANDMARK_TYPES.RIGHT]).mean(axis=0)
|
|
||||||
joints[JOINT_TYPES.SHOULDER_L] = Joint(Point3d(*shoulder_l), Quaternion.identity())
|
|
||||||
joints[JOINT_TYPES.SHOULDER_R] = Joint(Point3d(*shoulder_r), Quaternion.identity())
|
|
||||||
|
|
||||||
if landmarks.has(LANDMARK_TYPES.ELBOW):
|
|
||||||
elbow_l = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.LEFT]).mean(axis=0)
|
|
||||||
elbow_r = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.ELBOW | LANDMARK_TYPES.RIGHT]).mean(axis=0)
|
|
||||||
joints[JOINT_TYPES.ELBOW_L] = Joint(Point3d(*elbow_l), Quaternion.identity())
|
|
||||||
joints[JOINT_TYPES.ELBOW_R] = Joint(Point3d(*elbow_r), Quaternion.identity())
|
|
||||||
|
|
||||||
if landmarks.has(LANDMARK_TYPES.HIP):
|
|
||||||
hips = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.HIP]).mean(axis=0)
|
|
||||||
joints[JOINT_TYPES.HIPS] = Joint(Point3d(*hips), Quaternion.identity())
|
|
||||||
|
|
||||||
# Synthizise other joints from existing data
|
# Synthizise other joints from existing data
|
||||||
if not joints.get(JOINT_TYPES.CHEST):
|
if not joints.get(JOINT_TYPES.CHEST) and joints.get(JOINT_TYPES.HEAD):
|
||||||
if landmarks.has(LANDMARK_TYPES.SHOULDER) and landmarks.has(LANDMARK_TYPES.HIP):
|
chest_center = joints[JOINT_TYPES.HEAD].pos.as_np()
|
||||||
chest = Landmarks.to_numpy(landmarks[LANDMARK_TYPES.SHOULDER, LANDMARK_TYPES.HIP]).mean(axis=0)
|
chest_center = np.power(chest_center, 3) / (1e3 + np.power(chest_center, 2))
|
||||||
joints[JOINT_TYPES.CHEST] = Joint(Point3d(*chest), Quaternion.identity())
|
chest_center -= [0, 100, 0]
|
||||||
elif joints.get(JOINT_TYPES.HEAD):
|
|
||||||
chest_center = joints[JOINT_TYPES.HEAD].pos.as_np()
|
|
||||||
chest_center = np.power(chest_center, 3) / (1e3 + np.power(chest_center, 2))
|
|
||||||
chest_center -= [0, 100, 0]
|
|
||||||
chest_rot = Quaternion.identity().slerp(joints[JOINT_TYPES.HEAD].rot, 0.1)
|
|
||||||
|
|
||||||
joints[JOINT_TYPES.CHEST] = Joint(Point3d(*chest_center), chest_rot)
|
chest_rot = Quaternion.identity().slerp(joints[JOINT_TYPES.HEAD].rot, 0.1)
|
||||||
|
|
||||||
|
chest_joint = Joint(Point3d(*chest_center), chest_rot)
|
||||||
|
joints[JOINT_TYPES.CHEST] = chest_joint
|
||||||
|
|
||||||
skeleton = Skeleton(joints)
|
skeleton = Skeleton(joints)
|
||||||
|
|
||||||
|
|
|
@ -17,17 +17,6 @@ class LANDMARK_TYPES(Flag):
|
||||||
LIPS = auto()
|
LIPS = auto()
|
||||||
CHIN = auto()
|
CHIN = auto()
|
||||||
|
|
||||||
# Body
|
|
||||||
SHOULDER = auto()
|
|
||||||
ELBOW = auto()
|
|
||||||
HIP = auto()
|
|
||||||
KNEE = auto()
|
|
||||||
|
|
||||||
# Feet tracking lmao
|
|
||||||
ANKLE = auto()
|
|
||||||
HEEL = auto()
|
|
||||||
TOE_INDEX = auto()
|
|
||||||
|
|
||||||
# Hand
|
# Hand
|
||||||
HAND = auto()
|
HAND = auto()
|
||||||
WRIST = auto()
|
WRIST = auto()
|
||||||
|
@ -82,9 +71,8 @@ class Landmarks(Type):
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def draw(self, image, canvas, color=(255, 255, 255), label=True, filter=None):
|
def draw(self, image, canvas, color=(255, 255, 255), label=True):
|
||||||
points = self[filter] if filter else self.points
|
for i, (x, y, z) in enumerate(point.project_to_image(image) for point in self.points):
|
||||||
for i, (x, y, z) in enumerate(point.project_to_image(image) for point in points):
|
|
||||||
if x > image.width or x < 0 or y > image.height or y < 0:
|
if x > image.width or x < 0 or y > image.height or y < 0:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|
Loading…
Add table
Reference in a new issue