166 lines
6.0 KiB
Python
166 lines
6.0 KiB
Python
import json
|
|
from pathlib import Path
|
|
import logging
|
|
import torch
|
|
from ultralytics import YOLO
|
|
import cv2
|
|
from dodgeballtracker.helper import iter_frames
|
|
from dodgeballtracker.video_writer import VideoWriter, DetectedObjectBox
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pandera as pa
|
|
from boxmot import DeepOcSort as DeepOCSORT
|
|
from .typing import VideoFrame, DetectedObject_, TrackedDetectedObject
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
Color = tuple[int, int, int]
|
|
|
|
|
|
def get_best_available_torch_device() -> torch.device:
|
|
if torch.cuda.is_available():
|
|
logger.debug("Using the cuda (gpu) torch device.")
|
|
return torch.device('cuda:0')
|
|
if torch.backends.mps.is_available():
|
|
logger.debug("Using the mps (apple silicon) torch device.")
|
|
return torch.device('mps')
|
|
logger.debug("Default to cpu (no apple silicon or cuda) torch device.")
|
|
return torch.device('cpu')
|
|
|
|
|
|
def get_court_perspective_transform():
|
|
"""
|
|
"""
|
|
empirical_top_left = np.float32([416, 175])
|
|
empirical_top_right = np.float32([905, 173])
|
|
empirical_bottom_left = np.float32([51, 702])
|
|
empirical_bottom_right = np.float32([1188, 695])
|
|
|
|
court_top_left = np.float32([0, 0])
|
|
court_top_right = np.float32([9, 0])
|
|
court_bottom_left = np.float32([0, 18])
|
|
court_bottom_right = np.float32([9, 18])
|
|
|
|
empirical_source = np.float32([empirical_top_left,
|
|
empirical_top_right,
|
|
empirical_bottom_left,
|
|
empirical_bottom_right])
|
|
pixel_per_meter = 50
|
|
empirical_target = np.float32([court_top_left,
|
|
court_top_right,
|
|
court_bottom_left,
|
|
court_bottom_right]) * pixel_per_meter
|
|
return cv2.getPerspectiveTransform(empirical_source, empirical_target)
|
|
|
|
|
|
|
|
|
|
|
|
class DodgeballTracker:
|
|
|
|
def __init__(self,
|
|
input_file: Path,
|
|
output_file: Path,
|
|
video_writer: VideoWriter,
|
|
detected_box_objects: list[DetectedObjectBox],
|
|
transformation_matrix):
|
|
self.input_path = input_file
|
|
self.output_path = output_file
|
|
self.output_json_path = output_file.with_suffix(".json")
|
|
self.video_writer = video_writer
|
|
self.detected_box_objects = sorted(detected_box_objects, reverse=True)
|
|
self.transformation_matrix = transformation_matrix
|
|
self.points = []
|
|
|
|
def write_frame(self, writer, frame) -> None:
|
|
cv2.imshow("Frame", frame)
|
|
writer.write(frame)
|
|
|
|
def extract_box(self, data) -> tuple[int, int, int, int]:
|
|
return int(data[0]), int(data[1]), int(data[2]), int(data[3])
|
|
|
|
def print_box(self, data, frame_idx, frame_ts):
|
|
xmin, ymin, xmax, ymax = self.extract_box(data)
|
|
xavg = (xmax + xmin) / 2
|
|
v = np.array([xavg, ymax, 1])
|
|
ret = self.transformation_matrix @ v
|
|
x, y = ret[0] / ret[2], ret[1] / ret[2]
|
|
object_id = data[-1]
|
|
self.points.append({"x": x,
|
|
"y": y,
|
|
"frameIndex": frame_idx,
|
|
"frameTimestamp": frame_ts,
|
|
"objectId": int(object_id)
|
|
})
|
|
|
|
def print_tracker_data(self, tracker_data, frame_idx, frame_ts):
|
|
for d in tracker_data:
|
|
self.print_box(d, frame_idx, frame_ts)
|
|
|
|
def plot_box_to_frame(self, data, frame: VideoFrame):
|
|
confidence = float(data[4])
|
|
for detected_box_object in self.detected_box_objects:
|
|
|
|
if detected_box_object.min_threshold < confidence:
|
|
# xmin, ymin, xmax, ymax = self.extract_box(data)
|
|
# cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), detected_box_object.color, 2)
|
|
return True
|
|
return False
|
|
|
|
@pa.check_output(DetectedObject_)
|
|
def run_model_on_frame(self, model, frame: VideoFrame):
|
|
model_outputs = model(frame)[0]
|
|
boxes = model_outputs.boxes.data.cpu().numpy()
|
|
columns=['xmin', 'ymin', 'xmax', 'ymax', 'score', 'category']
|
|
ret = pd.DataFrame(boxes, columns=columns)
|
|
return ret
|
|
|
|
def persist_points(self):
|
|
with open(self.output_json_path, "wt") as file_:
|
|
json.dump({"points": self.points}, file_, indent=4)
|
|
|
|
def _get_tracker(self) -> DeepOCSORT:
|
|
return DeepOCSORT(
|
|
Path('lmbn_n_cuhk03_d.pt'), # which ReID model to use
|
|
get_best_available_torch_device(),
|
|
True,
|
|
)
|
|
|
|
def _get_detection_model(self):
|
|
"""
|
|
load the pre-trained YOLOv8n model
|
|
"""
|
|
my_model = Path(
|
|
'/home/didier/all-dressed-programming/dodgeball-tracker/dodgeballtracker/runs/detect/train31/weights/best.pt')
|
|
return YOLO(my_model, verbose=False)
|
|
|
|
@pa.check_io(detected_objs=DetectedObject_, out=TrackedDetectedObject)
|
|
def update_tracker(self, tracker: DeepOCSORT, detected_objs: pd.DataFrame, frame: VideoFrame):
|
|
data = tracker.update(detected_objs.to_numpy(), frame)
|
|
columns = ['xmin', 'ymin', 'xmax', 'ymax', 'object_id', 'score', 'category', 'index']
|
|
if not data.size:
|
|
return pd.DataFrame({col: pd.Series(dtype='float64') for col in columns})
|
|
return pd.DataFrame(data, columns=columns)
|
|
|
|
def train(self):
|
|
model = self._get_detection_model()
|
|
|
|
def main(self):
|
|
# define some constants
|
|
video_cap = cv2.VideoCapture(str(self.input_path))
|
|
tracker = self._get_tracker()
|
|
model = self._get_detection_model()
|
|
|
|
for i, (frame, ts) in enumerate(iter_frames(video_cap)):
|
|
# run the YOLO model on the frame
|
|
detected_objs = self.run_model_on_frame(model, frame)
|
|
tracked_obj = self.update_tracker(tracker, detected_objs, frame)
|
|
self.video_writer.plot_boxes_to_frame(tracked_obj, frame)
|
|
self.video_writer.write_frame(frame)
|
|
|
|
if self.video_writer.wait():
|
|
break
|
|
|
|
video_cap.release()
|
|
self.persist_points()
|