# Generated from: hand_landmarks.ipynb # Converted at: 2026-06-18T16:53:57.931Z # Next step (optional): refactor into modules & generate tests with RunCell # Quick start: pip install runcell #!pip3 install mediapipe import os, math, warnings from collections import Counter import numpy as np import pandas as pd import cv2 import matplotlib.pyplot as plt import matplotlib.patches as mpatches import mediapipe as mp from mediapipe import Image, ImageFormat from mediapipe.tasks.python.vision import HandLandmarker, HandLandmarkerOptions from mediapipe.tasks.python import vision from mediapipe.tasks.python.core import base_options as bo warnings.filterwarnings("ignore") print(f"MediaPipe : {mp.__version__}") print(f"OpenCV : {cv2.__version__}") print(f"NumPy : {np.__version__}") print(f"Pandas : {pd.__version__}") # Check if hand landmarker model exists MODEL_PATH = "hand_landmarker.task" if os.path.exists(MODEL_PATH): print("Hand model path exists ✔") else: print("Model NOT found ❌ - download hand_landmarker.task and place it here") # ── Configuration ───────────────────────────────────────────────────────────── TRAIN_DIR = "datasets" # root folder: one sub-folder per class IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} # set containing the allowed image file extensions OUTPUT_CSV = "hand_landmarks.csv" # filename where the extracted landmarks will be saved as CSV # ────────────────────────────────────────────────────────────────────────────── def collect_images(root: str) -> list: """ Walk `root` and return [{"path": ..., "label": ...}, ...]. The sub-folder name is used as the class label. """ records = [] for label in sorted(os.listdir(root)): class_dir = os.path.join(root, label) # Build full path to the class folder (e.g., TRAIN/Tree) if not os.path.isdir(class_dir): # Skip anything that is not a directory (e.g., files) continue # Continue to the next item for fname in sorted(os.listdir(class_dir)): # Loop through files inside the class directory if os.path.splitext(fname)[1].lower() in IMG_EXTS: # Get file extension and check if it's an allowed image type # Add a dictionary to the list: # "path" -> full image path # "label" -> class name (folder name) records.append({"path": os.path.join(class_dir, fname), "label": label}) return records image_records = collect_images(TRAIN_DIR) # Call the function to collect all dataset images counts = Counter(r["label"] for r in image_records) # Count how many images belong to each label print(f"Total images found: {len(image_records)}") # Print total number of images collected for label, n in counts.items(): # Loop through each label and its count print(f" {label:15s}: {n} images") # Print label name (left aligned in 15 chars) and number of images from PIL import Image as PILImage, ImageOps import numpy as np import cv2 def load_rgb(path: str, max_size: int = 800): """ 1. Loads an image and fixes hidden phone rotation. 2. Automatically scales down large images while keeping the aspect ratio intact. 3. Returns an RGB numpy array. """ try: # Load and fix rotation metadata pil_img = PILImage.open(path) pil_img = ImageOps.exif_transpose(pil_img) # Get current dimensions w, h = pil_img.size # If the image is larger than our max_size, calculate the new scaled dimensions if max(w, h) > max_size: if w > h: new_w = max_size new_h = int(h * (max_size / w)) else: new_h = max_size new_w = int(w * (max_size / h)) # Resize smoothly using high-quality resampling pil_img = pil_img.resize((new_w, new_h), PILImage.Resampling.LANCZOS) # Convert to RGB numpy array return np.array(pil_img.convert("RGB")) except Exception as e: print(f"Error loading and resizing image {path}: {e}") return None classes = sorted(counts.keys()) # Get all class names from the counts dictionary # Create a matplotlib figure with 1 row and one column per class # figsize dynamically scales width depending on number of classes fig, axes = plt.subplots(1, len(classes), figsize=(5 * len(classes), 5)) # Loop through each subplot axis and each class name together for ax, cls in zip(axes, classes): # Find the FIRST image belonging to this class # next(...) stops at the first match (used as sample image) record = next(r for r in image_records if r["label"] == cls) #! # Load the image using the helper function above img = load_rgb(record["path"]) if img is None: # Show warning in subplot title ax.set_title(f"{cls}\n⚠ LOAD FAILED") else: ax.imshow(img) # Show class name + image resolution ax.set_title(f"{cls}\n{img.shape[1]}×{img.shape[0]} px") # Hide x/y axis ticks for cleaner display ax.axis("off") # Add a main title above all subplots plt.suptitle("Sample image per class — RGB load check", fontsize=13) # Automatically adjust spacing to prevent overlap plt.tight_layout() # Render the figure to the screen plt.show() print("✅ If images appear above, loading is working correctly.") MODEL_PATH = "hand_landmarker.task" options = HandLandmarkerOptions( base_options=bo.BaseOptions(model_asset_path=MODEL_PATH), running_mode=vision.RunningMode.IMAGE, num_hands=2, min_hand_detection_confidence=0.3, min_hand_presence_confidence=0.3, min_tracking_confidence=0.3 ) print("✅ HandLandmarkerOptions configured.") import cv2 import numpy as np def draw_landmarks_on_image(rgb_image, detection_result): """ Draws hand landmarks and skeleton connections. Works dynamically for both MediaPipe Tasks objects and fallback Solutions outputs. """ # 1. Create a working copy in BGR for OpenCV operations annotated_bgr = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) h, w, _ = annotated_bgr.shape # 2. Extract landmark lists depending on which mode returned the result hand_list = [] # Check if it's a 'Tasks' API result object if hasattr(detection_result, 'hand_landmarks') and detection_result.hand_landmarks: hand_list = detection_result.hand_landmarks # Check if it's a legacy 'Solutions' API result object elif hasattr(detection_result, 'multi_hand_landmarks') and detection_result.multi_hand_landmarks: hand_list = detection_result.multi_hand_landmarks if not hand_list: return rgb_image # Return original image unchanged if no hands found # Hardcoded MediaPipe topology connections so we never depend on imports matching HAND_CONNECTIONS = [ (0, 1), (1, 2), (2, 3), (3, 4), # Thumb (0, 5), (5, 6), (6, 7), (7, 8), # Index finger (5, 9), (9, 10), (10, 11), (11, 12), # Middle finger (9, 13), (13, 14), (14, 15), (15, 16), # Ring finger (13, 17), (17, 18), (18, 19), (19, 20), # Pinky (0, 17) # Palm base connection ] # 3. Process and draw each hand independently to prevent data overwriting for hand in hand_list: # Check if landmarks are accessed directly or nested (.landmark attribute in solutions) landmarks = hand.landmark if hasattr(hand, 'landmark') else hand # Build pixel mapping for the current hand isolated current_hand_points = {} for idx, lm in enumerate(landmarks): cx, cy = int(lm.x * w), int(lm.y * h) current_hand_points[idx] = (cx, cy) # Draw structural lines (Bright Turquoise/Teal) for start, end in HAND_CONNECTIONS: a = current_hand_points.get(start) b = current_hand_points.get(end) if a and b: cv2.line(annotated_bgr, a, b, (255, 220, 0), 2, cv2.LINE_AA) # Vivid Sky Blue in BGR # Draw joints (Bright Neon Green) for pt in current_hand_points.values(): cv2.circle(annotated_bgr, pt, 4, (0, 255, 0), -1, cv2.LINE_AA) # 4. Convert back to RGB for matplotlib presentation return cv2.cvtColor(annotated_bgr, cv2.COLOR_BGR2RGB) # Run on the first image as a visual sanity check sample = image_records[0] # Load image in RGB format using helper function img = load_rgb(sample["path"]) # Wrap the numpy array into MediaPipe Image object — required by the Tasks API mp_img = Image(image_format=ImageFormat.SRGB, data=img) # 1. CHANGED: Use HandLandmarker instead of PoseLandmarker with HandLandmarker.create_from_options(options) as landmarker: result = landmarker.detect(mp_img) # Run hand detection on the image # 2. CHANGED: Check for hand_landmarks instead of pose_landmarks if result and result.hand_landmarks: annotated = draw_landmarks_on_image(img, result) # 3. UPDATED: Provide a clean status showing the number of detected hands status = f"✅ Detected {len(result.hand_landmarks)} hand(s)" else: annotated = img.copy() status = "⚠ No hand detected — try a different image" # Create figure with 2 side-by-side plots fig, axes = plt.subplots(1, 2, figsize=(12, 6)) axes[0].imshow(img); axes[0].set_title("Original"); axes[0].axis("off") axes[1].imshow(annotated); axes[1].set_title(f"Landmarks\n{status}"); axes[1].axis("off") plt.suptitle(f"{sample['label']} — {os.path.basename(sample['path'])}", fontsize=13) plt.tight_layout() plt.show() import math def distance(lm1, lm2): return math.sqrt( (lm1.x - lm2.x) ** 2 + (lm1.y - lm2.y) ** 2 + (lm1.z - lm2.z) ** 2 ) import numpy as np def angle_between(a, b, c): """ Returns angle ABC in degrees. b is the joint vertex. """ ba = np.array([a.x - b.x, a.y - b.y, a.z - b.z]) bc = np.array([c.x - b.x, c.y - b.y, c.z - b.z]) cosine = np.dot(ba, bc) / ( np.linalg.norm(ba) * np.linalg.norm(bc) ) cosine = np.clip(cosine, -1.0, 1.0) return np.arccos(cosine)/ np.pi def vector_angle(v1, v2): cosine = np.dot(v1, v2) / ( np.linalg.norm(v1) * np.linalg.norm(v2) ) cosine = np.clip(cosine, -1.0, 1.0) return np.arccos(cosine)/ np.pi def finger_vector(landmarks, base_idx, tip_idx): return np.array([ landmarks[tip_idx].x - landmarks[base_idx].x, landmarks[tip_idx].y - landmarks[base_idx].y, landmarks[tip_idx].z - landmarks[base_idx].z, ]) def extract_features(detection_result, label: str) -> dict | None: """ Extracts 21 hand landmarks and pairs them with explicit column names. Works perfectly for both MediaPipe Tasks and fallback Solutions structures. """ if not detection_result: return None # 1. Dynamically check which API format MediaPipe returned hand_list = [] if hasattr(detection_result, 'hand_landmarks') and detection_result.hand_landmarks: hand_list = detection_result.hand_landmarks elif hasattr(detection_result, 'multi_hand_landmarks') and detection_result.multi_hand_landmarks: hand_list = detection_result.multi_hand_landmarks # If both fields are empty, no hand was found if not hand_list: return None # 2. Extract the primary hand (index 0) primary_hand = hand_list[0] # Handle the fact that Solutions uses a nested '.landmark' attribute landmarks = primary_hand.landmark if hasattr(primary_hand, 'landmark') else primary_hand # 3. Build the feature dictionary mapping feature_dict = {} for idx, lm in enumerate(landmarks): feature_dict[f"x{idx}"] = lm.x feature_dict[f"y{idx}"] = lm.y feature_dict[f"z{idx}"] = lm.z wrist = landmarks[0] middle_mcp = landmarks[9] hand_size = distance(wrist, middle_mcp) distance_pairs = [ (4, 8), # thumb tip - index tip (4, 12), # thumb tip - middle tip (4, 16), # thumb tip - ring tip (4, 20), # thumb tip - pinky tip (8, 12), # index - middle (12, 16), # middle - ring (16, 20), # ring - pinky (0, 8), # wrist - index tip (0, 12), # wrist - middle tip (0, 20) # wrist - pinky tip ] for i, (a, b) in enumerate(distance_pairs): feature_dict[f"dist_{a}_{b}"] = distance( landmarks[a], landmarks[b] )/hand_size feature_dict["thumb_angle_1"] = angle_between(landmarks[1], landmarks[2], landmarks[3]) feature_dict["thumb_angle_2"] = angle_between(landmarks[2], landmarks[3], landmarks[4]) feature_dict["index_angle_1"] = angle_between(landmarks[5], landmarks[6], landmarks[7]) feature_dict["index_angle_2"] = angle_between(landmarks[6], landmarks[7], landmarks[8]) feature_dict["middle_angle_1"] = angle_between(landmarks[9], landmarks[10], landmarks[11]) feature_dict["middle_angle_2"] = angle_between(landmarks[10], landmarks[11], landmarks[12]) feature_dict["ring_angle_1"] = angle_between(landmarks[13], landmarks[14], landmarks[15]) feature_dict["ring_angle_2"] = angle_between(landmarks[14], landmarks[15], landmarks[16]) feature_dict["pinky_angle_1"] = angle_between(landmarks[17], landmarks[18], landmarks[19]) feature_dict["pinky_angle_2"] = angle_between(landmarks[18], landmarks[19], landmarks[20]) # finger vectors thumb_vec = finger_vector(landmarks, 1, 4) index_vec = finger_vector(landmarks, 5, 8) middle_vec = finger_vector(landmarks, 9, 12) ring_vec = finger_vector(landmarks, 13, 16) pinky_vec = finger_vector(landmarks, 17, 20) # finger spread angles feature_dict["thumb_index_angle"] = vector_angle( thumb_vec, index_vec ) feature_dict["index_middle_angle"] = vector_angle( index_vec, middle_vec ) feature_dict["middle_ring_angle"] = vector_angle( middle_vec, ring_vec ) feature_dict["ring_pinky_angle"] = vector_angle( ring_vec, pinky_vec ) feature_dict["thumb_pinky_angle"] = vector_angle( thumb_vec, pinky_vec ) # Append the category label feature_dict["label"] = label return feature_dict rows = [] # List that will store one feature dictionary per successful image failed = [] # List that will store paths of images that failed (no pose or load error) # Open the landmarker ONCE outside the loop. # Creating it inside the loop would reload the model weights on every image # which is very slow. The context manager ensures the model is freed afterwards. with HandLandmarker.create_from_options(options) as landmarker: # "with" automatically cleans up resources when finished for i, record in enumerate(image_records, start=1): path, label = record["path"], record["label"] # Extract image path and class label from record # 1. Load image img = load_rgb(path) if img is None: print(f" [{i:4d}] ⚠ Could not load: {path}") failed.append(path) continue # 2. Convert numpy array to MediaPipe Image object mp_img = Image(image_format=ImageFormat.SRGB, data=img) # 3. Run pose detection result = landmarker.detect(mp_img) # 4. Extract features row = extract_features(result, label) if row is None: print(f" [{i:4d}] ⚠ No pose detected: {path}") failed.append(path) continue rows.append(row) # Every 20 images (or last image), print progress if i % 20 == 0 or i == len(image_records): print(f" [{i:4d}/{len(image_records)}] ✓ rows so far: {len(rows)}") print(f"\n✅ Done. Successful: {len(rows)} | Failed: {len(failed)}") # Convert list of dictionaries (rows) into a pandas DataFrame df = pd.DataFrame(rows) # shuffle the dataset so that not every class is grouped together: df = df.sample(frac=1, random_state=42).reset_index(drop=True) print("Shape:", df.shape) # Print dataset dimensions: (number_of_rows, number_of_columns) print("\nSamples per class:") print(df["label"].value_counts().to_string()) df.head() df.to_csv(OUTPUT_CSV, index=False) # index=False keeps row numbers out of the file print(f"✅ Saved '{OUTPUT_CSV}'") print(f" Rows : {len(df)}") print(f" Columns : {len(df.columns)}") # Round-trip check: reload and verify shape matches df_check = pd.read_csv(OUTPUT_CSV) assert df_check.shape == df.shape, "Shape mismatch after reload!" print("✅ CSV round-trip check passed.")