feat: implement scoring and ranking for video frames, enhance candidate selection process

src/analysis/scanVideo.ts

@@ -1,5 +1,5 @@
 import { rankAndTrimCandidates } from "./candidateRanking";
-import { frameDifferenceRatio } from "./frameDifference";
+import { hybridFrameScore } from "./frameDifference";
 import { captureAnalysisFrame, createThumbnailUrl } from "../media/canvas";
 import { seekVideo } from "../media/video";
 import type { CandidateFrame, ScanSettings } from "../types/scan";
@@ -15,9 +15,11 @@ export async function scanVideoForCandidates(
   callbacks: ScanCallbacks,
 ): Promise<CandidateFrame[]> {
   const analysisCanvas = document.createElement("canvas");
-  const candidates: CandidateFrame[] = [];
-  let lastAcceptedFrame: ImageData | null = null;
-  let lastAcceptedTime: number | null = null;
+  const scoredFrames: Array<{ score: number; time: number }> = [];
+  const selectedCandidates: CandidateFrame[] = [];
+  let previousFrame: ImageData | null = null;
+  let windowFrame: ImageData | null = null;
+  let windowTime = 0;
 
   try {
     for (
@@ -38,64 +40,74 @@ export async function scanVideoForCandidates(
         settings.analysisHeight,
       );
 
-      if (!lastAcceptedFrame) {
-        lastAcceptedFrame = imageData;
-
-        if (settings.includeFirstFrame) {
-          const thumbnailUrl = await createThumbnailUrl(video);
-          candidates.push({
-            id: crypto.randomUUID(),
-            time,
-            score: 1,
-            reason: "initial-frame",
-            thumbnailUrl,
-          });
-          lastAcceptedTime = time;
-        }
+      if (!previousFrame) {
+        previousFrame = imageData;
+        windowFrame = imageData;
+        windowTime = time;
       } else {
-        const diffRatio = frameDifferenceRatio(
-          lastAcceptedFrame,
+        const scoreFromPrevious = hybridFrameScore(
+          previousFrame,
           imageData,
           settings.pixelDeltaThreshold,
         );
+        const scoreFromWindow =
+          windowFrame && time - windowTime >= settings.minSecondsBetweenCaptures * 0.5
+            ? hybridFrameScore(windowFrame, imageData, settings.pixelDeltaThreshold)
+            : scoreFromPrevious;
+        const score = Math.max(scoreFromPrevious, scoreFromWindow);
 
-        const hasEnoughChange = diffRatio >= settings.changedPixelRatioThreshold;
-        const hasEnoughTimeGap =
-          lastAcceptedTime === null ||
-          time - lastAcceptedTime >= settings.minSecondsBetweenCaptures;
+        scoredFrames.push({
+          time,
+          score,
+        });
 
-        if (hasEnoughChange && hasEnoughTimeGap) {
-          const thumbnailUrl = await createThumbnailUrl(video);
-          candidates.push({
-            id: crypto.randomUUID(),
-            time,
-            score: diffRatio,
-            reason: "visual-change",
-            thumbnailUrl,
-          });
-          lastAcceptedFrame = imageData;
-          lastAcceptedTime = time;
+        previousFrame = imageData;
+        if (time - windowTime >= settings.minSecondsBetweenCaptures) {
+          windowFrame = imageData;
+          windowTime = time;
         }
       }
 
       callbacks.onProgress(Math.min(time / video.duration, 1));
-
-      if (candidates.length >= settings.maxCandidates) {
-        break;
-      }
     }
 
     callbacks.onProgress(1);
 
-    const ranked = rankAndTrimCandidates(candidates, settings.finalTargetCount, video.duration);
-    const rankedIds = new Set(ranked.map((candidate) => candidate.id));
-    candidates
-      .filter((candidate) => !rankedIds.has(candidate.id))
-      .forEach((candidate) => URL.revokeObjectURL(candidate.thumbnailUrl));
+    const targetVisualCount = settings.includeFirstFrame
+      ? Math.max(0, settings.finalTargetCount - 1)
+      : settings.finalTargetCount;
+    const selectedMoments = rankAndTrimCandidates(
+      scoredFrames,
+      targetVisualCount,
+      video.duration,
+      settings.minSecondsBetweenCaptures,
+    );
 
-    return ranked;
+    if (settings.includeFirstFrame) {
+      await seekVideo(video, 0);
+      selectedCandidates.push({
+        id: crypto.randomUUID(),
+        time: 0,
+        score: 1,
+        reason: "initial-frame",
+        thumbnailUrl: await createThumbnailUrl(video),
+      });
+    }
+
+    for (const moment of selectedMoments) {
+      await seekVideo(video, moment.time);
+      selectedCandidates.push({
+        id: crypto.randomUUID(),
+        time: moment.time,
+        score: moment.score,
+        reason: "visual-change",
+        thumbnailUrl: await createThumbnailUrl(video),
+      });
+    }
+
+    return selectedCandidates.sort((a, b) => a.time - b.time);
   } catch (error) {
-    candidates.forEach((candidate) => URL.revokeObjectURL(candidate.thumbnailUrl));
+    selectedCandidates.forEach((candidate) => URL.revokeObjectURL(candidate.thumbnailUrl));
     throw error;
   }
 }