feat: implement scoring and ranking for video frames, enhance candidate selection process

.dockerignore

@@ -0,0 +1,5 @@
+node_modules
+dist
+.git
+.DS_Store
+npm-debug.log*

Dockerfile

@@ -0,0 +1,12 @@
+FROM node:22-alpine AS build
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm ci
+
+COPY . .
+RUN npm run build
+
+FROM nginx:1.27-alpine
+COPY --from=build /app/dist /usr/share/nginx/html
+EXPOSE 80

captain-definition

@@ -0,0 +1,4 @@
+{
+  "schemaVersion": 2,
+  "dockerfilePath": "./Dockerfile"
+}

src/analysis/candidateRanking.ts

@@ -1,34 +1,91 @@
-import type { CandidateFrame } from "../types/scan";
+export type ScoredFrame = {
+  score: number;
+  time: number;
+};
 
 export function rankAndTrimCandidates(
-  candidates: CandidateFrame[],
+  candidates: ScoredFrame[],
   targetCount: number,
   duration: number,
-): CandidateFrame[] {
+  minGapSeconds: number,
-  if (candidates.length <= targetCount) {
+): ScoredFrame[] {
-    return [...candidates].sort((a, b) => a.time - b.time);
+  if (targetCount <= 0 || candidates.length === 0) {
+    return [];
   }
 
+  const uniqueCandidates = dedupeByRoundedTime(candidates);
+  if (uniqueCandidates.length <= targetCount) {
+    return [...uniqueCandidates].sort((a, b) => a.time - b.time);
+  }
+
+  const scoreFloor = Math.max(percentile(uniqueCandidates, 0.6) * 0.6, 0.02);
+  const localPeaks = uniqueCandidates.filter((candidate, index, items) => {
+    const previous = items[index - 1];
+    const next = items[index + 1];
+    const higherThanPrevious = !previous || candidate.score >= previous.score;
+    const higherThanNext = !next || candidate.score > next.score;
+    return candidate.score >= scoreFloor && higherThanPrevious && higherThanNext;
+  });
+
+  const pool =
+    localPeaks.length >= targetCount / 2
+      ? localPeaks
+      : [...localPeaks, ...uniqueCandidates.filter((candidate) => !localPeaks.includes(candidate))];
+
   const bucketCount = Math.max(1, targetCount);
   const bucketSize = Math.max(duration / bucketCount, 1);
-  const selected = new Map<string, CandidateFrame>();
+  const selected = new Map<string, ScoredFrame>();
 
   for (let bucket = 0; bucket < bucketCount; bucket++) {
     const start = bucket * bucketSize;
     const end = bucket === bucketCount - 1 ? duration + 0.001 : start + bucketSize;
-    const best = candidates
+    const best = pool
       .filter((candidate) => candidate.time >= start && candidate.time < end)
       .sort((a, b) => b.score - a.score)[0];
 
-    if (best) {
+    if (best && respectsMinGap([...selected.values()], best, minGapSeconds)) {
-      selected.set(best.id, best);
+      selected.set(frameKey(best.time), best);
     }
   }
 
-  for (const candidate of [...candidates].sort((a, b) => b.score - a.score)) {
+  for (const candidate of [...pool].sort((a, b) => b.score - a.score)) {
     if (selected.size >= targetCount) break;
-    selected.set(candidate.id, candidate);
+    if (respectsMinGap([...selected.values()], candidate, minGapSeconds)) {
+      selected.set(frameKey(candidate.time), candidate);
+    }
+  }
+
+  for (const candidate of [...uniqueCandidates].sort((a, b) => b.score - a.score)) {
+    if (selected.size >= targetCount) break;
+    if (respectsMinGap([...selected.values()], candidate, minGapSeconds * 0.5)) {
+      selected.set(frameKey(candidate.time), candidate);
+    }
   }
 
   return [...selected.values()].sort((a, b) => a.time - b.time);
 }
+
+function percentile(candidates: ScoredFrame[], quantile: number) {
+  const scores = candidates.map((candidate) => candidate.score).sort((a, b) => a - b);
+  const index = Math.min(scores.length - 1, Math.max(0, Math.floor((scores.length - 1) * quantile)));
+  return scores[index];
+}
+
+function dedupeByRoundedTime(candidates: ScoredFrame[]) {
+  const selected = new Map<string, ScoredFrame>();
+  for (const candidate of [...candidates].sort((a, b) => b.score - a.score)) {
+    const key = frameKey(candidate.time);
+    if (!selected.has(key)) {
+      selected.set(key, candidate);
+    }
+  }
+  return [...selected.values()].sort((a, b) => a.time - b.time);
+}
+
+function frameKey(time: number) {
+  return time.toFixed(2);
+}
+
+function respectsMinGap(selected: ScoredFrame[], candidate: ScoredFrame, minGapSeconds: number) {
+  return selected.every((item) => Math.abs(item.time - candidate.time) >= minGapSeconds);
+}

src/analysis/frameDifference.ts

@@ -19,3 +19,35 @@ export function frameDifferenceRatio(
 
   return changed / pixels;
 }
+
+export function histogramDifferenceRatio(a: ImageData, b: ImageData, bins = 16): number {
+  const histogramA = new Array<number>(bins).fill(0);
+  const histogramB = new Array<number>(bins).fill(0);
+  const dataA = a.data;
+  const dataB = b.data;
+  const pixels = dataA.length / 4;
+
+  for (let i = 0; i < dataA.length; i += 4) {
+    const lumA = 0.299 * dataA[i] + 0.587 * dataA[i + 1] + 0.114 * dataA[i + 2];
+    const lumB = 0.299 * dataB[i] + 0.587 * dataB[i + 1] + 0.114 * dataB[i + 2];
+    histogramA[Math.min(bins - 1, Math.floor((lumA / 256) * bins))]++;
+    histogramB[Math.min(bins - 1, Math.floor((lumB / 256) * bins))]++;
+  }
+
+  let difference = 0;
+  for (let index = 0; index < bins; index++) {
+    difference += Math.abs(histogramA[index] - histogramB[index]);
+  }
+
+  return difference / (pixels * 2);
+}
+
+export function hybridFrameScore(
+  a: ImageData,
+  b: ImageData,
+  pixelDeltaThreshold: number,
+): number {
+  const pixelDifference = frameDifferenceRatio(a, b, pixelDeltaThreshold);
+  const histogramDifference = histogramDifferenceRatio(a, b);
+  return histogramDifference * 0.7 + pixelDifference * 0.3;
+}

src/analysis/scanVideo.ts

@@ -1,5 +1,5 @@
 import { rankAndTrimCandidates } from "./candidateRanking";
-import { frameDifferenceRatio } from "./frameDifference";
+import { hybridFrameScore } from "./frameDifference";
 import { captureAnalysisFrame, createThumbnailUrl } from "../media/canvas";
 import { seekVideo } from "../media/video";
 import type { CandidateFrame, ScanSettings } from "../types/scan";
@@ -15,9 +15,11 @@ export async function scanVideoForCandidates(
   callbacks: ScanCallbacks,
 ): Promise<CandidateFrame[]> {
   const analysisCanvas = document.createElement("canvas");
-  const candidates: CandidateFrame[] = [];
+  const scoredFrames: Array<{ score: number; time: number }> = [];
-  let lastAcceptedFrame: ImageData | null = null;
+  const selectedCandidates: CandidateFrame[] = [];
-  let lastAcceptedTime: number | null = null;
+  let previousFrame: ImageData | null = null;
+  let windowFrame: ImageData | null = null;
+  let windowTime = 0;
 
   try {
     for (
@@ -38,64 +40,74 @@ export async function scanVideoForCandidates(
         settings.analysisHeight,
       );
 
-      if (!lastAcceptedFrame) {
+      if (!previousFrame) {
-        lastAcceptedFrame = imageData;
+        previousFrame = imageData;
-
+        windowFrame = imageData;
-        if (settings.includeFirstFrame) {
+        windowTime = time;
-          const thumbnailUrl = await createThumbnailUrl(video);
-          candidates.push({
-            id: crypto.randomUUID(),
-            time,
-            score: 1,
-            reason: "initial-frame",
-            thumbnailUrl,
-          });
-          lastAcceptedTime = time;
-        }
       } else {
-        const diffRatio = frameDifferenceRatio(
+        const scoreFromPrevious = hybridFrameScore(
-          lastAcceptedFrame,
+          previousFrame,
           imageData,
           settings.pixelDeltaThreshold,
         );
+        const scoreFromWindow =
+          windowFrame && time - windowTime >= settings.minSecondsBetweenCaptures * 0.5
+            ? hybridFrameScore(windowFrame, imageData, settings.pixelDeltaThreshold)
+            : scoreFromPrevious;
+        const score = Math.max(scoreFromPrevious, scoreFromWindow);
 
-        const hasEnoughChange = diffRatio >= settings.changedPixelRatioThreshold;
+        scoredFrames.push({
-        const hasEnoughTimeGap =
+          time,
-          lastAcceptedTime === null ||
+          score,
-          time - lastAcceptedTime >= settings.minSecondsBetweenCaptures;
+        });
 
-        if (hasEnoughChange && hasEnoughTimeGap) {
+        previousFrame = imageData;
-          const thumbnailUrl = await createThumbnailUrl(video);
+        if (time - windowTime >= settings.minSecondsBetweenCaptures) {
-          candidates.push({
+          windowFrame = imageData;
-            id: crypto.randomUUID(),
+          windowTime = time;
-            time,
-            score: diffRatio,
-            reason: "visual-change",
-            thumbnailUrl,
-          });
-          lastAcceptedFrame = imageData;
-          lastAcceptedTime = time;
         }
       }
 
       callbacks.onProgress(Math.min(time / video.duration, 1));
-
-      if (candidates.length >= settings.maxCandidates) {
-        break;
-      }
     }
 
     callbacks.onProgress(1);
 
-    const ranked = rankAndTrimCandidates(candidates, settings.finalTargetCount, video.duration);
+    const targetVisualCount = settings.includeFirstFrame
-    const rankedIds = new Set(ranked.map((candidate) => candidate.id));
+      ? Math.max(0, settings.finalTargetCount - 1)
-    candidates
+      : settings.finalTargetCount;
-      .filter((candidate) => !rankedIds.has(candidate.id))
+    const selectedMoments = rankAndTrimCandidates(
-      .forEach((candidate) => URL.revokeObjectURL(candidate.thumbnailUrl));
+      scoredFrames,
+      targetVisualCount,
+      video.duration,
+      settings.minSecondsBetweenCaptures,
+    );
 
-    return ranked;
+    if (settings.includeFirstFrame) {
+      await seekVideo(video, 0);
+      selectedCandidates.push({
+        id: crypto.randomUUID(),
+        time: 0,
+        score: 1,
+        reason: "initial-frame",
+        thumbnailUrl: await createThumbnailUrl(video),
+      });
+    }
+
+    for (const moment of selectedMoments) {
+      await seekVideo(video, moment.time);
+      selectedCandidates.push({
+        id: crypto.randomUUID(),
+        time: moment.time,
+        score: moment.score,
+        reason: "visual-change",
+        thumbnailUrl: await createThumbnailUrl(video),
+      });
+    }
+
+    return selectedCandidates.sort((a, b) => a.time - b.time);
   } catch (error) {
-    candidates.forEach((candidate) => URL.revokeObjectURL(candidate.thumbnailUrl));
+    selectedCandidates.forEach((candidate) => URL.revokeObjectURL(candidate.thumbnailUrl));
     throw error;
   }
 }