feat: enhance image processing with new seam repair algorithms and improve blending strength

This commit is contained in:
Ben
2026-05-15 02:08:06 -07:00
parent 67260d78e7
commit 3af3aa0558
+529 -3
View File
@@ -19,7 +19,7 @@ export const defaultSettings: TilesetSettings = {
outputHeight: 1024,
resizeMode: 'crop-square',
seamWidth: 32,
blendStrength: 70,
blendStrength: 85,
detailPreservation: 'medium',
webpQuality: 90,
previewMode: 'repeat-3',
@@ -40,6 +40,62 @@ function makeCanvas(w: number, h: number): HTMLCanvasElement {
return c;
}
function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}
function pixelIndex(w: number, x: number, y: number): number {
return (y * w + x) * 4;
}
function colorDistanceSq(
a: Uint8ClampedArray,
ai: number,
b: Uint8ClampedArray,
bi: number,
): number {
const dr = a[ai] - b[bi];
const dg = a[ai + 1] - b[bi + 1];
const db = a[ai + 2] - b[bi + 2];
return dr * dr + dg * dg + db * db;
}
function copyImageData(data: ImageData): ImageData {
return new ImageData(new Uint8ClampedArray(data.data), data.width, data.height);
}
function blurImageData(data: Uint8ClampedArray, w: number, h: number, radius: number): Uint8ClampedArray {
const out = new Uint8ClampedArray(data.length);
const size = radius * 2 + 1;
const area = size * size;
for (let y = 0; y < h; y++) {
for (let x = 0; x < w; x++) {
const idx = pixelIndex(w, x, y);
let r = 0, g = 0, b = 0, a = 0;
for (let oy = -radius; oy <= radius; oy++) {
const sy = clamp(y + oy, 0, h - 1);
for (let ox = -radius; ox <= radius; ox++) {
const sx = clamp(x + ox, 0, w - 1);
const sidx = pixelIndex(w, sx, sy);
r += data[sidx];
g += data[sidx + 1];
b += data[sidx + 2];
a += data[sidx + 3];
}
}
out[idx] = Math.round(r / area);
out[idx + 1] = Math.round(g / area);
out[idx + 2] = Math.round(b / area);
out[idx + 3] = Math.round(a / area);
}
}
return out;
}
// ── Step 1+2: Load & Normalize ───────────────────────────────────────────────
export async function loadAndNormalizeImage(
@@ -134,7 +190,7 @@ export function applyWrappedOffset(
// ── Step 4: Seam Repair ──────────────────────────────────────────────────────
export function repairCenterSeams(
function repairCenterSeamsWithBlend(
source: HTMLCanvasElement,
original: HTMLCanvasElement,
settings: Pick<TilesetSettings, 'seamWidth' | 'blendStrength' | 'detailPreservation'>,
@@ -203,6 +259,476 @@ export function repairCenterSeams(
return out;
}
type RepairSettings = Pick<TilesetSettings, 'seamWidth' | 'blendStrength' | 'detailPreservation'>;
interface VerticalPatch {
sx: number;
sy: number;
y: number;
h: number;
}
interface HorizontalPatch {
sx: number;
sy: number;
x: number;
w: number;
}
function detailAlphaMultiplier(detail: DetailPreservation): number {
if (detail === 'high') return 0.82;
if (detail === 'medium') return 0.92;
return 1;
}
function candidatePositions(
w: number,
h: number,
patchW: number,
patchH: number,
context: number,
seamWidth: number,
): Array<[number, number]> {
const maxX = w - patchW - context;
const maxY = h - patchH - context;
if (maxX < context || maxY < context) return [];
const cx = Math.floor(w / 2);
const cy = Math.floor(h / 2);
const avoid = Math.max(seamWidth * 2, 24);
const positions: Array<[number, number]> = [];
const cols = 8;
const rows = 8;
for (let row = 0; row < rows; row++) {
for (let col = 0; col < cols; col++) {
const jitterX = ((row * 37 + col * 17) % 100) / 100;
const jitterY = ((row * 23 + col * 31) % 100) / 100;
const x = Math.round(context + ((col + jitterX) / cols) * (maxX - context));
const y = Math.round(context + ((row + jitterY) / rows) * (maxY - context));
if (Math.abs((x + patchW / 2) - cx) < avoid && Math.abs((y + patchH / 2) - cy) < avoid) {
continue;
}
positions.push([x, y]);
}
}
positions.push([context, context], [maxX, context], [context, maxY], [maxX, maxY]);
return positions;
}
function findHorizontalCut(
existing: Uint8ClampedArray,
candidate: Uint8ClampedArray,
w: number,
bandX: number,
destY: number,
sampleX: number,
sampleY: number,
cutW: number,
cutH: number,
): number[] {
const costs = new Float64Array(cutW * cutH);
const prev = new Int16Array(cutW * cutH);
for (let x = 0; x < cutW; x++) {
for (let y = 0; y < cutH; y++) {
const idx = y * cutW + x;
const di = pixelIndex(w, bandX + x, destY + y);
const si = pixelIndex(w, sampleX + x, sampleY + y);
const localCost = colorDistanceSq(existing, di, candidate, si);
if (x === 0) {
costs[idx] = localCost;
prev[idx] = y;
continue;
}
let bestY = y;
let best = costs[y * cutW + x - 1];
if (y > 0 && costs[(y - 1) * cutW + x - 1] < best) {
best = costs[(y - 1) * cutW + x - 1];
bestY = y - 1;
}
if (y < cutH - 1 && costs[(y + 1) * cutW + x - 1] < best) {
best = costs[(y + 1) * cutW + x - 1];
bestY = y + 1;
}
costs[idx] = localCost + best;
prev[idx] = bestY;
}
}
let bestEnd = 0;
let bestCost = costs[(cutW - 1)];
for (let y = 1; y < cutH; y++) {
const c = costs[y * cutW + cutW - 1];
if (c < bestCost) {
bestCost = c;
bestEnd = y;
}
}
const path = new Array<number>(cutW);
path[cutW - 1] = bestEnd;
for (let x = cutW - 1; x > 0; x--) {
path[x - 1] = prev[path[x] * cutW + x];
}
return path;
}
function findVerticalCut(
existing: Uint8ClampedArray,
candidate: Uint8ClampedArray,
w: number,
bandY: number,
destX: number,
sampleX: number,
sampleY: number,
cutW: number,
cutH: number,
): number[] {
const costs = new Float64Array(cutW * cutH);
const prev = new Int16Array(cutW * cutH);
for (let y = 0; y < cutH; y++) {
for (let x = 0; x < cutW; x++) {
const idx = y * cutW + x;
const di = pixelIndex(w, destX + x, bandY + y);
const si = pixelIndex(w, sampleX + x, sampleY + y);
const localCost = colorDistanceSq(existing, di, candidate, si);
if (y === 0) {
costs[idx] = localCost;
prev[idx] = x;
continue;
}
let bestX = x;
let best = costs[(y - 1) * cutW + x];
if (x > 0 && costs[(y - 1) * cutW + x - 1] < best) {
best = costs[(y - 1) * cutW + x - 1];
bestX = x - 1;
}
if (x < cutW - 1 && costs[(y - 1) * cutW + x + 1] < best) {
best = costs[(y - 1) * cutW + x + 1];
bestX = x + 1;
}
costs[idx] = localCost + best;
prev[idx] = bestX;
}
}
let bestEnd = 0;
let bestCost = costs[(cutH - 1) * cutW];
for (let x = 1; x < cutW; x++) {
const c = costs[(cutH - 1) * cutW + x];
if (c < bestCost) {
bestCost = c;
bestEnd = x;
}
}
const path = new Array<number>(cutH);
path[cutH - 1] = bestEnd;
for (let y = cutH - 1; y > 0; y--) {
path[y - 1] = prev[y * cutW + path[y]];
}
return path;
}
function scoreVerticalPatch(
existing: Uint8ClampedArray,
candidate: Uint8ClampedArray,
w: number,
bandX: number,
bandW: number,
patch: VerticalPatch,
context: number,
overlap: number,
): number {
let score = 0;
let samples = 0;
const stepY = Math.max(1, Math.floor(patch.h / 24));
const stepX = Math.max(1, Math.floor(bandW / 12));
const stepC = Math.max(1, Math.floor(context / 4));
for (let yy = 0; yy < patch.h; yy += stepY) {
const y = patch.y + yy;
const sy = patch.sy + yy;
for (let c = 1; c <= context; c += stepC) {
const leftIdx = pixelIndex(w, bandX - c, y);
const sampleLeftIdx = pixelIndex(w, patch.sx - c, sy);
const rightIdx = pixelIndex(w, bandX + bandW - 1 + c, y);
const sampleRightIdx = pixelIndex(w, patch.sx + bandW - 1 + c, sy);
score += colorDistanceSq(existing, leftIdx, candidate, sampleLeftIdx);
score += colorDistanceSq(existing, rightIdx, candidate, sampleRightIdx);
samples += 2;
}
}
if (patch.y > 0) {
const cutH = Math.min(overlap, patch.h);
for (let yy = 0; yy < cutH; yy += Math.max(1, Math.floor(cutH / 8))) {
for (let xx = 0; xx < bandW; xx += stepX) {
const di = pixelIndex(w, bandX + xx, patch.y + yy);
const si = pixelIndex(w, patch.sx + xx, patch.sy + yy);
score += colorDistanceSq(existing, di, candidate, si);
samples++;
}
}
}
return score / Math.max(1, samples);
}
function scoreHorizontalPatch(
existing: Uint8ClampedArray,
candidate: Uint8ClampedArray,
w: number,
bandY: number,
bandH: number,
patch: HorizontalPatch,
context: number,
overlap: number,
): number {
let score = 0;
let samples = 0;
const stepX = Math.max(1, Math.floor(patch.w / 24));
const stepY = Math.max(1, Math.floor(bandH / 12));
const stepC = Math.max(1, Math.floor(context / 4));
for (let xx = 0; xx < patch.w; xx += stepX) {
const x = patch.x + xx;
const sx = patch.sx + xx;
for (let c = 1; c <= context; c += stepC) {
const topIdx = pixelIndex(w, x, bandY - c);
const sampleTopIdx = pixelIndex(w, sx, patch.sy - c);
const bottomIdx = pixelIndex(w, x, bandY + bandH - 1 + c);
const sampleBottomIdx = pixelIndex(w, sx, patch.sy + bandH - 1 + c);
score += colorDistanceSq(existing, topIdx, candidate, sampleTopIdx);
score += colorDistanceSq(existing, bottomIdx, candidate, sampleBottomIdx);
samples += 2;
}
}
if (patch.x > 0) {
const cutW = Math.min(overlap, patch.w);
for (let xx = 0; xx < cutW; xx += Math.max(1, Math.floor(cutW / 8))) {
for (let yy = 0; yy < bandH; yy += stepY) {
const di = pixelIndex(w, patch.x + xx, bandY + yy);
const si = pixelIndex(w, patch.sx + xx, patch.sy + yy);
score += colorDistanceSq(existing, di, candidate, si);
samples++;
}
}
}
return score / Math.max(1, samples);
}
function blendPixelTwoBand(
dst: Uint8ClampedArray,
dstLow: Uint8ClampedArray,
sample: Uint8ClampedArray,
sampleLow: Uint8ClampedArray,
di: number,
si: number,
alpha: number,
): void {
const detailAlpha = Math.min(1, alpha * 1.12);
for (let ch = 0; ch < 3; ch++) {
const low = dstLow[di + ch] * (1 - alpha) + sampleLow[si + ch] * alpha;
const dstHigh = dst[di + ch] - dstLow[di + ch];
const sampleHigh = sample[si + ch] - sampleLow[si + ch];
dst[di + ch] = clamp(Math.round(low + dstHigh * (1 - detailAlpha) + sampleHigh * detailAlpha), 0, 255);
}
dst[di + 3] = Math.round(dst[di + 3] * (1 - alpha) + sample[si + 3] * alpha);
}
function quiltVerticalBand(
dst: Uint8ClampedArray,
dstLow: Uint8ClampedArray,
sample: Uint8ClampedArray,
sampleLow: Uint8ClampedArray,
w: number,
h: number,
settings: RepairSettings,
): boolean {
const bandW = clamp(Math.round(settings.seamWidth), 8, Math.floor(w / 3));
const bandX = Math.floor(w / 2) - Math.floor(bandW / 2);
const context = clamp(Math.round(bandW * 0.75), 6, 48);
const overlap = clamp(Math.round(bandW * 0.75), 6, 64);
const patchH = clamp(Math.round(bandW * 3), 48, Math.max(48, Math.floor(h / 2)));
const step = Math.max(8, patchH - overlap);
const influence = (settings.blendStrength / 100) * detailAlphaMultiplier(settings.detailPreservation);
if (bandX - context < 0 || bandX + bandW + context >= w || h < patchH + context * 2) {
return false;
}
let usedPatch = false;
for (let y = 0; y < h; y += step) {
const patch: VerticalPatch = { sx: 0, sy: 0, y, h: Math.min(patchH, h - y) };
const candidates = candidatePositions(w, h, bandW, patch.h, context, settings.seamWidth);
if (candidates.length === 0) return false;
let best: VerticalPatch | null = null;
let bestScore = Number.POSITIVE_INFINITY;
for (const [sx, sy] of candidates) {
const candidate = { ...patch, sx, sy };
const score = scoreVerticalPatch(dst, sample, w, bandX, bandW, candidate, context, overlap);
if (score < bestScore) {
bestScore = score;
best = candidate;
}
}
if (!best) return false;
const cutH = y > 0 ? Math.min(overlap, patch.h) : 0;
const cut = cutH > 1
? findHorizontalCut(dst, sample, w, bandX, y, best.sx, best.sy, bandW, cutH)
: null;
const edgeFeather = Math.max(2, Math.floor(bandW / 3));
const cutFeather = Math.max(2, Math.floor(overlap / 8));
for (let yy = 0; yy < patch.h; yy++) {
for (let xx = 0; xx < bandW; xx++) {
const di = pixelIndex(w, bandX + xx, y + yy);
const si = pixelIndex(w, best.sx + xx, best.sy + yy);
const edgeAlpha = Math.min(
smoothstep(0, edgeFeather, xx + 1),
smoothstep(0, edgeFeather, bandW - xx),
);
let alpha = influence * edgeAlpha;
if (cut) {
alpha *= smoothstep(cut[xx] - cutFeather, cut[xx] + cutFeather, yy);
}
if (alpha > 0) blendPixelTwoBand(dst, dstLow, sample, sampleLow, di, si, alpha);
}
}
usedPatch = true;
if (y + patch.h >= h) break;
}
return usedPatch;
}
function quiltHorizontalBand(
dst: Uint8ClampedArray,
dstLow: Uint8ClampedArray,
sample: Uint8ClampedArray,
sampleLow: Uint8ClampedArray,
w: number,
h: number,
settings: RepairSettings,
): boolean {
const bandH = clamp(Math.round(settings.seamWidth), 8, Math.floor(h / 3));
const bandY = Math.floor(h / 2) - Math.floor(bandH / 2);
const context = clamp(Math.round(bandH * 0.75), 6, 48);
const overlap = clamp(Math.round(bandH * 0.75), 6, 64);
const patchW = clamp(Math.round(bandH * 3), 48, Math.max(48, Math.floor(w / 2)));
const step = Math.max(8, patchW - overlap);
const influence = (settings.blendStrength / 100) * detailAlphaMultiplier(settings.detailPreservation);
if (bandY - context < 0 || bandY + bandH + context >= h || w < patchW + context * 2) {
return false;
}
let usedPatch = false;
for (let x = 0; x < w; x += step) {
const patch: HorizontalPatch = { sx: 0, sy: 0, x, w: Math.min(patchW, w - x) };
const candidates = candidatePositions(w, h, patch.w, bandH, context, settings.seamWidth);
if (candidates.length === 0) return false;
let best: HorizontalPatch | null = null;
let bestScore = Number.POSITIVE_INFINITY;
for (const [sx, sy] of candidates) {
const candidate = { ...patch, sx, sy };
const score = scoreHorizontalPatch(dst, sample, w, bandY, bandH, candidate, context, overlap);
if (score < bestScore) {
bestScore = score;
best = candidate;
}
}
if (!best) return false;
const cutW = x > 0 ? Math.min(overlap, patch.w) : 0;
const cut = cutW > 1
? findVerticalCut(dst, sample, w, bandY, x, best.sx, best.sy, cutW, bandH)
: null;
const edgeFeather = Math.max(2, Math.floor(bandH / 3));
const cutFeather = Math.max(2, Math.floor(overlap / 8));
for (let yy = 0; yy < bandH; yy++) {
for (let xx = 0; xx < patch.w; xx++) {
const di = pixelIndex(w, x + xx, bandY + yy);
const si = pixelIndex(w, best.sx + xx, best.sy + yy);
const edgeAlpha = Math.min(
smoothstep(0, edgeFeather, yy + 1),
smoothstep(0, edgeFeather, bandH - yy),
);
let alpha = influence * edgeAlpha;
if (cut) {
alpha *= smoothstep(cut[yy] - cutFeather, cut[yy] + cutFeather, xx);
}
if (alpha > 0) blendPixelTwoBand(dst, dstLow, sample, sampleLow, di, si, alpha);
}
}
usedPatch = true;
if (x + patch.w >= w) break;
}
return usedPatch;
}
export function repairCenterSeams(
source: HTMLCanvasElement,
original: HTMLCanvasElement,
settings: RepairSettings,
): HTMLCanvasElement {
const { width: w, height: h } = source;
const minimumQuiltSize = Math.max(96, settings.seamWidth * 4);
if (w < minimumQuiltSize || h < minimumQuiltSize) {
return repairCenterSeamsWithBlend(source, original, settings);
}
const out = makeCanvas(w, h);
const outCtx = out.getContext('2d')!;
outCtx.drawImage(source, 0, 0);
const outData = outCtx.getImageData(0, 0, w, h);
const sourceData = source.getContext('2d')!.getImageData(0, 0, w, h);
const sampleData = original.getContext('2d')!.getImageData(0, 0, w, h);
const dst = outData.data;
const dstLow = blurImageData(sourceData.data, w, h, 2);
const sampleLow = blurImageData(sampleData.data, w, h, 2);
const verticalOk = quiltVerticalBand(dst, dstLow, sampleData.data, sampleLow, w, h, settings);
const afterVertical = copyImageData(outData);
const afterVerticalLow = blurImageData(afterVertical.data, w, h, 2);
const horizontalOk = quiltHorizontalBand(dst, afterVerticalLow, sampleData.data, sampleLow, w, h, settings);
if (!verticalOk || !horizontalOk) {
return repairCenterSeamsWithBlend(source, original, settings);
}
outCtx.putImageData(outData, 0, 0);
return out;
}
// ── Full Pipeline ────────────────────────────────────────────────────────────
export async function generateTileableTexture(
@@ -223,7 +749,7 @@ export async function generateTileableTexture(
settings.outputHeight / 2,
);
onProgress?.('Repairing seams…');
onProgress?.('Quilting seam patches…');
// Run seam repair in next microtask so the UI can update
await new Promise<void>((res) => setTimeout(res, 0));
const repaired = repairCenterSeams(offsetCanvas, normalized, settings);