docs(project): document DUPLICATE_HASH_THRESHOLD in .env.example

The dedup pairs rebuild reads this tunable (default 10/64); it was added to the
backend config but never documented for operators. No other new env vars were
introduced by duplicate detection — the dedup compose service reuses the
existing PUID/PGID/FILES_DIR/THUMBS_DIR.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -125,6 +125,16 @@ THUMB_CONCURRENCY=0
 # ---------------------------------------------------------------------------
 IMPORT_PATH=/data/import
 
+# ---------------------------------------------------------------------------
+# Duplicate detection
+# ---------------------------------------------------------------------------
+# Maximum perceptual-hash distance (Hamming, out of 64 bits) for two files to be
+# treated as duplicate candidates. Lower = stricter (fewer, more confident
+# matches); higher = looser (catches more re-encodes/resizes but risks false
+# positives). Used only by the dedup tool's pairs rebuild — see the dedup CLI /
+# `docker compose run --rm dedup`. Default 10.
+DUPLICATE_HASH_THRESHOLD=10
+
 # ---------------------------------------------------------------------------
 # Static SPA
 # ---------------------------------------------------------------------------

Dockerfile

@@ -44,6 +44,9 @@ COPY backend/ ./
 # metadata) and falls back to pure-Go image processing (disintegration/imaging)
 # when vips is absent, so it stays fully static and portable across base images.
 RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
+# dedup: offline maintenance CLI for duplicate detection (hash backfill + pairs
+# rescan). Shipped alongside the server so it can be run with `docker exec`.
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/dedup ./cmd/dedup
 
 # -----------------------------------------------------------------------------
 # Stage 3 — minimal runtime
@@ -68,6 +71,8 @@ WORKDIR /app
 COPY --from=frontend --chown=tanabata:tanabata /src/frontend/build /app/static
 # The server binary.
 COPY --from=backend --chown=tanabata:tanabata /out/server /app/server
+# The dedup maintenance CLI (run via `docker exec`, not the entrypoint).
+COPY --from=backend --chown=tanabata:tanabata /out/dedup /app/dedup
 
 # Data directories (overridable via FILES_PATH/THUMBS_CACHE_PATH/IMPORT_PATH).
 # Created and owned by the tanabata user so a fresh named volume inherits write access.

backend/cmd/dedup/main.go

@@ -0,0 +1,165 @@
+// Command dedup is the offline maintenance tool for duplicate detection. It runs
+// in two phases:
+//
+//	hashes — compute the perceptual hash of every live image/video that has none
+//	         yet (images from their bytes, videos from a middle frame via ffmpeg).
+//	pairs  — rebuild data.duplicate_pairs from all current hashes.
+//
+// Both phases run by default; pass -hashes or -pairs to run only one. It reuses
+// the server's configuration (DATABASE_URL, FILES_PATH, THUMBS_CACHE_PATH, …) and
+// is safe to re-run: hashing only touches files whose phash is NULL, and the
+// pairs rebuild is a full replace.
+//
+//	go run ./cmd/dedup            # hashes, then pairs
+//	go run ./cmd/dedup -pairs      # only rebuild pairs
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/google/uuid"
+
+	"tanabata/backend/internal/config"
+	"tanabata/backend/internal/db/postgres"
+	"tanabata/backend/internal/imagehash"
+	"tanabata/backend/internal/service"
+	"tanabata/backend/internal/storage"
+)
+
+func main() {
+	hashesOnly := flag.Bool("hashes", false, "only (re)compute missing perceptual hashes")
+	pairsOnly := flag.Bool("pairs", false, "only rebuild the duplicate pairs table")
+	flag.Parse()
+
+	// No flag, or both, means run everything.
+	doHashes := *hashesOnly || !*pairsOnly
+	doPairs := *pairsOnly || !*hashesOnly
+
+	cfg, err := config.Load()
+	if err != nil {
+		fatal("load config", err)
+	}
+
+	ctx := context.Background()
+	pool, err := postgres.NewPool(ctx, cfg.DatabaseURL)
+	if err != nil {
+		fatal("connect to database", err)
+	}
+	defer pool.Close()
+
+	diskStorage, err := storage.NewDiskStorage(
+		cfg.FilesPath, cfg.ThumbsCachePath,
+		cfg.ThumbWidth, cfg.ThumbHeight,
+		cfg.PreviewWidth, cfg.PreviewHeight,
+		cfg.ThumbMaxPixels, cfg.ThumbConcurrency,
+	)
+	if err != nil {
+		fatal("init storage", err)
+	}
+
+	fileRepo := postgres.NewFileRepo(pool)
+	pairRepo := postgres.NewDuplicatePairRepo(pool)
+	dismissalRepo := postgres.NewDismissalRepo(pool)
+	aclRepo := postgres.NewACLRepo(pool)
+	auditRepo := postgres.NewAuditRepo(pool)
+	tagRepo := postgres.NewTagRepo(pool)
+	categoryRepo := postgres.NewCategoryRepo(pool)
+	poolRepo := postgres.NewPoolRepo(pool)
+	transactor := postgres.NewTransactor(pool)
+
+	aclSvc := service.NewACLService(aclRepo, fileRepo, tagRepo, categoryRepo, poolRepo, transactor)
+	auditSvc := service.NewAuditService(auditRepo)
+	dupSvc := service.NewDuplicateService(
+		fileRepo, pairRepo, dismissalRepo, aclSvc, auditSvc, transactor, cfg.DuplicateHashThreshold,
+	)
+
+	if doHashes {
+		if err := backfillHashes(ctx, fileRepo, diskStorage); err != nil {
+			fatal("backfill hashes", err)
+		}
+	}
+	if doPairs {
+		fmt.Printf("rebuilding duplicate pairs (threshold %d)...\n", cfg.DuplicateHashThreshold)
+		if err := dupSvc.Rescan(ctx, func(done, total int) {
+			fmt.Printf("\r  hashed %d/%d", done, total)
+		}); err != nil {
+			fatal("rescan pairs", err)
+		}
+		fmt.Println("\n  done")
+	}
+}
+
+// backfillHashes computes and stores a perceptual hash for every live image/video
+// that lacks one. Failures on individual files are counted and reported, not
+// fatal, so one unreadable file doesn't abort the whole run.
+func backfillHashes(ctx context.Context, files *postgres.FileRepo, store *storage.DiskStorage) error {
+	pending, err := files.ListMissingPHash(ctx)
+	if err != nil {
+		return err
+	}
+	total := len(pending)
+	fmt.Printf("hashing %d files without a perceptual hash...\n", total)
+
+	var hashed, skipped, failed int
+	for i, f := range pending {
+		ph, err := hashOne(ctx, store, f.ID, f.MIMEType)
+		switch {
+		case err != nil:
+			failed++
+			fmt.Fprintf(os.Stderr, "\n  %s (%s): %v\n", f.ID, f.MIMEType, err)
+		case ph == nil:
+			skipped++ // not decodable; leave phash NULL
+		default:
+			if err := files.SetPHash(ctx, f.ID, ph); err != nil {
+				return fmt.Errorf("set phash for %s: %w", f.ID, err)
+			}
+			hashed++
+		}
+		if (i+1)%200 == 0 || i+1 == total {
+			fmt.Printf("\r  processed %d/%d", i+1, total)
+		}
+	}
+	fmt.Printf("\n  hashed %d, skipped %d, failed %d\n", hashed, skipped, failed)
+	return nil
+}
+
+// hashOne returns the perceptual hash for one file, or nil when it isn't hashable
+// (e.g. an image that won't decode). Images are hashed from their bytes; videos
+// from a middle frame.
+func hashOne(ctx context.Context, store *storage.DiskStorage, id uuid.UUID, mime string) (*int64, error) {
+	switch {
+	case strings.HasPrefix(mime, "image/"):
+		rc, err := store.Read(ctx, id)
+		if err != nil {
+			return nil, err
+		}
+		defer rc.Close()
+		data, err := io.ReadAll(rc)
+		if err != nil {
+			return nil, err
+		}
+		if h, ok := imagehash.FromBytes(data); ok {
+			return &h, nil
+		}
+		return nil, nil
+	case strings.HasPrefix(mime, "video/"):
+		img, err := store.VideoFrameMiddle(ctx, id)
+		if err != nil {
+			return nil, err
+		}
+		h := imagehash.FromImage(img)
+		return &h, nil
+	default:
+		return nil, nil
+	}
+}
+
+func fatal(what string, err error) {
+	fmt.Fprintf(os.Stderr, "dedup: %s: %v\n", what, err)
+	os.Exit(1)
+}

backend/cmd/server/main.go

@@ -70,6 +70,8 @@ func main() {
 	tagRuleRepo := postgres.NewTagRuleRepo(pool)
 	categoryRepo := postgres.NewCategoryRepo(pool)
 	poolRepo := postgres.NewPoolRepo(pool)
+	duplicatePairRepo := postgres.NewDuplicatePairRepo(pool)
+	dismissalRepo := postgres.NewDismissalRepo(pool)
 	transactor := postgres.NewTransactor(pool)
 
 	// Services
@@ -86,6 +88,9 @@ func main() {
 	tagSvc := service.NewTagService(tagRepo, tagRuleRepo, aclSvc, auditSvc, transactor)
 	categorySvc := service.NewCategoryService(categoryRepo, tagRepo, aclSvc, auditSvc)
 	poolSvc := service.NewPoolService(poolRepo, aclSvc, auditSvc)
+	duplicateSvc := service.NewDuplicateService(
+		fileRepo, duplicatePairRepo, dismissalRepo, aclSvc, auditSvc, transactor, cfg.DuplicateHashThreshold,
+	)
 	fileSvc := service.NewFileService(
 		fileRepo,
 		mimeRepo,
@@ -108,6 +113,7 @@ func main() {
 	authMiddleware := handler.NewAuthMiddleware(authSvc)
 	authHandler := handler.NewAuthHandler(authSvc)
 	fileHandler := handler.NewFileHandler(fileSvc, tagSvc, authSvc, cfg.MaxUploadBytes)
+	duplicateHandler := handler.NewDuplicateHandler(duplicateSvc)
 	tagHandler := handler.NewTagHandler(tagSvc, fileSvc)
 	categoryHandler := handler.NewCategoryHandler(categorySvc)
 	poolHandler := handler.NewPoolHandler(poolSvc)
@@ -117,7 +123,7 @@ func main() {
 
 	r, err := handler.NewRouter(
 		authMiddleware, authHandler,
-		fileHandler, tagHandler, categoryHandler, poolHandler,
+		fileHandler, duplicateHandler, tagHandler, categoryHandler, poolHandler,
 		userHandler, aclHandler, auditHandler,
 		cfg.StaticDir,
 		cfg.TrustedProxies,

backend/internal/config/config.go

@@ -63,6 +63,12 @@ type Config struct {
 	// Import
 	ImportPath string
 
+	// DuplicateHashThreshold is the maximum Hamming distance (out of 64) between
+	// two perceptual hashes for the files to be treated as duplicate candidates.
+	// Lower = stricter (fewer, more confident matches); higher = looser. Used only
+	// by the dedup rescan that (re)builds data.duplicate_pairs.
+	DuplicateHashThreshold int
+
 	// Static SPA. When set, the server serves the built frontend (and falls
 	// back to index.html for client routes) on the same port as the API. Empty
 	// in local development, where the Vite dev server serves the UI separately.
@@ -176,6 +182,8 @@ func Load() (*Config, error) {
 
 		ImportPath: requireStr("IMPORT_PATH"),
 
+		DuplicateHashThreshold: parseInt("DUPLICATE_HASH_THRESHOLD", 10),
+
 		StaticDir: defaultStr("STATIC_DIR", ""),
 	}
 

backend/internal/db/postgres/duplicate_repo.go

@@ -0,0 +1,145 @@
+package postgres
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"tanabata/backend/internal/domain"
+	"tanabata/backend/internal/port"
+)
+
+// ---------------------------------------------------------------------------
+// DuplicatePairRepo
+// ---------------------------------------------------------------------------
+
+// DuplicatePairRepo implements port.DuplicatePairRepo using PostgreSQL.
+type DuplicatePairRepo struct {
+	pool *pgxpool.Pool
+}
+
+// NewDuplicatePairRepo creates a DuplicatePairRepo backed by pool.
+func NewDuplicatePairRepo(pool *pgxpool.Pool) *DuplicatePairRepo {
+	return &DuplicatePairRepo{pool: pool}
+}
+
+var _ port.DuplicatePairRepo = (*DuplicatePairRepo)(nil)
+
+// ReplaceAll atomically replaces the entire pairs table with the given set.
+// The rescan recomputes pairs from scratch, so a full DELETE + COPY is both
+// correct and the simplest way to drop pairs that no longer qualify.
+func (r *DuplicatePairRepo) ReplaceAll(ctx context.Context, pairs []domain.DuplicatePair) error {
+	tx, err := r.pool.Begin(ctx)
+	if err != nil {
+		return fmt.Errorf("DuplicatePairRepo.ReplaceAll begin: %w", err)
+	}
+	defer tx.Rollback(ctx) //nolint:errcheck // no-op after a successful commit
+
+	if _, err := tx.Exec(ctx, `DELETE FROM data.duplicate_pairs`); err != nil {
+		return fmt.Errorf("DuplicatePairRepo.ReplaceAll delete: %w", err)
+	}
+
+	if len(pairs) > 0 {
+		rows := make([][]any, len(pairs))
+		for i, p := range pairs {
+			rows[i] = []any{p.FileA, p.FileB, int16(p.Distance)}
+		}
+		if _, err := tx.CopyFrom(ctx,
+			pgx.Identifier{"data", "duplicate_pairs"},
+			[]string{"file_a", "file_b", "distance"},
+			pgx.CopyFromRows(rows),
+		); err != nil {
+			return fmt.Errorf("DuplicatePairRepo.ReplaceAll copy: %w", err)
+		}
+	}
+
+	if err := tx.Commit(ctx); err != nil {
+		return fmt.Errorf("DuplicatePairRepo.ReplaceAll commit: %w", err)
+	}
+	return nil
+}
+
+type pairRow struct {
+	FileA    uuid.UUID `db:"file_a"`
+	FileB    uuid.UUID `db:"file_b"`
+	Distance int16     `db:"distance"`
+}
+
+// ListVisible returns every stored pair where both files are live (not trashed),
+// the pair is not dismissed, and — for non-admins — both files are visible to the
+// viewer under the private-by-default model. This is the input to clustering.
+func (r *DuplicatePairRepo) ListVisible(ctx context.Context, viewerID int16, isAdmin bool) ([]domain.DuplicatePair, error) {
+	args := make([]any, 0, 4)
+	n := 1
+	aclWhere := ""
+	if !isAdmin {
+		var ca, cb string
+		ca, n, args = aclVisibilityCond("fa", objTypeFile, viewerID, n, args)
+		cb, n, args = aclVisibilityCond("fb", objTypeFile, viewerID, n, args)
+		aclWhere = "AND " + ca + " AND " + cb
+	}
+
+	sqlStr := fmt.Sprintf(`
+        SELECT p.file_a, p.file_b, p.distance
+        FROM data.duplicate_pairs p
+        JOIN data.files fa ON fa.id = p.file_a AND fa.is_deleted = false
+        JOIN data.files fb ON fb.id = p.file_b AND fb.is_deleted = false
+        WHERE NOT EXISTS (
+            SELECT 1 FROM data.duplicate_dismissals d
+            WHERE d.file_a = p.file_a AND d.file_b = p.file_b
+        )
+        %s
+        ORDER BY p.file_a, p.file_b`, aclWhere)
+
+	rows, err := r.pool.Query(ctx, sqlStr, args...)
+	if err != nil {
+		return nil, fmt.Errorf("DuplicatePairRepo.ListVisible: %w", err)
+	}
+	collected, err := pgx.CollectRows(rows, pgx.RowToStructByName[pairRow])
+	if err != nil {
+		return nil, fmt.Errorf("DuplicatePairRepo.ListVisible scan: %w", err)
+	}
+	out := make([]domain.DuplicatePair, len(collected))
+	for i, row := range collected {
+		out[i] = domain.DuplicatePair{FileA: row.FileA, FileB: row.FileB, Distance: int(row.Distance)}
+	}
+	return out, nil
+}
+
+// ---------------------------------------------------------------------------
+// DismissalRepo
+// ---------------------------------------------------------------------------
+
+// DismissalRepo implements port.DismissalRepo using PostgreSQL.
+type DismissalRepo struct {
+	pool *pgxpool.Pool
+}
+
+// NewDismissalRepo creates a DismissalRepo backed by pool.
+func NewDismissalRepo(pool *pgxpool.Pool) *DismissalRepo {
+	return &DismissalRepo{pool: pool}
+}
+
+var _ port.DismissalRepo = (*DismissalRepo)(nil)
+
+// Add records a pair as "not a duplicate". The two ids are stored in canonical
+// (file_a < file_b) order to match the table's CHECK and avoid (a,b)/(b,a)
+// duplicates; a repeated dismissal is a no-op.
+func (r *DismissalRepo) Add(ctx context.Context, a, b uuid.UUID, userID int16) error {
+	if bytes.Compare(a[:], b[:]) > 0 {
+		a, b = b, a
+	}
+	const sqlStr = `
+        INSERT INTO data.duplicate_dismissals (file_a, file_b, dismissed_by)
+        VALUES ($1, $2, $3)
+        ON CONFLICT (file_a, file_b) DO NOTHING`
+	q := connOrTx(ctx, r.pool)
+	if _, err := q.Exec(ctx, sqlStr, a, b, userID); err != nil {
+		return fmt.Errorf("DismissalRepo.Add: %w", err)
+	}
+	return nil
+}

backend/internal/db/postgres/file_repo.go

@@ -434,6 +434,101 @@ func (r *FileRepo) SetNeedsReview(ctx context.Context, ids []uuid.UUID, value bo
 	return nil
 }
 
+// SetPHash sets (or clears, when phash is nil) the perceptual hash of a file.
+// Used by the dedup backfill and on content replacement; phash is non-critical,
+// recomputable metadata, so callers may treat failures as best-effort.
+func (r *FileRepo) SetPHash(ctx context.Context, id uuid.UUID, phash *int64) error {
+	const sqlStr = `UPDATE data.files SET phash = $2 WHERE id = $1`
+	q := connOrTx(ctx, r.pool)
+	if _, err := q.Exec(ctx, sqlStr, id, phash); err != nil {
+		return fmt.Errorf("FileRepo.SetPHash: %w", err)
+	}
+	return nil
+}
+
+// ---------------------------------------------------------------------------
+// Perceptual-hash / duplicate support
+// ---------------------------------------------------------------------------
+
+// ListMissingPHash returns live image/video files that have no perceptual hash
+// yet — the work list for the dedup backfill. Tags are not loaded (the backfill
+// only needs the id and MIME type to choose image vs video hashing).
+func (r *FileRepo) ListMissingPHash(ctx context.Context) ([]domain.File, error) {
+	const sqlStr = `
+        SELECT f.id, f.original_name,
+               mt.name AS mime_type, mt.extension AS mime_extension,
+               f.content_datetime, f.notes, f.metadata, f.exif, f.phash,
+               f.creator_id, u.name AS creator_name,
+               f.is_public, f.is_deleted, f.needs_review
+        FROM data.files f
+        JOIN core.mime_types mt ON mt.id = f.mime_id
+        JOIN core.users u       ON u.id  = f.creator_id
+        WHERE f.phash IS NULL AND f.is_deleted = false
+          AND (mt.name LIKE 'image/%' OR mt.name LIKE 'video/%')
+        ORDER BY f.id`
+
+	q := connOrTx(ctx, r.pool)
+	rows, err := q.Query(ctx, sqlStr)
+	if err != nil {
+		return nil, fmt.Errorf("FileRepo.ListMissingPHash: %w", err)
+	}
+	collected, err := pgx.CollectRows(rows, pgx.RowToStructByName[fileRow])
+	if err != nil {
+		return nil, fmt.Errorf("FileRepo.ListMissingPHash scan: %w", err)
+	}
+	files := make([]domain.File, len(collected))
+	for i, row := range collected {
+		files[i] = toFile(row)
+	}
+	return files, nil
+}
+
+// phashRow is the minimal projection used to build duplicate clusters.
+type phashRow struct {
+	ID    uuid.UUID `db:"id"`
+	PHash int64     `db:"phash"`
+}
+
+// ListAllPHashes returns the id and perceptual hash of every live, hashed file.
+// It is the global input to the dedup rescan, so it deliberately ignores ACL —
+// the rescan builds the shared pairs table; visibility is enforced on read.
+func (r *FileRepo) ListAllPHashes(ctx context.Context) ([]domain.PHashEntry, error) {
+	const sqlStr = `SELECT id, phash FROM data.files WHERE is_deleted = false AND phash IS NOT NULL`
+	q := connOrTx(ctx, r.pool)
+	rows, err := q.Query(ctx, sqlStr)
+	if err != nil {
+		return nil, fmt.Errorf("FileRepo.ListAllPHashes: %w", err)
+	}
+	collected, err := pgx.CollectRows(rows, pgx.RowToStructByName[phashRow])
+	if err != nil {
+		return nil, fmt.Errorf("FileRepo.ListAllPHashes scan: %w", err)
+	}
+	out := make([]domain.PHashEntry, len(collected))
+	for i, row := range collected {
+		out[i] = domain.PHashEntry{ID: row.ID, PHash: row.PHash}
+	}
+	return out, nil
+}
+
+// CopyPoolMemberships adds targetID to every pool sourceID belongs to (copying
+// the source's position), skipping pools the target is already in. Used by the
+// duplicate merge to preserve the discarded file's pool memberships on the
+// survivor. The merge is authorised at the file level, so pool ACL is not
+// re-checked here.
+func (r *FileRepo) CopyPoolMemberships(ctx context.Context, targetID, sourceID uuid.UUID) error {
+	const sqlStr = `
+        INSERT INTO data.file_pool (file_id, pool_id, position)
+        SELECT $1, fp.pool_id, fp.position
+        FROM data.file_pool fp
+        WHERE fp.file_id = $2
+        ON CONFLICT (file_id, pool_id) DO NOTHING`
+	q := connOrTx(ctx, r.pool)
+	if _, err := q.Exec(ctx, sqlStr, targetID, sourceID); err != nil {
+		return fmt.Errorf("FileRepo.CopyPoolMemberships: %w", err)
+	}
+	return nil
+}
+
 // ---------------------------------------------------------------------------
 // SoftDelete / Restore / DeletePermanent
 // ---------------------------------------------------------------------------

backend/internal/domain/duplicate.go

@@ -0,0 +1,18 @@
+package domain
+
+import "github.com/google/uuid"
+
+// PHashEntry is a file's perceptual hash, the input to duplicate clustering.
+type PHashEntry struct {
+	ID    uuid.UUID
+	PHash int64
+}
+
+// DuplicatePair is an unordered pair of files whose perceptual hashes are within
+// the configured Hamming threshold. FileA < FileB by UUID byte order (canonical),
+// so a pair is represented exactly once.
+type DuplicatePair struct {
+	FileA    uuid.UUID
+	FileB    uuid.UUID
+	Distance int
+}

backend/internal/handler/duplicate_handler.go

@@ -0,0 +1,122 @@
+package handler
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+
+	"tanabata/backend/internal/domain"
+	"tanabata/backend/internal/service"
+)
+
+// DuplicateHandler handles the /files/duplicates endpoints.
+type DuplicateHandler struct {
+	dupSvc *service.DuplicateService
+}
+
+// NewDuplicateHandler creates a DuplicateHandler.
+func NewDuplicateHandler(dupSvc *service.DuplicateService) *DuplicateHandler {
+	return &DuplicateHandler{dupSvc: dupSvc}
+}
+
+// List handles GET /files/duplicates — an offset-paginated list of duplicate
+// clusters, each a group of files within the perceptual-hash threshold.
+func (h *DuplicateHandler) List(c *gin.Context) {
+	limit, offset := 20, 0
+	if n, err := strconv.Atoi(c.Query("limit")); err == nil {
+		limit = n
+	}
+	if n, err := strconv.Atoi(c.Query("offset")); err == nil {
+		offset = n
+	}
+	if limit < 1 {
+		limit = 1
+	}
+	if limit > 50 {
+		limit = 50
+	}
+	if offset < 0 {
+		offset = 0
+	}
+
+	clusters, total, err := h.dupSvc.Clusters(c.Request.Context(), limit, offset)
+	if err != nil {
+		respondError(c, err)
+		return
+	}
+
+	items := make([]gin.H, len(clusters))
+	for i, files := range clusters {
+		fs := make([]fileJSON, len(files))
+		for j, f := range files {
+			fs[j] = toFileJSON(f)
+		}
+		items[i] = gin.H{"files": fs}
+	}
+	respondJSON(c, http.StatusOK, gin.H{
+		"items":  items,
+		"total":  total,
+		"limit":  limit,
+		"offset": offset,
+	})
+}
+
+// Dismiss handles POST /files/duplicates/dismiss — mark a pair "not a duplicate".
+func (h *DuplicateHandler) Dismiss(c *gin.Context) {
+	var body struct {
+		FileIDA string `json:"file_id_a" binding:"required"`
+		FileIDB string `json:"file_id_b" binding:"required"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil {
+		respondError(c, domain.ErrValidation)
+		return
+	}
+	ids, err := parseUUIDs([]string{body.FileIDA, body.FileIDB})
+	if err != nil {
+		respondError(c, domain.ErrValidation)
+		return
+	}
+	if err := h.dupSvc.Dismiss(c.Request.Context(), ids[0], ids[1]); err != nil {
+		respondError(c, err)
+		return
+	}
+	c.Status(http.StatusNoContent)
+}
+
+// Resolve handles POST /files/duplicates/resolve — merge a duplicate pair,
+// keeping one file and folding the chosen fields in from the other. Returns the
+// updated survivor. delete_discarded defaults to true.
+func (h *DuplicateHandler) Resolve(c *gin.Context) {
+	var body struct {
+		Keep            string              `json:"keep"    binding:"required"`
+		Discard         string              `json:"discard" binding:"required"`
+		Fields          service.MergeFields `json:"fields"`
+		DeleteDiscarded *bool               `json:"delete_discarded"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil {
+		respondError(c, domain.ErrValidation)
+		return
+	}
+	ids, err := parseUUIDs([]string{body.Keep, body.Discard})
+	if err != nil {
+		respondError(c, domain.ErrValidation)
+		return
+	}
+
+	del := true
+	if body.DeleteDiscarded != nil {
+		del = *body.DeleteDiscarded
+	}
+	f, err := h.dupSvc.Resolve(c.Request.Context(), service.MergeSpec{
+		Keep:            ids[0],
+		Discard:         ids[1],
+		Fields:          body.Fields,
+		DeleteDiscarded: del,
+	})
+	if err != nil {
+		respondError(c, err)
+		return
+	}
+	respondJSON(c, http.StatusOK, toFileJSON(*f))
+}

backend/internal/handler/router.go

@@ -26,6 +26,7 @@ func NewRouter(
 	auth *AuthMiddleware,
 	authHandler *AuthHandler,
 	fileHandler *FileHandler,
+	duplicateHandler *DuplicateHandler,
 	tagHandler *TagHandler,
 	categoryHandler *CategoryHandler,
 	poolHandler *PoolHandler,
@@ -80,7 +81,11 @@ func NewRouter(
 		files.GET("", fileHandler.List)
 		files.POST("", fileHandler.Upload)
 
-		// Bulk + import routes registered before /:id to prevent param collision.
+		// Bulk + import + duplicates routes registered before /:id to prevent
+		// param collision (e.g. "duplicates" being captured as :id).
+		files.GET("/duplicates", duplicateHandler.List)
+		files.POST("/duplicates/dismiss", duplicateHandler.Dismiss)
+		files.POST("/duplicates/resolve", duplicateHandler.Resolve)
 		files.POST("/bulk/tags", fileHandler.BulkSetTags)
 		files.POST("/bulk/delete", fileHandler.BulkDelete)
 		files.POST("/bulk/review", fileHandler.BulkReview)

backend/internal/handler/router_test.go

@@ -10,7 +10,7 @@ import "testing"
 func TestNewRouterRegisters(t *testing.T) {
 	r, err := NewRouter(
 		(*AuthMiddleware)(nil), (*AuthHandler)(nil),
-		(*FileHandler)(nil), (*TagHandler)(nil), (*CategoryHandler)(nil), (*PoolHandler)(nil),
+		(*FileHandler)(nil), (*DuplicateHandler)(nil), (*TagHandler)(nil), (*CategoryHandler)(nil), (*PoolHandler)(nil),
 		(*UserHandler)(nil), (*ACLHandler)(nil), (*AuditHandler)(nil),
 		"", nil,
 	)

backend/internal/imagehash/imagehash.go

@@ -0,0 +1,70 @@
+// Package imagehash computes a 64-bit perceptual hash (dHash) of an image and
+// compares two hashes by Hamming distance. It is used for near-duplicate
+// detection: visually similar images (re-encoded, resized, recompressed) produce
+// hashes a small distance apart, while unrelated images are far apart.
+//
+// dHash is chosen for its robustness and simplicity: the image is reduced to a
+// 9×8 grayscale and each pixel is compared to its right-hand neighbour, yielding
+// 64 gradient-direction bits. It tolerates scaling and brightness/contrast
+// changes well, which is exactly what re-encoded duplicates exhibit.
+package imagehash
+
+import (
+	"bytes"
+	"image"
+	_ "image/gif"  // register GIF decoder
+	_ "image/jpeg" // register JPEG decoder
+	_ "image/png"  // register PNG decoder
+	"math/bits"
+
+	"github.com/disintegration/imaging"
+	_ "golang.org/x/image/webp" // register WebP decoder
+)
+
+// hashWidth/hashHeight define the reduced grayscale used for dHash. The extra
+// column (width = height+1) provides the right-hand neighbour for the 64
+// horizontal comparisons that make up the hash.
+const (
+	hashHeight = 8
+	hashWidth  = hashHeight + 1
+)
+
+// FromImage reduces img to a 9×8 grayscale and returns its 64-bit dHash. The
+// uint64 of gradient bits is returned as int64 (a plain bit reinterpretation) so
+// it fits PostgreSQL's bigint; equality and Distance are bitwise, so the signed
+// interpretation never matters.
+func FromImage(img image.Image) int64 {
+	small := imaging.Grayscale(imaging.Resize(img, hashWidth, hashHeight, imaging.Lanczos))
+
+	var hash uint64
+	bit := 0
+	for y := 0; y < hashHeight; y++ {
+		for x := 0; x < hashHeight; x++ {
+			// After Grayscale, R == G == B, so the red channel is the luminance.
+			left := small.Pix[small.PixOffset(x, y)]
+			right := small.Pix[small.PixOffset(x+1, y)]
+			if left < right {
+				hash |= 1 << uint(63-bit)
+			}
+			bit++
+		}
+	}
+	return int64(hash)
+}
+
+// FromBytes decodes data (JPEG/PNG/GIF/WebP) and returns its dHash. ok is false
+// when the bytes are not a decodable image, so callers can simply skip hashing
+// (e.g. leave phash NULL) rather than fail.
+func FromBytes(data []byte) (hash int64, ok bool) {
+	img, _, err := image.Decode(bytes.NewReader(data))
+	if err != nil {
+		return 0, false
+	}
+	return FromImage(img), true
+}
+
+// Distance returns the Hamming distance (0–64) between two hashes: the number of
+// differing bits. 0 means identical; small values mean near-duplicate.
+func Distance(a, b int64) int {
+	return bits.OnesCount64(uint64(a) ^ uint64(b))
+}

backend/internal/imagehash/imagehash_test.go

@@ -0,0 +1,99 @@
+package imagehash
+
+import (
+	"bytes"
+	"image"
+	"image/color"
+	"image/jpeg"
+	"image/png"
+	"math"
+	"testing"
+)
+
+// radial renders a smooth grayscale image whose brightness falls off with
+// distance from (cx, cy). Smooth gradients are the realistic case for perceptual
+// hashing and survive JPEG re-encoding well, so they make stable test fixtures.
+func radial(w, h int, cx, cy float64) image.Image {
+	img := image.NewRGBA(image.Rect(0, 0, w, h))
+	maxD := math.Hypot(float64(w), float64(h))
+	for y := 0; y < h; y++ {
+		for x := 0; x < w; x++ {
+			d := math.Hypot(float64(x)-cx, float64(y)-cy)
+			v := uint8(255 * (1 - d/maxD))
+			img.Set(x, y, color.RGBA{v, v, v, 255})
+		}
+	}
+	return img
+}
+
+func encodePNG(t *testing.T, img image.Image) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	if err := png.Encode(&buf, img); err != nil {
+		t.Fatalf("png encode: %v", err)
+	}
+	return buf.Bytes()
+}
+
+func encodeJPEG(t *testing.T, img image.Image, quality int) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: quality}); err != nil {
+		t.Fatalf("jpeg encode: %v", err)
+	}
+	return buf.Bytes()
+}
+
+// The same image re-encoded as PNG (lossless) and JPEG (lossy) must hash to a
+// small Hamming distance — that is the whole point of a perceptual hash.
+func TestFromBytes_SameImageAcrossEncodings(t *testing.T) {
+	img := radial(64, 64, 32, 32)
+
+	pngHash, ok := FromBytes(encodePNG(t, img))
+	if !ok {
+		t.Fatal("FromBytes(PNG): ok=false")
+	}
+	jpgHash, ok := FromBytes(encodeJPEG(t, img, 90))
+	if !ok {
+		t.Fatal("FromBytes(JPEG): ok=false")
+	}
+
+	if d := Distance(pngHash, jpgHash); d > 8 {
+		t.Errorf("same image, different encodings: distance = %d, want <= 8", d)
+	}
+}
+
+// Visually different images must be far apart, and clearly farther than the same
+// image across encodings.
+func TestDistance_DifferentImagesAreFarApart(t *testing.T) {
+	a := FromImage(radial(64, 64, 32, 32)) // centred
+	b := FromImage(radial(64, 64, 0, 0))   // corner
+
+	same, _ := FromBytes(encodeJPEG(t, radial(64, 64, 32, 32), 90))
+
+	d := Distance(a, b)
+	if d < 12 {
+		t.Errorf("different images: distance = %d, want >= 12", d)
+	}
+	if d <= Distance(a, same) {
+		t.Errorf("different images (%d) not farther than re-encoded same image (%d)", d, Distance(a, same))
+	}
+}
+
+func TestDistance_SymmetricAndZeroForEqual(t *testing.T) {
+	a := FromImage(radial(64, 64, 20, 40))
+	b := FromImage(radial(64, 64, 40, 20))
+
+	if Distance(a, a) != 0 {
+		t.Errorf("Distance(a, a) = %d, want 0", Distance(a, a))
+	}
+	if Distance(a, b) != Distance(b, a) {
+		t.Errorf("Distance not symmetric: %d vs %d", Distance(a, b), Distance(b, a))
+	}
+}
+
+func TestFromBytes_RejectsNonImage(t *testing.T) {
+	if _, ok := FromBytes([]byte("definitely not an image")); ok {
+		t.Error("FromBytes on garbage: ok=true, want false")
+	}
+}

backend/internal/integration/server_test.go

@@ -56,6 +56,9 @@ type harness struct {
 	client    *http.Client
 	importDir string
 	pool      *pgxpool.Pool
+	// dupSvc lets duplicate tests trigger a pairs rescan directly: rebuilding the
+	// pairs table is a CLI/maintenance action with no HTTP endpoint.
+	dupSvc *service.DuplicateService
 }
 
 // setupSuite creates an ephemeral database, runs migrations, wires the full
@@ -125,6 +128,8 @@ func setupSuite(t *testing.T) *harness {
 	tagRuleRepo := postgres.NewTagRuleRepo(pool)
 	categoryRepo := postgres.NewCategoryRepo(pool)
 	poolRepo := postgres.NewPoolRepo(pool)
+	duplicatePairRepo := postgres.NewDuplicatePairRepo(pool)
+	dismissalRepo := postgres.NewDismissalRepo(pool)
 	transactor := postgres.NewTransactor(pool)
 
 	// --- Services ------------------------------------------------------------
@@ -134,6 +139,7 @@ func setupSuite(t *testing.T) *harness {
 	tagSvc := service.NewTagService(tagRepo, tagRuleRepo, aclSvc, auditSvc, transactor)
 	categorySvc := service.NewCategoryService(categoryRepo, tagRepo, aclSvc, auditSvc)
 	poolSvc := service.NewPoolService(poolRepo, aclSvc, auditSvc)
+	duplicateSvc := service.NewDuplicateService(fileRepo, duplicatePairRepo, dismissalRepo, aclSvc, auditSvc, transactor, 10)
 	fileSvc := service.NewFileService(fileRepo, mimeRepo, diskStorage, aclSvc, auditSvc, tagSvc, transactor, importDir)
 	userSvc := service.NewUserService(userRepo, sessionRepo, auditSvc)
 
@@ -145,6 +151,7 @@ func setupSuite(t *testing.T) *harness {
 	authMiddleware := handler.NewAuthMiddleware(authSvc)
 	authHandler := handler.NewAuthHandler(authSvc)
 	fileHandler := handler.NewFileHandler(fileSvc, tagSvc, authSvc, 500<<20)
+	duplicateHandler := handler.NewDuplicateHandler(duplicateSvc)
 	tagHandler := handler.NewTagHandler(tagSvc, fileSvc)
 	categoryHandler := handler.NewCategoryHandler(categorySvc)
 	poolHandler := handler.NewPoolHandler(poolSvc)
@@ -154,7 +161,7 @@ func setupSuite(t *testing.T) *harness {
 
 	r, err := handler.NewRouter(
 		authMiddleware, authHandler,
-		fileHandler, tagHandler, categoryHandler, poolHandler,
+		fileHandler, duplicateHandler, tagHandler, categoryHandler, poolHandler,
 		userHandler, aclHandler, auditHandler,
 		"",
 		nil,
@@ -170,6 +177,7 @@ func setupSuite(t *testing.T) *harness {
 		client:    srv.Client(),
 		importDir: importDir,
 		pool:      pool,
+		dupSvc:    duplicateSvc,
 	}
 }
 
@@ -1643,3 +1651,103 @@ var (
 	_ = freePort
 	_ = writeFile
 )
+
+// dupListResponse decodes GET /files/duplicates.
+type dupListResponse struct {
+	Items []struct {
+		Files []struct {
+			ID   string `json:"id"`
+			Tags []struct {
+				ID string `json:"id"`
+			} `json:"tags"`
+		} `json:"files"`
+	} `json:"items"`
+	Total int `json:"total"`
+}
+
+// TestDuplicateDetection exercises the full duplicate lifecycle: perceptual hashes
+// are computed on upload, a rescan builds the pairs table, the cluster surfaces,
+// a field-by-field merge unions tags and trashes the discarded file, and a
+// dismissal hides a pair permanently (surviving a re-rescan).
+//
+// minimalJPEG() is a 1×1 image, so every upload hashes identically — in a fresh
+// database any two uploads form one duplicate pair, which keeps this deterministic.
+func TestDuplicateDetection(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	h := setupSuite(t)
+	ctx := context.Background()
+	admin := h.login("admin", "admin")
+
+	// --- two uploads => one duplicate pair after a rescan ---------------------
+	f1 := h.uploadJPEG(admin, "a.jpg")["id"].(string)
+	f2 := h.uploadJPEG(admin, "b.jpg")["id"].(string)
+
+	// Tag f2 so the merge has something to union onto the survivor.
+	resp := h.doJSON("POST", "/tags", map[string]any{"name": "kept", "is_public": true}, admin)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, resp.String())
+	var tag map[string]any
+	resp.decode(t, &tag)
+	tagID := tag["id"].(string)
+	resp = h.doJSON("PUT", "/files/"+f2+"/tags", map[string]any{"tag_ids": []string{tagID}}, admin)
+	require.Equal(t, http.StatusOK, resp.StatusCode, resp.String())
+
+	require.NoError(t, h.dupSvc.Rescan(ctx, nil))
+
+	resp = h.doJSON("GET", "/files/duplicates", nil, admin)
+	require.Equal(t, http.StatusOK, resp.StatusCode, resp.String())
+	var list dupListResponse
+	resp.decode(t, &list)
+	require.Equal(t, 1, list.Total, "expected one duplicate cluster: %s", resp)
+	require.Len(t, list.Items, 1)
+	require.Len(t, list.Items[0].Files, 2)
+
+	// --- resolve: keep f1, union tags from f2, trash f2 ----------------------
+	resp = h.doJSON("POST", "/files/duplicates/resolve", map[string]any{
+		"keep":             f1,
+		"discard":          f2,
+		"fields":           map[string]any{"tags": "both"},
+		"delete_discarded": true,
+	}, admin)
+	require.Equal(t, http.StatusOK, resp.StatusCode, resp.String())
+	var survivor struct {
+		ID   string `json:"id"`
+		Tags []struct {
+			ID string `json:"id"`
+		} `json:"tags"`
+	}
+	resp.decode(t, &survivor)
+	assert.Equal(t, f1, survivor.ID)
+	require.Len(t, survivor.Tags, 1, "survivor should have inherited the discarded file's tag")
+	assert.Equal(t, tagID, survivor.Tags[0].ID)
+
+	// f2 is now trashed, so the pair drops out of the duplicates view.
+	resp = h.doJSON("GET", "/files/duplicates", nil, admin)
+	resp.decode(t, &list)
+	assert.Equal(t, 0, list.Total, "resolved pair should no longer surface: %s", resp)
+
+	// --- dismiss: a new pair, hidden and staying hidden across a rescan -------
+	f3 := h.uploadJPEG(admin, "c.jpg")["id"].(string)
+	require.NoError(t, h.dupSvc.Rescan(ctx, nil))
+
+	resp = h.doJSON("GET", "/files/duplicates", nil, admin)
+	resp.decode(t, &list)
+	require.Equal(t, 1, list.Total, "f1 and f3 should now form a cluster: %s", resp)
+
+	resp = h.doJSON("POST", "/files/duplicates/dismiss", map[string]any{
+		"file_id_a": f1, "file_id_b": f3,
+	}, admin)
+	require.Equal(t, http.StatusNoContent, resp.StatusCode, resp.String())
+
+	resp = h.doJSON("GET", "/files/duplicates", nil, admin)
+	resp.decode(t, &list)
+	assert.Equal(t, 0, list.Total, "dismissed pair should be hidden")
+
+	// A rescan re-finds the pair but the dismissal still hides it.
+	require.NoError(t, h.dupSvc.Rescan(ctx, nil))
+	resp = h.doJSON("GET", "/files/duplicates", nil, admin)
+	resp.decode(t, &list)
+	assert.Equal(t, 0, list.Total, "dismissal must survive a rescan")
+}

backend/internal/port/repository.go

@@ -50,6 +50,17 @@ type FileRepo interface {
 	Update(ctx context.Context, id uuid.UUID, f *domain.File) (*domain.File, error)
 	// SetNeedsReview sets the review status on the given (non-trashed) files.
 	SetNeedsReview(ctx context.Context, ids []uuid.UUID, value bool) error
+	// SetPHash sets (or clears, when nil) the perceptual hash of a file.
+	SetPHash(ctx context.Context, id uuid.UUID, phash *int64) error
+	// ListMissingPHash returns live image/video files that have no perceptual
+	// hash yet (the dedup backfill work list).
+	ListMissingPHash(ctx context.Context) ([]domain.File, error)
+	// ListAllPHashes returns the id and perceptual hash of every live, hashed
+	// file (the global input to the dedup rescan; not ACL-filtered).
+	ListAllPHashes(ctx context.Context) ([]domain.PHashEntry, error)
+	// CopyPoolMemberships adds targetID to every pool sourceID belongs to,
+	// skipping pools target is already in (used by the duplicate merge).
+	CopyPoolMemberships(ctx context.Context, targetID, sourceID uuid.UUID) error
 	// SoftDelete moves a file to trash (sets is_deleted = true).
 	SoftDelete(ctx context.Context, id uuid.UUID) error
 	// Restore moves a file out of trash (sets is_deleted = false).
@@ -70,6 +81,21 @@ type FileRepo interface {
 	RecordTagUses(ctx context.Context, userID int16, filterDSL string) error
 }
 
+// DuplicatePairRepo persists the precomputed near-duplicate candidate pairs.
+type DuplicatePairRepo interface {
+	// ReplaceAll atomically replaces the whole pairs table (used by the rescan).
+	ReplaceAll(ctx context.Context, pairs []domain.DuplicatePair) error
+	// ListVisible returns pairs whose both files are live, not dismissed, and
+	// (for non-admins) visible to the viewer.
+	ListVisible(ctx context.Context, viewerID int16, isAdmin bool) ([]domain.DuplicatePair, error)
+}
+
+// DismissalRepo persists "not a duplicate" decisions.
+type DismissalRepo interface {
+	// Add records a pair as dismissed (canonical order, idempotent).
+	Add(ctx context.Context, a, b uuid.UUID, userID int16) error
+}
+
 // TagRepo is the persistence interface for tags.
 type TagRepo interface {
 	List(ctx context.Context, params OffsetParams) (*domain.TagOffsetPage, error)

backend/internal/service/duplicate_index.go

@@ -0,0 +1,148 @@
+package service
+
+import (
+	"bytes"
+	"math/bits"
+	"sort"
+
+	"github.com/google/uuid"
+
+	"tanabata/backend/internal/domain"
+)
+
+// hamming returns the number of differing bits between two perceptual hashes.
+func hamming(a, b uint64) int { return bits.OnesCount64(a ^ b) }
+
+// bkNode is a node in a BK-tree over Hamming distance. Files that share the exact
+// same hash are collected in ids (a distance-0 collision), so identical images
+// don't degenerate the tree into a chain.
+type bkNode struct {
+	hash     uint64
+	ids      []uuid.UUID
+	children map[int]*bkNode
+}
+
+// bkTree indexes perceptual hashes for sublinear radius queries. Building one and
+// querying every element with a small radius is far cheaper than the O(N²) all-
+// pairs comparison at 100k+ files.
+type bkTree struct{ root *bkNode }
+
+func (t *bkTree) insert(hash uint64, id uuid.UUID) {
+	if t.root == nil {
+		t.root = &bkNode{hash: hash, ids: []uuid.UUID{id}, children: map[int]*bkNode{}}
+		return
+	}
+	node := t.root
+	for {
+		d := hamming(hash, node.hash)
+		if d == 0 {
+			node.ids = append(node.ids, id)
+			return
+		}
+		child, ok := node.children[d]
+		if !ok {
+			node.children[d] = &bkNode{hash: hash, ids: []uuid.UUID{id}, children: map[int]*bkNode{}}
+			return
+		}
+		node = child
+	}
+}
+
+// query visits every node whose hash is within radius of target. The triangle
+// inequality bounds which children can hold a match to [d-radius, d+radius].
+func (t *bkTree) query(target uint64, radius int, visit func(node *bkNode, dist int)) {
+	if t.root == nil {
+		return
+	}
+	stack := []*bkNode{t.root}
+	for len(stack) > 0 {
+		node := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+
+		d := hamming(target, node.hash)
+		if d <= radius {
+			visit(node, d)
+		}
+		lo, hi := d-radius, d+radius
+		for cd, child := range node.children {
+			if cd >= lo && cd <= hi {
+				stack = append(stack, child)
+			}
+		}
+	}
+}
+
+// buildPairs returns every unordered pair of files whose hashes are within
+// threshold, each emitted exactly once with FileA < FileB (UUID byte order).
+// onProgress, if set, is called periodically with (processed, total).
+func buildPairs(entries []domain.PHashEntry, threshold int, onProgress func(done, total int)) []domain.DuplicatePair {
+	tree := &bkTree{}
+	for _, e := range entries {
+		tree.insert(uint64(e.PHash), e.ID)
+	}
+
+	var pairs []domain.DuplicatePair
+	total := len(entries)
+	for i := range entries {
+		e := entries[i]
+		tree.query(uint64(e.PHash), threshold, func(node *bkNode, dist int) {
+			for _, other := range node.ids {
+				// Emit each pair once, from the smaller id, which also skips self.
+				if bytes.Compare(e.ID[:], other[:]) < 0 {
+					pairs = append(pairs, domain.DuplicatePair{FileA: e.ID, FileB: other, Distance: dist})
+				}
+			}
+		})
+		if onProgress != nil && (i+1)%1000 == 0 {
+			onProgress(i+1, total)
+		}
+	}
+	if onProgress != nil {
+		onProgress(total, total)
+	}
+	return pairs
+}
+
+// clusterPairs groups pairs into connected components (transitive closure) via
+// union-find. Every returned cluster has at least two files; clusters and the ids
+// within them are sorted by UUID for stable pagination.
+func clusterPairs(pairs []domain.DuplicatePair) [][]uuid.UUID {
+	parent := map[uuid.UUID]uuid.UUID{}
+	var find func(uuid.UUID) uuid.UUID
+	find = func(x uuid.UUID) uuid.UUID {
+		p, ok := parent[x]
+		if !ok {
+			parent[x] = x
+			return x
+		}
+		if p != x {
+			parent[x] = find(p)
+		}
+		return parent[x]
+	}
+	union := func(a, b uuid.UUID) {
+		ra, rb := find(a), find(b)
+		if ra != rb {
+			parent[ra] = rb
+		}
+	}
+	for _, p := range pairs {
+		union(p.FileA, p.FileB)
+	}
+
+	groups := map[uuid.UUID][]uuid.UUID{}
+	for node := range parent {
+		root := find(node)
+		groups[root] = append(groups[root], node)
+	}
+
+	clusters := make([][]uuid.UUID, 0, len(groups))
+	for _, ids := range groups {
+		sort.Slice(ids, func(i, j int) bool { return bytes.Compare(ids[i][:], ids[j][:]) < 0 })
+		clusters = append(clusters, ids)
+	}
+	sort.Slice(clusters, func(i, j int) bool {
+		return bytes.Compare(clusters[i][0][:], clusters[j][0][:]) < 0
+	})
+	return clusters
+}

backend/internal/service/duplicate_service.go

@@ -0,0 +1,361 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/google/uuid"
+
+	"tanabata/backend/internal/domain"
+	"tanabata/backend/internal/port"
+)
+
+// Merge field source values.
+const (
+	mergeKeep    = "keep"
+	mergeDiscard = "discard"
+	mergeBoth    = "both"
+	mergeMerge   = "merge"
+)
+
+// MergeFields chooses, per field, which file supplies the survivor's value when
+// resolving a duplicate. Scalars accept "keep"/"discard"; metadata also accepts
+// "merge" (shallow object merge, survivor wins on key conflicts); relations
+// (tags, pools) accept "keep"/"both" (union) — there is deliberately no option
+// to drop the survivor's own tags/pools. An empty value defaults to "keep".
+type MergeFields struct {
+	OriginalName    string `json:"original_name"`
+	Notes           string `json:"notes"`
+	ContentDatetime string `json:"content_datetime"`
+	IsPublic        string `json:"is_public"`
+	Metadata        string `json:"metadata"`
+	Tags            string `json:"tags"`
+	Pools           string `json:"pools"`
+}
+
+// MergeSpec is the input to a duplicate resolution: keep one file, fold chosen
+// fields in from the other, and (usually) trash the other.
+type MergeSpec struct {
+	Keep            uuid.UUID
+	Discard         uuid.UUID
+	Fields          MergeFields
+	DeleteDiscarded bool
+}
+
+// normalize fills empty choices with "keep" and rejects unknown values.
+func (m *MergeSpec) normalize() error {
+	scalar := func(v *string) error {
+		if *v == "" {
+			*v = mergeKeep
+		}
+		if *v != mergeKeep && *v != mergeDiscard {
+			return domain.ErrValidation
+		}
+		return nil
+	}
+	relation := func(v *string) error {
+		if *v == "" {
+			*v = mergeKeep
+		}
+		if *v != mergeKeep && *v != mergeBoth {
+			return domain.ErrValidation
+		}
+		return nil
+	}
+	f := &m.Fields
+	if err := scalar(&f.OriginalName); err != nil {
+		return err
+	}
+	if err := scalar(&f.Notes); err != nil {
+		return err
+	}
+	if err := scalar(&f.ContentDatetime); err != nil {
+		return err
+	}
+	if err := scalar(&f.IsPublic); err != nil {
+		return err
+	}
+	if f.Metadata == "" {
+		f.Metadata = mergeKeep
+	}
+	if f.Metadata != mergeKeep && f.Metadata != mergeDiscard && f.Metadata != mergeMerge {
+		return domain.ErrValidation
+	}
+	if err := relation(&f.Tags); err != nil {
+		return err
+	}
+	if err := relation(&f.Pools); err != nil {
+		return err
+	}
+	return nil
+}
+
+// DuplicateService finds near-duplicate clusters and resolves them.
+type DuplicateService struct {
+	files      port.FileRepo
+	pairs      port.DuplicatePairRepo
+	dismissals port.DismissalRepo
+	acl        *ACLService
+	audit      *AuditService
+	tx         port.Transactor
+	threshold  int
+}
+
+// NewDuplicateService creates a DuplicateService. threshold is the maximum
+// Hamming distance for two files to be treated as duplicate candidates.
+func NewDuplicateService(
+	files port.FileRepo,
+	pairs port.DuplicatePairRepo,
+	dismissals port.DismissalRepo,
+	acl *ACLService,
+	audit *AuditService,
+	tx port.Transactor,
+	threshold int,
+) *DuplicateService {
+	return &DuplicateService{
+		files:      files,
+		pairs:      pairs,
+		dismissals: dismissals,
+		acl:        acl,
+		audit:      audit,
+		tx:         tx,
+		threshold:  threshold,
+	}
+}
+
+// Clusters returns a page of duplicate clusters visible to the caller. Pairs are
+// read from the precomputed table (no all-pairs scan here) and grouped into
+// connected components; pagination is over whole clusters.
+func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clusters [][]domain.File, total int, err error) {
+	userID, isAdmin, _ := domain.UserFromContext(ctx)
+
+	pairs, err := s.pairs.ListVisible(ctx, userID, isAdmin)
+	if err != nil {
+		return nil, 0, err
+	}
+	groups := clusterPairs(pairs)
+	total = len(groups)
+
+	if offset < 0 {
+		offset = 0
+	}
+	if offset >= len(groups) {
+		return [][]domain.File{}, total, nil
+	}
+	end := offset + limit
+	if end > len(groups) || limit <= 0 {
+		end = len(groups)
+	}
+
+	out := make([][]domain.File, 0, end-offset)
+	for _, ids := range groups[offset:end] {
+		files := make([]domain.File, 0, len(ids))
+		for _, id := range ids {
+			f, err := s.files.GetByID(ctx, id)
+			if err != nil {
+				// A file deleted between the pair read and now just drops out.
+				if errors.Is(err, domain.ErrNotFound) {
+					continue
+				}
+				return nil, 0, err
+			}
+			files = append(files, *f)
+		}
+		if len(files) >= 2 {
+			out = append(out, files)
+		}
+	}
+	return out, total, nil
+}
+
+// Rescan recomputes the entire duplicate_pairs table from the current set of
+// perceptual hashes. It is the only thing that populates the table, so the
+// duplicates view reflects state as of the last rescan. Called by the dedup CLI.
+func (s *DuplicateService) Rescan(ctx context.Context, onProgress func(done, total int)) error {
+	entries, err := s.files.ListAllPHashes(ctx)
+	if err != nil {
+		return err
+	}
+	pairs := buildPairs(entries, s.threshold, onProgress)
+	return s.pairs.ReplaceAll(ctx, pairs)
+}
+
+// Dismiss records two files as "not a duplicate" so the pair stops surfacing.
+// The caller must be able to view both files.
+func (s *DuplicateService) Dismiss(ctx context.Context, a, b uuid.UUID) error {
+	if a == b {
+		return domain.ErrValidation
+	}
+	userID, isAdmin, _ := domain.UserFromContext(ctx)
+	for _, id := range []uuid.UUID{a, b} {
+		f, err := s.files.GetByID(ctx, id)
+		if err != nil {
+			return err
+		}
+		ok, err := s.acl.CanView(ctx, userID, isAdmin, f.CreatorID, f.IsPublic, fileObjectTypeID, id)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			return domain.ErrForbidden
+		}
+	}
+	if err := s.dismissals.Add(ctx, a, b, userID); err != nil {
+		return err
+	}
+	objType := fileObjectType
+	_ = s.audit.Log(ctx, "duplicate_dismiss", &objType, &a, map[string]any{"other": b.String()})
+	return nil
+}
+
+// Resolve merges a duplicate pair: the survivor (keep) takes the chosen fields
+// from the other (discard), and the other is trashed when DeleteDiscarded is set.
+// The caller must be able to edit both files. Returns the updated survivor.
+func (s *DuplicateService) Resolve(ctx context.Context, spec MergeSpec) (*domain.File, error) {
+	if spec.Keep == spec.Discard {
+		return nil, domain.ErrValidation
+	}
+	if err := spec.normalize(); err != nil {
+		return nil, err
+	}
+
+	keep, err := s.files.GetByID(ctx, spec.Keep)
+	if err != nil {
+		return nil, err
+	}
+	discard, err := s.files.GetByID(ctx, spec.Discard)
+	if err != nil {
+		return nil, err
+	}
+
+	userID, isAdmin, _ := domain.UserFromContext(ctx)
+	for _, f := range []*domain.File{keep, discard} {
+		ok, err := s.acl.CanEdit(ctx, userID, isAdmin, f.CreatorID, fileObjectTypeID, f.ID)
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			return nil, domain.ErrForbidden
+		}
+	}
+
+	// FileRepo.Update rewrites all editable scalar columns, so build the complete
+	// resolved set (each field from keep or discard) rather than a sparse patch.
+	patch := &domain.File{
+		OriginalName:    pickPtr(spec.Fields.OriginalName, keep.OriginalName, discard.OriginalName),
+		Notes:           pickPtr(spec.Fields.Notes, keep.Notes, discard.Notes),
+		ContentDatetime: pickTime(spec.Fields.ContentDatetime, keep.ContentDatetime, discard.ContentDatetime),
+		IsPublic:        pickBool(spec.Fields.IsPublic, keep.IsPublic, discard.IsPublic),
+		Metadata:        pickMetadata(spec.Fields.Metadata, keep.Metadata, discard.Metadata),
+	}
+
+	var result *domain.File
+	txErr := s.tx.WithTx(ctx, func(ctx context.Context) error {
+		updated, err := s.files.Update(ctx, keep.ID, patch)
+		if err != nil {
+			return err
+		}
+
+		if spec.Fields.Tags == mergeBoth {
+			if err := s.files.SetTags(ctx, keep.ID, unionTagIDs(keep.Tags, discard.Tags)); err != nil {
+				return err
+			}
+			tags, err := s.files.ListTags(ctx, keep.ID)
+			if err != nil {
+				return err
+			}
+			updated.Tags = tags
+		}
+		if spec.Fields.Pools == mergeBoth {
+			if err := s.files.CopyPoolMemberships(ctx, keep.ID, discard.ID); err != nil {
+				return err
+			}
+		}
+		if spec.DeleteDiscarded {
+			if err := s.files.SoftDelete(ctx, discard.ID); err != nil {
+				return err
+			}
+		}
+		result = updated
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+
+	objType := fileObjectType
+	_ = s.audit.Log(ctx, "file_merge", &objType, &keep.ID, map[string]any{
+		"discard":           spec.Discard.String(),
+		"fields":            spec.Fields,
+		"deleted_discarded": spec.DeleteDiscarded,
+	})
+	return result, nil
+}
+
+// --- field pickers ---------------------------------------------------------
+
+func pickPtr(choice string, keep, discard *string) *string {
+	if choice == mergeDiscard {
+		return discard
+	}
+	return keep
+}
+
+func pickBool(choice string, keep, discard bool) bool {
+	if choice == mergeDiscard {
+		return discard
+	}
+	return keep
+}
+
+func pickTime(choice string, keep, discard time.Time) time.Time {
+	if choice == mergeDiscard {
+		return discard
+	}
+	return keep
+}
+
+func unionTagIDs(a, b []domain.Tag) []uuid.UUID {
+	seen := make(map[uuid.UUID]bool, len(a)+len(b))
+	ids := make([]uuid.UUID, 0, len(a)+len(b))
+	for _, t := range append(append([]domain.Tag{}, a...), b...) {
+		if !seen[t.ID] {
+			seen[t.ID] = true
+			ids = append(ids, t.ID)
+		}
+	}
+	return ids
+}
+
+// pickMetadata returns keep's metadata, discard's, or a shallow merge in which
+// the survivor's keys win on conflict.
+func pickMetadata(choice string, keep, discard json.RawMessage) json.RawMessage {
+	switch choice {
+	case mergeDiscard:
+		return discard
+	case mergeMerge:
+		km := map[string]json.RawMessage{}
+		dm := map[string]json.RawMessage{}
+		_ = json.Unmarshal(keep, &km)
+		_ = json.Unmarshal(discard, &dm)
+		out := make(map[string]json.RawMessage, len(km)+len(dm))
+		for k, v := range dm {
+			out[k] = v
+		}
+		for k, v := range km { // survivor wins
+			out[k] = v
+		}
+		if len(out) == 0 {
+			return keep
+		}
+		b, err := json.Marshal(out)
+		if err != nil {
+			return keep
+		}
+		return b
+	default:
+		return keep
+	}
+}

backend/internal/service/duplicate_test.go

@@ -0,0 +1,152 @@
+package service
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/google/uuid"
+
+	"tanabata/backend/internal/domain"
+)
+
+// id builds a deterministic UUID whose byte order matches n, so tests can reason
+// about the canonical (FileA < FileB) ordering buildPairs produces.
+func id(n int) uuid.UUID {
+	return uuid.MustParse(fmt.Sprintf("00000000-0000-0000-0000-%012d", n))
+}
+
+func entry(n int, hash uint64) domain.PHashEntry {
+	return domain.PHashEntry{ID: id(n), PHash: int64(hash)}
+}
+
+// pairKey canonicalises a pair for set comparison regardless of emission order.
+func pairKey(p domain.DuplicatePair) string {
+	a, b := p.FileA, p.FileB
+	if bytes.Compare(a[:], b[:]) > 0 {
+		a, b = b, a
+	}
+	return fmt.Sprintf("%s|%s|%d", a, b, p.Distance)
+}
+
+func TestBuildPairs_ThresholdAndCanonicalOrder(t *testing.T) {
+	entries := []domain.PHashEntry{
+		entry(1, 0x0000000000000000),
+		entry(2, 0x0000000000000001), // distance 1 from #1
+		entry(3, 0x00000000000000FF), // distance 8 from #1, 7 from #2
+		entry(4, 0xFFFFFFFFFFFFFFFF), // distance 64 from #1
+	}
+
+	// Tight threshold: only the distance-1 pair qualifies.
+	got := buildPairs(entries, 2, nil)
+	if len(got) != 1 {
+		t.Fatalf("threshold 2: got %d pairs, want 1: %+v", len(got), got)
+	}
+	if got[0].FileA != id(1) || got[0].FileB != id(2) || got[0].Distance != 1 {
+		t.Errorf("threshold 2: unexpected pair %+v", got[0])
+	}
+	// Canonical order always FileA < FileB.
+	if bytes.Compare(got[0].FileA[:], got[0].FileB[:]) >= 0 {
+		t.Error("pair not in canonical FileA < FileB order")
+	}
+
+	// Looser threshold pulls in #3's pairs but never #4.
+	got8 := buildPairs(entries, 8, nil)
+	want := map[string]bool{
+		pairKey(domain.DuplicatePair{FileA: id(1), FileB: id(2), Distance: 1}): true,
+		pairKey(domain.DuplicatePair{FileA: id(1), FileB: id(3), Distance: 8}): true,
+		pairKey(domain.DuplicatePair{FileA: id(2), FileB: id(3), Distance: 7}): true,
+	}
+	if len(got8) != len(want) {
+		t.Fatalf("threshold 8: got %d pairs, want %d: %+v", len(got8), len(want), got8)
+	}
+	for _, p := range got8 {
+		if !want[pairKey(p)] {
+			t.Errorf("threshold 8: unexpected pair %+v", p)
+		}
+	}
+}
+
+func TestBuildPairs_IdenticalHashesPairAtDistanceZero(t *testing.T) {
+	entries := []domain.PHashEntry{
+		entry(1, 0xABCDABCDABCDABCD),
+		entry(2, 0xABCDABCDABCDABCD),
+	}
+	got := buildPairs(entries, 0, nil)
+	if len(got) != 1 || got[0].Distance != 0 || got[0].FileA != id(1) || got[0].FileB != id(2) {
+		t.Fatalf("identical hashes: got %+v, want one distance-0 pair (1,2)", got)
+	}
+}
+
+func TestClusterPairs_ConnectedComponents(t *testing.T) {
+	pairs := []domain.DuplicatePair{
+		{FileA: id(1), FileB: id(2)},
+		{FileA: id(2), FileB: id(3)}, // transitively joins 1-2-3
+		{FileA: id(5), FileB: id(6)},
+	}
+	clusters := clusterPairs(pairs)
+	if len(clusters) != 2 {
+		t.Fatalf("got %d clusters, want 2: %+v", len(clusters), clusters)
+	}
+	// Sorted by smallest id: {1,2,3} then {5,6}.
+	if len(clusters[0]) != 3 || clusters[0][0] != id(1) || clusters[0][2] != id(3) {
+		t.Errorf("cluster 0 = %v, want [1 2 3]", clusters[0])
+	}
+	if len(clusters[1]) != 2 || clusters[1][0] != id(5) {
+		t.Errorf("cluster 1 = %v, want [5 6]", clusters[1])
+	}
+	// Each cluster's ids are sorted.
+	for _, c := range clusters {
+		if !sort.SliceIsSorted(c, func(i, j int) bool { return bytes.Compare(c[i][:], c[j][:]) < 0 }) {
+			t.Errorf("cluster not sorted: %v", c)
+		}
+	}
+}
+
+func TestPickMetadata_Merge(t *testing.T) {
+	keep := json.RawMessage(`{"a":1,"b":2}`)
+	discard := json.RawMessage(`{"b":9,"c":3}`)
+
+	out := pickMetadata(mergeMerge, keep, discard)
+	var m map[string]int
+	if err := json.Unmarshal(out, &m); err != nil {
+		t.Fatalf("merge result not valid JSON: %v (%s)", err, out)
+	}
+	want := map[string]int{"a": 1, "b": 2, "c": 3} // survivor wins on "b"
+	if fmt.Sprint(m) != fmt.Sprint(want) {
+		t.Errorf("merge = %v, want %v", m, want)
+	}
+
+	if string(pickMetadata(mergeKeep, keep, discard)) != string(keep) {
+		t.Error("keep choice should return survivor metadata unchanged")
+	}
+	if string(pickMetadata(mergeDiscard, keep, discard)) != string(discard) {
+		t.Error("discard choice should return the other file's metadata")
+	}
+}
+
+func TestMergeSpec_Normalize(t *testing.T) {
+	// Empty fields default to "keep".
+	spec := MergeSpec{Keep: id(1), Discard: id(2)}
+	if err := spec.normalize(); err != nil {
+		t.Fatalf("normalize empty: %v", err)
+	}
+	if spec.Fields.OriginalName != mergeKeep || spec.Fields.Tags != mergeKeep || spec.Fields.Metadata != mergeKeep {
+		t.Errorf("empty fields not defaulted to keep: %+v", spec.Fields)
+	}
+
+	// "both" is invalid for a scalar field.
+	bad := MergeSpec{Keep: id(1), Discard: id(2), Fields: MergeFields{Notes: mergeBoth}}
+	if err := bad.normalize(); !errors.Is(err, domain.ErrValidation) {
+		t.Errorf("scalar=both: got %v, want ErrValidation", err)
+	}
+
+	// "discard" is invalid for a relation field.
+	badRel := MergeSpec{Keep: id(1), Discard: id(2), Fields: MergeFields{Tags: mergeDiscard}}
+	if err := badRel.normalize(); !errors.Is(err, domain.ErrValidation) {
+		t.Errorf("relation=discard: got %v, want ErrValidation", err)
+	}
+}

backend/internal/service/file_service.go

@@ -16,6 +16,7 @@ import (
 	"github.com/google/uuid"
 
 	"tanabata/backend/internal/domain"
+	"tanabata/backend/internal/imagehash"
 	"tanabata/backend/internal/port"
 )
 
@@ -154,6 +155,17 @@ func (s *FileService) Upload(ctx context.Context, p UploadParams) (*domain.File,
 	}
 	exifData, exifDatetime := extractMetadata(data, origName, p.ContentDatetimeFallback)
 
+	// Compute a perceptual hash for images so duplicate detection can later match
+	// near-identical files. Best-effort: a decode failure just leaves phash unset
+	// (the dedup CLI backfills it). Video is hashed by that CLI, not inline, to keep
+	// ffmpeg off the upload path.
+	var phash *int64
+	if strings.HasPrefix(mime.Name, "image/") {
+		if h, ok := imagehash.FromBytes(data); ok {
+			phash = &h
+		}
+	}
+
 	// Resolve content datetime: explicit > metadata date > fallback (e.g. import mtime) > zero.
 	var contentDatetime time.Time
 	if p.ContentDatetime != nil {
@@ -187,6 +199,7 @@ func (s *FileService) Upload(ctx context.Context, p UploadParams) (*domain.File,
 			Notes:           p.Notes,
 			Metadata:        p.Metadata,
 			EXIF:            exifData,
+			PHash:           phash,
 			CreatorID:       userID,
 			IsPublic:        p.IsPublic,
 		}
@@ -453,6 +466,18 @@ func (s *FileService) Replace(ctx context.Context, id uuid.UUID, p UploadParams)
 		return nil, err
 	}
 
+	// Recompute the perceptual hash from the new content: images inline, anything
+	// else cleared to NULL so the old content's hash never lingers (the dedup CLI
+	// recomputes video). Best-effort, like on upload — phash is recomputable.
+	var phash *int64
+	if strings.HasPrefix(mime.Name, "image/") {
+		if h, ok := imagehash.FromBytes(data); ok {
+			phash = &h
+		}
+	}
+	_ = s.files.SetPHash(ctx, id, phash)
+	updated.PHash = phash
+
 	objType := fileObjectType
 	_ = s.audit.Log(ctx, "file_replace", &objType, &id, nil)
 	return updated, nil

backend/internal/storage/disk.go

@@ -16,6 +16,8 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/disintegration/imaging"
@@ -155,6 +157,30 @@ func (s *DiskStorage) Preview(ctx context.Context, id uuid.UUID) (io.ReadCloser,
 	return s.serveGenerated(ctx, id, s.previewCachePath(id), s.previewWidth, s.previewHeight)
 }
 
+// VideoFrameMiddle decodes a representative frame from the middle of a video
+// (duration/2). The midpoint avoids the shared intros, title cards and black
+// lead-in frames that make a fixed early offset collide across unrelated clips,
+// so it is the right source for the video's perceptual (duplicate-detection)
+// hash. The file must already exist in storage; ffmpeg/ffprobe must be installed.
+// This is not part of port.FileStorage — only the dedup CLI needs it, with a
+// concrete *DiskStorage — so the interface stays lean and ffmpeg stays out of the
+// upload path.
+func (s *DiskStorage) VideoFrameMiddle(ctx context.Context, id uuid.UUID) (image.Image, error) {
+	srcPath := s.originalPath(id)
+	if _, err := os.Stat(srcPath); err != nil {
+		if os.IsNotExist(err) {
+			return nil, domain.ErrNotFound
+		}
+		return nil, fmt.Errorf("storage: stat %q: %w", srcPath, err)
+	}
+	// Fall back to a 1s offset if duration can't be probed — better a frame than none.
+	at := 1.0
+	if d, err := videoDurationSeconds(ctx, srcPath); err == nil && d > 0 {
+		at = d / 2
+	}
+	return extractVideoFrameAt(ctx, srcPath, at)
+}
+
 // ---------------------------------------------------------------------------
 // Internal helpers
 // ---------------------------------------------------------------------------
@@ -342,19 +368,25 @@ func (s *DiskStorage) vipsThumbnail(ctx context.Context, srcPath, cachePath stri
 	return f, nil
 }
 
-// extractVideoFrame uses ffmpeg to extract a single frame from a video file.
+// extractVideoFrame extracts a single frame ~1 second into the video — a safe
-// It seeks 1 second in (keyframe-accurate fast seek) and pipes the frame out
+// default for thumbnails. See extractVideoFrameAt for the mechanics.
-// as PNG. If the video is shorter than 1 s the seek is silently ignored by
-// ffmpeg and the first available frame is returned instead.
-// Returns an error if ffmpeg is not installed or produces no output. The run is
-// bounded by a timeout so a malformed file cannot hang the request indefinitely.
 func extractVideoFrame(ctx context.Context, srcPath string) (image.Image, error) {
+	return extractVideoFrameAt(ctx, srcPath, 1)
+}
+
+// extractVideoFrameAt uses ffmpeg to extract a single frame at atSec seconds into
+// the video, piped out as PNG. The fast input seek (-ss before -i) is keyframe-
+// accurate and cheap; if atSec is past the end the seek is silently ignored and
+// the first available frame is returned instead. Returns an error if ffmpeg is
+// not installed or produces no output. The run is bounded by a timeout so a
+// malformed file cannot hang the caller indefinitely.
+func extractVideoFrameAt(ctx context.Context, srcPath string, atSec float64) (image.Image, error) {
 	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
 
 	var out bytes.Buffer
 	cmd := exec.CommandContext(ctx, "ffmpeg",
-		"-ss", "1", // fast input seek; ignored gracefully on short files
+		"-ss", strconv.FormatFloat(atSec, 'f', 3, 64), // fast input seek; ignored gracefully past end
 		"-i", srcPath,
 		"-vframes", "1",
 		"-f", "image2",
@@ -370,6 +402,29 @@ func extractVideoFrame(ctx context.Context, srcPath string) (image.Image, error)
 	return imaging.Decode(&out)
 }
 
+// videoDurationSeconds returns the container duration in seconds via ffprobe.
+// Used to seek to the middle of a clip for perceptual hashing.
+func videoDurationSeconds(ctx context.Context, srcPath string) (float64, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, "ffprobe",
+		"-v", "error",
+		"-show_entries", "format=duration",
+		"-of", "default=noprint_wrappers=1:nokey=1",
+		srcPath,
+	)
+	out, err := cmd.Output()
+	if err != nil {
+		return 0, fmt.Errorf("ffprobe duration: %w", err)
+	}
+	d, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
+	if err != nil {
+		return 0, fmt.Errorf("ffprobe duration parse %q: %w", out, err)
+	}
+	return d, nil
+}
+
 // ---------------------------------------------------------------------------
 // Path helpers
 // ---------------------------------------------------------------------------

backend/migrations/003_data_tables.sql

@@ -92,6 +92,31 @@ CREATE TABLE data.file_pool (
     PRIMARY KEY (file_id, pool_id)
 );
 
+-- Precomputed near-duplicate candidates (phash Hamming distance <= threshold),
+-- (re)built in full by the dedup rescan. Stored once per unordered pair with a
+-- canonical file_a < file_b ordering so a pair is never duplicated as (a,b)/(b,a).
+CREATE TABLE data.duplicate_pairs (
+    file_a   uuid     NOT NULL REFERENCES data.files(id) ON UPDATE CASCADE ON DELETE CASCADE,
+    file_b   uuid     NOT NULL REFERENCES data.files(id) ON UPDATE CASCADE ON DELETE CASCADE,
+    distance smallint NOT NULL,
+
+    CONSTRAINT chk__duplicate_pairs__order CHECK (file_a < file_b),
+    PRIMARY KEY (file_a, file_b)
+);
+
+-- "Not a duplicate" decisions: a global overlay that hides a candidate pair from
+-- the duplicates view. Survives rescans (the pair may be re-found but stays
+-- hidden). Same canonical file_a < file_b ordering as data.duplicate_pairs.
+CREATE TABLE data.duplicate_dismissals (
+    file_a       uuid        NOT NULL REFERENCES data.files(id) ON UPDATE CASCADE ON DELETE CASCADE,
+    file_b       uuid        NOT NULL REFERENCES data.files(id) ON UPDATE CASCADE ON DELETE CASCADE,
+    dismissed_by smallint    NOT NULL REFERENCES core.users(id) ON UPDATE CASCADE ON DELETE RESTRICT,
+    dismissed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+
+    CONSTRAINT chk__duplicate_dismissals__order CHECK (file_a < file_b),
+    PRIMARY KEY (file_a, file_b)
+);
+
 COMMENT ON TABLE  data.categories IS 'Logical grouping of tags';
 COMMENT ON TABLE  data.tags       IS 'File labels/tags';
 COMMENT ON TABLE  data.tag_rules  IS 'Auto-tagging rules: when when_tag is assigned, then_tag follows';
@@ -99,6 +124,8 @@ COMMENT ON TABLE  data.files      IS 'Managed files; actual content stored on di
 COMMENT ON TABLE  data.file_tag   IS 'Many-to-many: files <-> tags';
 COMMENT ON TABLE  data.pools      IS 'Ordered collections of files';
 COMMENT ON TABLE  data.file_pool  IS 'Many-to-many: files <-> pools, with ordering';
+COMMENT ON TABLE  data.duplicate_pairs       IS 'Precomputed near-duplicate candidate pairs (perceptual-hash distance)';
+COMMENT ON TABLE  data.duplicate_dismissals  IS 'Pairs marked "not a duplicate"; hidden from the duplicates view';
 
 COMMENT ON COLUMN data.files.original_name    IS 'Original filename at upload time';
 COMMENT ON COLUMN data.files.content_datetime IS 'Content datetime (e.g. when photo was taken); falls back to EXIF DateTimeOriginal';
@@ -110,6 +137,8 @@ COMMENT ON COLUMN data.file_pool.position     IS 'Manual ordering within pool; u
 
 -- +goose Down
 
+DROP TABLE IF EXISTS data.duplicate_dismissals;
+DROP TABLE IF EXISTS data.duplicate_pairs;
 DROP TABLE IF EXISTS data.file_pool;
 DROP TABLE IF EXISTS data.pools;
 DROP TABLE IF EXISTS data.file_tag;

backend/migrations/006_indexes.sql

@@ -26,6 +26,12 @@ CREATE INDEX idx__files__needs_review      ON data.files USING btree (id) WHERE
 CREATE INDEX idx__file_tag__tag_id  ON data.file_tag USING hash (tag_id);
 CREATE INDEX idx__file_tag__file_id ON data.file_tag USING hash (file_id);
 
+-- data.duplicate_pairs / data.duplicate_dismissals
+-- The composite primary keys cover lookups on file_a; these add the file_b side
+-- (used by the ON DELETE CASCADE and by the visibility join on the second file).
+CREATE INDEX idx__duplicate_pairs__file_b      ON data.duplicate_pairs      USING hash (file_b);
+CREATE INDEX idx__duplicate_dismissals__file_b ON data.duplicate_dismissals USING hash (file_b);
+
 -- data.pools
 CREATE INDEX idx__pools__creator_id ON data.pools USING hash (creator_id);
 
@@ -70,6 +76,8 @@ DROP INDEX IF EXISTS activity.idx__sessions__token_hash;
 DROP INDEX IF EXISTS activity.idx__sessions__user_id;
 DROP INDEX IF EXISTS acl.idx__acl__user;
 DROP INDEX IF EXISTS acl.idx__acl__object;
+DROP INDEX IF EXISTS data.idx__duplicate_dismissals__file_b;
+DROP INDEX IF EXISTS data.idx__duplicate_pairs__file_b;
 DROP INDEX IF EXISTS data.idx__file_pool__file_id;
 DROP INDEX IF EXISTS data.idx__file_pool__pool_id;
 DROP INDEX IF EXISTS data.idx__pools__creator_id;

backend/migrations/007_seed_data.sql

@@ -21,6 +21,7 @@ INSERT INTO activity.action_types (name) VALUES
     -- Files
     ('file_create'), ('file_edit'), ('file_delete'), ('file_restore'),
     ('file_permanent_delete'), ('file_replace'), ('file_review'),
+    ('file_merge'), ('duplicate_dismiss'),
     -- Tags
     ('tag_create'), ('tag_edit'), ('tag_delete'),
     -- Categories

docker-compose.yml

@@ -114,6 +114,40 @@ services:
       timeout: 5s
       retries: 10
 
+  # One-shot maintenance task for duplicate detection: computes missing
+  # perceptual hashes (images + video) and rebuilds the duplicate-pairs table.
+  # It is NOT a daemon — the "tools" profile keeps it out of `docker compose up`;
+  # run it on demand, and it exits when done:
+  #
+  #   docker compose run --rm dedup            # hashes, then rebuild pairs
+  #   docker compose run --rm dedup -pairs     # only rebuild pairs (after uploads)
+  #   docker compose run --rm dedup -hashes    # only backfill hashes
+  #
+  # Reuses the app image, .env, volumes and networks; only the entrypoint differs
+  # (/app/dedup instead of the server). Connects to the same DB the app uses, so
+  # the app's DB (bundled or host) must be reachable when it runs.
+  dedup:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    profiles: ["tools"]
+    env_file: .env
+    networks:
+      - web
+      - backend
+    depends_on:
+      db:
+        condition: service_healthy
+        required: false
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    user: "${PUID:-42776}:${PGID:-42776}"
+    volumes:
+      - "${FILES_DIR:-app_files}:/data/files"
+      - "${THUMBS_DIR:-app_thumbs}:/data/thumbs"
+    entrypoint: ["/app/dedup"]
+    restart: "no"
+
 networks:
   # Public-facing bridge for this app. The explicit bridge name (instead of
   # Docker's random br-<hash>) makes it identifiable on the host for tcpdump and

frontend/src/lib/api/duplicates.ts

@@ -0,0 +1,52 @@
+import { api } from '$lib/api/client';
+import type { File } from '$lib/api/types';
+
+/** A group of mutually similar files. */
+export interface DuplicateCluster {
+	files: File[];
+}
+
+export interface DuplicateClusterPage {
+	items: DuplicateCluster[];
+	total: number;
+	offset: number;
+	limit: number;
+}
+
+/** Per-field source for a merge. Scalars choose keep/discard; relations
+ *  (tags, pools) choose keep/both; metadata can also be shallow-merged. */
+export type ScalarChoice = 'keep' | 'discard';
+export type RelationChoice = 'keep' | 'both';
+export type MetadataChoice = 'keep' | 'discard' | 'merge';
+
+export interface MergeFields {
+	original_name?: ScalarChoice;
+	notes?: ScalarChoice;
+	content_datetime?: ScalarChoice;
+	is_public?: ScalarChoice;
+	metadata?: MetadataChoice;
+	tags?: RelationChoice;
+	pools?: RelationChoice;
+}
+
+export interface ResolveRequest {
+	keep: string;
+	discard: string;
+	fields?: MergeFields;
+	delete_discarded?: boolean;
+}
+
+/** Fetch a page of duplicate clusters (server reads a precomputed table). */
+export function getDuplicates(limit = 20, offset = 0): Promise<DuplicateClusterPage> {
+	return api.get<DuplicateClusterPage>(`/files/duplicates?limit=${limit}&offset=${offset}`);
+}
+
+/** Mark two files as "not a duplicate" so the pair stops surfacing. */
+export function dismissDuplicate(a: string, b: string): Promise<void> {
+	return api.post<void>('/files/duplicates/dismiss', { file_id_a: a, file_id_b: b });
+}
+
+/** Merge a duplicate pair, returning the updated survivor. */
+export function resolveDuplicate(req: ResolveRequest): Promise<File> {
+	return api.post<File>('/files/duplicates/resolve', req);
+}

frontend/src/lib/components/file/DuplicateMergeDialog.svelte

@@ -0,0 +1,402 @@
+<script lang="ts">
+	import type { File } from '$lib/api/types';
+	import {
+		resolveDuplicate,
+		type MergeFields,
+		type MetadataChoice,
+		type RelationChoice,
+		type ScalarChoice
+	} from '$lib/api/duplicates';
+	import Thumb from '$lib/components/file/Thumb.svelte';
+
+	interface Props {
+		/** The two files to merge; `keep` is the default survivor (swappable here). */
+		keep: File;
+		discard: File;
+		/** Called with the updated survivor after a successful merge. */
+		onResolved: (survivor: File) => void;
+		onClose: () => void;
+	}
+
+	let { keep, discard, onResolved, onClose }: Props = $props();
+
+	// Which file survives is swappable; derive the two sides from a single flag so
+	// the choice stays in sync with the props.
+	let swapped = $state(false);
+	let a = $derived<File>(swapped ? discard : keep);
+	let b = $derived<File>(swapped ? keep : discard);
+
+	// Per-field source, all defaulting to the survivor ("keep").
+	let original_name = $state<ScalarChoice>('keep');
+	let notes = $state<ScalarChoice>('keep');
+	let content_datetime = $state<ScalarChoice>('keep');
+	let is_public = $state<ScalarChoice>('keep');
+	let metadata = $state<MetadataChoice>('keep');
+	let tags = $state<RelationChoice>('keep');
+	let pools = $state<RelationChoice>('keep');
+	let deleteDiscarded = $state(true);
+
+	let busy = $state(false);
+	let error = $state('');
+
+	function swap() {
+		swapped = !swapped;
+	}
+
+	function fmtDate(s?: string | null): string {
+		if (!s) return '—';
+		const d = new Date(s);
+		return isNaN(d.getTime()) ? s : d.toLocaleString();
+	}
+	function metaCount(m: unknown): number {
+		return m && typeof m === 'object' ? Object.keys(m as object).length : 0;
+	}
+
+	async function submit() {
+		if (busy) return;
+		busy = true;
+		error = '';
+		const fields: MergeFields = {
+			original_name,
+			notes,
+			content_datetime,
+			is_public,
+			metadata,
+			tags,
+			pools
+		};
+		try {
+			const survivor = await resolveDuplicate({
+				keep: a.id,
+				discard: b.id,
+				fields,
+				delete_discarded: deleteDiscarded
+			});
+			onResolved(survivor);
+			onClose();
+		} catch {
+			error = 'Failed to merge';
+			busy = false;
+		}
+	}
+</script>
+
+<!-- svelte-ignore a11y_click_events_have_key_events a11y_no_static_element_interactions -->
+<div class="backdrop" role="presentation" onclick={onClose}></div>
+<div class="sheet" class:busy role="dialog" aria-label="Merge duplicates">
+	<div class="head">
+		<span class="title">Merge duplicates</span>
+		<button class="x" onclick={onClose} aria-label="Close">
+			<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
+				<path d="M3 3l10 10M13 3L3 13" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" />
+			</svg>
+		</button>
+	</div>
+
+	<div class="body">
+		<!-- Survivor / other headers -->
+		<div class="files">
+			<div class="file">
+				<Thumb id={a.id} size={80} alt={a.original_name ?? ''} />
+				<span class="badge keep">Keep</span>
+				<span class="fname" title={a.original_name ?? ''}>{a.original_name ?? '—'}</span>
+			</div>
+			<button class="swap" onclick={swap} title="Swap which file is kept" aria-label="Swap">
+				<svg width="18" height="18" viewBox="0 0 18 18" fill="none" aria-hidden="true">
+					<path
+						d="M5 4h8l-2.5-2.5M13 14H5l2.5 2.5"
+						stroke="currentColor"
+						stroke-width="1.6"
+						stroke-linecap="round"
+						stroke-linejoin="round"
+					/>
+				</svg>
+			</button>
+			<div class="file">
+				<Thumb id={b.id} size={80} alt={b.original_name ?? ''} />
+				<span class="badge other">Other</span>
+				<span class="fname" title={b.original_name ?? ''}>{b.original_name ?? '—'}</span>
+			</div>
+		</div>
+
+		<!-- Scalar fields: keep vs discard -->
+		{#snippet scalarRow(label: string, value: ScalarChoice, set: (v: ScalarChoice) => void, keepVal: string, otherVal: string)}
+			<div class="row">
+				<span class="label">{label}</span>
+				<div class="seg">
+					<button class:on={value === 'keep'} onclick={() => set('keep')} title={keepVal}>
+						{keepVal || '—'}
+					</button>
+					<button class:on={value === 'discard'} onclick={() => set('discard')} title={otherVal}>
+						{otherVal || '—'}
+					</button>
+				</div>
+			</div>
+		{/snippet}
+
+		{@render scalarRow(
+			'Name',
+			original_name,
+			(v) => (original_name = v),
+			a.original_name ?? '',
+			b.original_name ?? ''
+		)}
+		{@render scalarRow('Notes', notes, (v) => (notes = v), a.notes ?? '', b.notes ?? '')}
+		{@render scalarRow(
+			'Date',
+			content_datetime,
+			(v) => (content_datetime = v),
+			fmtDate(a.content_datetime),
+			fmtDate(b.content_datetime)
+		)}
+		{@render scalarRow(
+			'Visibility',
+			is_public,
+			(v) => (is_public = v),
+			a.is_public ? 'Public' : 'Private',
+			b.is_public ? 'Public' : 'Private'
+		)}
+
+		<!-- Metadata: keep / other / merge -->
+		<div class="row">
+			<span class="label">Metadata</span>
+			<div class="seg">
+				<button class:on={metadata === 'keep'} onclick={() => (metadata = 'keep')}>
+					Keep ({metaCount(a.metadata)})
+				</button>
+				<button class:on={metadata === 'discard'} onclick={() => (metadata = 'discard')}>
+					Other ({metaCount(b.metadata)})
+				</button>
+				<button class:on={metadata === 'merge'} onclick={() => (metadata = 'merge')}>Merge</button>
+			</div>
+		</div>
+
+		<!-- Relations: keep vs union both -->
+		<div class="row">
+			<span class="label">Tags</span>
+			<div class="seg">
+				<button class:on={tags === 'keep'} onclick={() => (tags = 'keep')}>
+					Keep ({a.tags?.length ?? 0})
+				</button>
+				<button class:on={tags === 'both'} onclick={() => (tags = 'both')}>Union both</button>
+			</div>
+		</div>
+		<div class="row">
+			<span class="label">Pools</span>
+			<div class="seg">
+				<button class:on={pools === 'keep'} onclick={() => (pools = 'keep')}>Keep</button>
+				<button class:on={pools === 'both'} onclick={() => (pools = 'both')}>Union both</button>
+			</div>
+		</div>
+
+		<label class="del">
+			<input type="checkbox" bind:checked={deleteDiscarded} />
+			Move the “Other” file to trash after merging
+		</label>
+
+		{#if error}<p class="error">{error}</p>{/if}
+	</div>
+
+	<div class="foot">
+		<button class="btn ghost" onclick={onClose}>Cancel</button>
+		<button class="btn primary" onclick={submit} disabled={busy}>Merge</button>
+	</div>
+</div>
+
+<style>
+	.backdrop {
+		position: fixed;
+		inset: 0;
+		z-index: 120;
+		background: rgba(0, 0, 0, 0.5);
+	}
+	.sheet {
+		position: fixed;
+		left: 0;
+		right: 0;
+		bottom: 0;
+		z-index: 121;
+		background-color: var(--color-bg-secondary);
+		border-radius: 14px 14px 0 0;
+		padding-bottom: env(safe-area-inset-bottom, 0px);
+		max-height: 88dvh;
+		display: flex;
+		flex-direction: column;
+		animation: slide-up 0.18s ease-out;
+	}
+	.sheet.busy {
+		opacity: 0.6;
+		pointer-events: none;
+	}
+	@keyframes slide-up {
+		from {
+			transform: translateY(20px);
+			opacity: 0;
+		}
+		to {
+			transform: translateY(0);
+			opacity: 1;
+		}
+	}
+	.head {
+		display: flex;
+		align-items: center;
+		padding: 14px 16px 10px;
+		gap: 8px;
+	}
+	.title {
+		flex: 1;
+		font-size: 0.95rem;
+		font-weight: 600;
+	}
+	.x {
+		background: none;
+		border: none;
+		cursor: pointer;
+		color: var(--color-text-muted);
+		padding: 4px;
+		display: flex;
+	}
+	.x:hover {
+		color: var(--color-text-primary);
+	}
+	.body {
+		overflow-y: auto;
+		padding: 0 14px 8px;
+	}
+	.files {
+		display: flex;
+		align-items: center;
+		justify-content: center;
+		gap: 12px;
+		margin-bottom: 12px;
+	}
+	.file {
+		display: flex;
+		flex-direction: column;
+		align-items: center;
+		gap: 4px;
+		min-width: 0;
+		max-width: 40%;
+	}
+	.badge {
+		font-size: 0.7rem;
+		font-weight: 600;
+		padding: 1px 7px;
+		border-radius: 6px;
+	}
+	.badge.keep {
+		background-color: color-mix(in srgb, var(--color-accent) 30%, transparent);
+		color: var(--color-accent);
+	}
+	.badge.other {
+		background-color: var(--color-bg-elevated);
+		color: var(--color-text-muted);
+	}
+	.fname {
+		font-size: 0.78rem;
+		color: var(--color-text-muted);
+		max-width: 100%;
+		white-space: nowrap;
+		overflow: hidden;
+		text-overflow: ellipsis;
+	}
+	.swap {
+		background-color: var(--color-bg-elevated);
+		border: 1px solid color-mix(in srgb, var(--color-accent) 30%, transparent);
+		color: var(--color-text-muted);
+		border-radius: 8px;
+		width: 32px;
+		height: 32px;
+		display: flex;
+		align-items: center;
+		justify-content: center;
+		cursor: pointer;
+		flex-shrink: 0;
+	}
+	.swap:hover {
+		color: var(--color-accent);
+		border-color: var(--color-accent);
+	}
+	.row {
+		display: flex;
+		align-items: center;
+		gap: 10px;
+		padding: 7px 0;
+		border-top: 1px solid color-mix(in srgb, var(--color-accent) 12%, transparent);
+	}
+	.label {
+		font-size: 0.82rem;
+		color: var(--color-text-muted);
+		width: 74px;
+		flex-shrink: 0;
+	}
+	.seg {
+		display: flex;
+		flex: 1;
+		gap: 4px;
+		min-width: 0;
+	}
+	.seg button {
+		flex: 1;
+		min-width: 0;
+		padding: 6px 8px;
+		border-radius: 7px;
+		border: 1px solid color-mix(in srgb, var(--color-accent) 25%, transparent);
+		background-color: var(--color-bg-elevated);
+		color: var(--color-text-muted);
+		font-size: 0.8rem;
+		font-family: inherit;
+		cursor: pointer;
+		white-space: nowrap;
+		overflow: hidden;
+		text-overflow: ellipsis;
+	}
+	.seg button.on {
+		background-color: color-mix(in srgb, var(--color-accent) 22%, var(--color-bg-elevated));
+		color: var(--color-accent);
+		border-color: var(--color-accent);
+	}
+	.del {
+		display: flex;
+		align-items: center;
+		gap: 8px;
+		font-size: 0.82rem;
+		color: var(--color-text-muted);
+		padding: 12px 0 4px;
+	}
+	.error {
+		color: var(--color-danger);
+		font-size: 0.85rem;
+		text-align: center;
+	}
+	.foot {
+		display: flex;
+		gap: 8px;
+		padding: 10px 14px calc(10px + env(safe-area-inset-bottom, 0px));
+		border-top: 1px solid color-mix(in srgb, var(--color-accent) 15%, transparent);
+	}
+	.btn {
+		flex: 1;
+		height: 38px;
+		border-radius: 8px;
+		border: 1px solid transparent;
+		font-size: 0.9rem;
+		font-family: inherit;
+		cursor: pointer;
+	}
+	.btn.ghost {
+		background-color: var(--color-bg-elevated);
+		border-color: color-mix(in srgb, var(--color-accent) 25%, transparent);
+		color: var(--color-text-muted);
+	}
+	.btn.primary {
+		background-color: var(--color-accent);
+		color: var(--color-bg-primary);
+		font-weight: 600;
+	}
+	.btn.primary:disabled {
+		opacity: 0.6;
+		cursor: default;
+	}
+</style>

frontend/src/lib/components/file/Thumb.svelte

@@ -0,0 +1,98 @@
+<script lang="ts">
+	import { get } from 'svelte/store';
+	import { authStore } from '$lib/stores/auth';
+
+	interface Props {
+		/** File id whose thumbnail to load. */
+		id: string;
+		alt?: string;
+		/** Square edge length in px. */
+		size?: number;
+	}
+
+	let { id, alt = '', size = 96 }: Props = $props();
+
+	let imgSrc = $state<string | null>(null);
+	let failed = $state(false);
+
+	// Thumbnails are auth-gated, so fetch with the bearer token and render the blob
+	// (mirrors FileCard's loader). Re-runs whenever the id changes.
+	$effect(() => {
+		const token = get(authStore).accessToken;
+		let objectUrl: string | null = null;
+		let cancelled = false;
+		imgSrc = null;
+		failed = false;
+
+		fetch(`/api/v1/files/${id}/thumbnail`, {
+			headers: token ? { Authorization: `Bearer ${token}` } : {}
+		})
+			.then((res) => (res.ok ? res.blob() : null))
+			.then((blob) => {
+				if (cancelled || !blob) {
+					if (!cancelled) failed = true;
+					return;
+				}
+				objectUrl = URL.createObjectURL(blob);
+				imgSrc = objectUrl;
+			})
+			.catch(() => {
+				if (!cancelled) failed = true;
+			});
+
+		return () => {
+			cancelled = true;
+			if (objectUrl) URL.revokeObjectURL(objectUrl);
+		};
+	});
+</script>
+
+<div class="thumb" style="width:{size}px;height:{size}px">
+	{#if imgSrc}
+		<img src={imgSrc} {alt} draggable="false" />
+	{:else if failed}
+		<div class="ph failed" aria-label="Failed to load"></div>
+	{:else}
+		<div class="ph loading" aria-label="Loading"></div>
+	{/if}
+</div>
+
+<style>
+	.thumb {
+		overflow: hidden;
+		border-radius: 8px;
+		background-color: var(--color-bg-elevated);
+		flex-shrink: 0;
+	}
+	img {
+		width: 100%;
+		height: 100%;
+		object-fit: contain;
+		display: block;
+	}
+	.ph {
+		width: 100%;
+		height: 100%;
+	}
+	.ph.loading {
+		background: linear-gradient(
+			90deg,
+			var(--color-bg-elevated) 25%,
+			color-mix(in srgb, var(--color-accent) 12%, var(--color-bg-elevated)) 50%,
+			var(--color-bg-elevated) 75%
+		);
+		background-size: 200% 100%;
+		animation: shimmer 1.4s infinite;
+	}
+	.ph.failed {
+		background-color: color-mix(in srgb, var(--color-danger) 15%, var(--color-bg-elevated));
+	}
+	@keyframes shimmer {
+		0% {
+			background-position: 200% 0;
+		}
+		100% {
+			background-position: -200% 0;
+		}
+	}
+</style>

frontend/src/lib/components/layout/Header.svelte

@@ -12,6 +12,7 @@
 		onFilterToggle: () => void;
 		onUpload?: () => void;
 		onTrash?: () => void;
+		onDuplicates?: () => void;
 	}
 
 	let {
@@ -23,7 +24,8 @@
 		onOrderToggle,
 		onFilterToggle,
 		onUpload,
-		onTrash
+		onTrash,
+		onDuplicates
 	}: Props = $props();
 </script>
 
@@ -51,6 +53,20 @@
 		</button>
 	{/if}
 
+	{#if onDuplicates}
+		<button class="icon-btn dup-btn" onclick={onDuplicates} title="Duplicates">
+			<svg width="15" height="15" viewBox="0 0 15 15" fill="none" aria-hidden="true">
+				<rect x="2" y="2" width="8" height="8" rx="1.5" stroke="currentColor" stroke-width="1.5" />
+				<path
+					d="M5 12.5h6A1.5 1.5 0 0 0 12.5 11V5"
+					stroke="currentColor"
+					stroke-width="1.5"
+					stroke-linecap="round"
+				/>
+			</svg>
+		</button>
+	{/if}
+
 	{#if onTrash}
 		<button class="icon-btn trash-btn" onclick={onTrash} title="Trash">
 			<svg width="15" height="15" viewBox="0 0 15 15" fill="none" aria-hidden="true">

frontend/src/routes/files/+page.svelte

@@ -722,6 +722,7 @@
 		onFilterToggle={() => (filterOpen = !filterOpen)}
 		onUpload={() => uploader?.open()}
 		onTrash={() => goto('/files/trash')}
+		onDuplicates={() => goto('/files/duplicates')}
 	/>
 
 	{#if filterOpen}

frontend/src/routes/files/duplicates/+page.svelte

@@ -0,0 +1,377 @@
+<script lang="ts">
+	import { goto } from '$app/navigation';
+	import { api } from '$lib/api/client';
+	import { getDuplicates, dismissDuplicate, type DuplicateCluster } from '$lib/api/duplicates';
+	import Thumb from '$lib/components/file/Thumb.svelte';
+	import DuplicateMergeDialog from '$lib/components/file/DuplicateMergeDialog.svelte';
+	import type { File } from '$lib/api/types';
+
+	const LIMIT = 20;
+
+	let clusters = $state<DuplicateCluster[]>([]);
+	let total = $state(0);
+	let loading = $state(false);
+	let initialLoaded = $state(false);
+	let error = $state('');
+	let busyKey = $state(''); // cluster currently performing an action
+
+	// Which file is the survivor for a given cluster (keyed by its file-id set).
+	let keepers = $state<Record<string, string>>({});
+
+	// Merge dialog state.
+	let mergeKeep = $state<File | null>(null);
+	let mergeDiscard = $state<File | null>(null);
+
+	$effect(() => {
+		if (!initialLoaded && !loading) void load();
+	});
+
+	function clusterKey(c: DuplicateCluster): string {
+		return c.files.map((f) => f.id).join(',');
+	}
+	function keeperId(c: DuplicateCluster): string {
+		return keepers[clusterKey(c)] ?? c.files[0]?.id ?? '';
+	}
+
+	async function load() {
+		if (loading) return;
+		loading = true;
+		error = '';
+		try {
+			const res = await getDuplicates(LIMIT, clusters.length);
+			clusters = [...clusters, ...(res.items ?? [])];
+			total = res.total ?? clusters.length;
+		} catch {
+			error = 'Failed to load duplicates';
+		} finally {
+			loading = false;
+			initialLoaded = true;
+		}
+	}
+
+	async function reload() {
+		clusters = [];
+		keepers = {};
+		total = 0;
+		initialLoaded = false;
+		await load();
+	}
+
+	function setKeeper(c: DuplicateCluster, id: string) {
+		keepers = { ...keepers, [clusterKey(c)]: id };
+	}
+
+	function openMerge(c: DuplicateCluster, other: File) {
+		const keep = c.files.find((f) => f.id === keeperId(c));
+		if (!keep) return;
+		mergeKeep = keep;
+		mergeDiscard = other;
+	}
+
+	async function deleteFile(c: DuplicateCluster, id: string) {
+		if (busyKey) return;
+		busyKey = clusterKey(c);
+		try {
+			await api.post('/files/bulk/delete', { file_ids: [id] });
+			await reload();
+		} catch {
+			error = 'Failed to delete file';
+		} finally {
+			busyKey = '';
+		}
+	}
+
+	async function notDuplicate(c: DuplicateCluster, other: File) {
+		if (busyKey) return;
+		busyKey = clusterKey(c);
+		try {
+			await dismissDuplicate(keeperId(c), other.id);
+			await reload();
+		} catch {
+			error = 'Failed to dismiss pair';
+		} finally {
+			busyKey = '';
+		}
+	}
+
+	function onMerged() {
+		mergeKeep = null;
+		mergeDiscard = null;
+		void reload();
+	}
+
+	let hasMore = $derived(clusters.length < total);
+</script>
+
+<svelte:head>
+	<title>Duplicates | Tanabata</title>
+</svelte:head>
+
+<div class="page">
+	<header>
+		<button class="back" onclick={() => goto('/files')} aria-label="Back to files">
+			<svg width="18" height="18" viewBox="0 0 18 18" fill="none" aria-hidden="true">
+				<path
+					d="M11 4l-5 5 5 5"
+					stroke="currentColor"
+					stroke-width="1.8"
+					stroke-linecap="round"
+					stroke-linejoin="round"
+				/>
+			</svg>
+		</button>
+		<span class="htitle">Duplicates{total ? ` (${total})` : ''}</span>
+		<button class="refresh" onclick={reload} title="Refresh" aria-label="Refresh">
+			<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
+				<path
+					d="M13 8a5 5 0 1 1-1.5-3.5M13 2v3h-3"
+					stroke="currentColor"
+					stroke-width="1.6"
+					stroke-linecap="round"
+					stroke-linejoin="round"
+				/>
+			</svg>
+		</button>
+	</header>
+
+	<main>
+		{#if error}<p class="error" role="alert">{error}</p>{/if}
+
+		{#if initialLoaded && clusters.length === 0 && !error}
+			<div class="empty">
+				<p>No duplicates found.</p>
+				<p class="hint">
+					The list reflects the last <code>dedup</code> run. New uploads appear after the next rescan.
+				</p>
+			</div>
+		{/if}
+
+		{#each clusters as c (clusterKey(c))}
+			{@const keep = keeperId(c)}
+			<section class="cluster" class:busy={busyKey === clusterKey(c)}>
+				<div class="files">
+					{#each c.files as f (f.id)}
+						<!-- svelte-ignore a11y_click_events_have_key_events a11y_no_static_element_interactions -->
+						<div
+							class="file"
+							class:keep={f.id === keep}
+							role="button"
+							tabindex="0"
+							onclick={() => setKeeper(c, f.id)}
+							title="Click to keep this one"
+						>
+							<Thumb id={f.id} size={96} alt={f.original_name ?? ''} />
+							{#if f.id === keep}<span class="kbadge">Keep</span>{/if}
+							<span class="fname" title={f.original_name ?? ''}>{f.original_name ?? '—'}</span>
+							<span class="fmeta">{f.mime_type} · {f.tags?.length ?? 0} tags</span>
+						</div>
+					{/each}
+				</div>
+
+				<div class="actions">
+					{#each c.files.filter((f) => f.id !== keep) as other (other.id)}
+						<div class="actrow">
+							<span class="aname" title={other.original_name ?? ''}>{other.original_name ?? '—'}</span>
+							<button class="abtn" onclick={() => openMerge(c, other)}>Merge</button>
+							<button class="abtn" onclick={() => deleteFile(c, other.id)}>Delete</button>
+							<button class="abtn ghost" onclick={() => notDuplicate(c, other)}>Not a dup</button>
+						</div>
+					{/each}
+				</div>
+			</section>
+		{/each}
+
+		{#if hasMore}
+			<button class="more" onclick={load} disabled={loading}>
+				{loading ? 'Loading…' : 'Load more'}
+			</button>
+		{:else if loading && !initialLoaded}
+			<p class="loadingp">Loading…</p>
+		{/if}
+	</main>
+</div>
+
+{#if mergeKeep && mergeDiscard}
+	<DuplicateMergeDialog
+		keep={mergeKeep}
+		discard={mergeDiscard}
+		onResolved={onMerged}
+		onClose={() => {
+			mergeKeep = null;
+			mergeDiscard = null;
+		}}
+	/>
+{/if}
+
+<style>
+	.page {
+		display: flex;
+		flex-direction: column;
+		height: 100%;
+	}
+	header {
+		display: flex;
+		align-items: center;
+		gap: 8px;
+		padding: 8px 12px;
+		background-color: var(--color-bg-primary);
+		border-bottom: 1px solid color-mix(in srgb, var(--color-accent) 15%, transparent);
+		position: sticky;
+		top: 0;
+		z-index: 10;
+		flex-shrink: 0;
+	}
+	.back,
+	.refresh {
+		width: 32px;
+		height: 32px;
+		display: flex;
+		align-items: center;
+		justify-content: center;
+		border-radius: 7px;
+		border: 1px solid color-mix(in srgb, var(--color-accent) 25%, transparent);
+		background-color: var(--color-bg-elevated);
+		color: var(--color-text-muted);
+		cursor: pointer;
+	}
+	.back:hover,
+	.refresh:hover {
+		color: var(--color-text-primary);
+		border-color: var(--color-accent);
+	}
+	.htitle {
+		flex: 1;
+		font-size: 0.95rem;
+		font-weight: 600;
+	}
+	main {
+		flex: 1;
+		overflow-y: auto;
+		padding: 10px 12px calc(72px + env(safe-area-inset-bottom, 0px));
+	}
+	.error {
+		color: var(--color-danger);
+		font-size: 0.9rem;
+		text-align: center;
+	}
+	.empty {
+		text-align: center;
+		color: var(--color-text-muted);
+		padding: 40px 16px;
+	}
+	.empty .hint {
+		font-size: 0.82rem;
+		opacity: 0.8;
+	}
+	code {
+		font-family: monospace;
+		background-color: var(--color-bg-elevated);
+		padding: 0 4px;
+		border-radius: 4px;
+	}
+	.cluster {
+		background-color: var(--color-bg-secondary);
+		border-radius: 12px;
+		padding: 12px;
+		margin-bottom: 12px;
+	}
+	.cluster.busy {
+		opacity: 0.55;
+		pointer-events: none;
+	}
+	.files {
+		display: flex;
+		flex-wrap: wrap;
+		gap: 10px;
+		margin-bottom: 10px;
+	}
+	.file {
+		display: flex;
+		flex-direction: column;
+		align-items: center;
+		gap: 3px;
+		width: 96px;
+		cursor: pointer;
+		border-radius: 10px;
+		padding: 4px;
+		border: 2px solid transparent;
+	}
+	.file.keep {
+		border-color: var(--color-accent);
+		background-color: color-mix(in srgb, var(--color-accent) 10%, transparent);
+	}
+	.kbadge {
+		font-size: 0.68rem;
+		font-weight: 600;
+		color: var(--color-accent);
+	}
+	.fname {
+		font-size: 0.74rem;
+		max-width: 96px;
+		white-space: nowrap;
+		overflow: hidden;
+		text-overflow: ellipsis;
+		color: var(--color-text-primary);
+	}
+	.fmeta {
+		font-size: 0.68rem;
+		color: var(--color-text-muted);
+		max-width: 96px;
+		white-space: nowrap;
+		overflow: hidden;
+		text-overflow: ellipsis;
+	}
+	.actions {
+		display: flex;
+		flex-direction: column;
+		gap: 6px;
+	}
+	.actrow {
+		display: flex;
+		align-items: center;
+		gap: 6px;
+	}
+	.aname {
+		flex: 1;
+		min-width: 0;
+		font-size: 0.78rem;
+		color: var(--color-text-muted);
+		white-space: nowrap;
+		overflow: hidden;
+		text-overflow: ellipsis;
+	}
+	.abtn {
+		padding: 5px 10px;
+		border-radius: 7px;
+		border: 1px solid color-mix(in srgb, var(--color-accent) 25%, transparent);
+		background-color: var(--color-bg-elevated);
+		color: var(--color-text-primary);
+		font-size: 0.78rem;
+		font-family: inherit;
+		cursor: pointer;
+		flex-shrink: 0;
+	}
+	.abtn:hover {
+		border-color: var(--color-accent);
+	}
+	.abtn.ghost {
+		color: var(--color-text-muted);
+	}
+	.more {
+		display: block;
+		width: 100%;
+		padding: 10px;
+		border-radius: 8px;
+		border: 1px solid color-mix(in srgb, var(--color-accent) 25%, transparent);
+		background-color: var(--color-bg-elevated);
+		color: var(--color-text-primary);
+		font-family: inherit;
+		font-size: 0.85rem;
+		cursor: pointer;
+	}
+	.loadingp {
+		text-align: center;
+		color: var(--color-text-muted);
+		font-size: 0.9rem;
+	}
+</style>

openapi.yaml

@@ -709,6 +709,89 @@ paths:
                       type: string
                       format: uuid
 
+  # --- Duplicate detection ---
+  /files/duplicates:
+    get:
+      tags: [Files]
+      summary: List duplicate clusters
+      description: >-
+        Groups of perceptually similar files (within the server's hash-distance
+        threshold), read from a precomputed pairs table — this never compares all
+        files on each call. Pairs are (re)built offline by the dedup tool, so the
+        result reflects state as of the last rescan. Only files the caller may view
+        are included; dismissed and trashed pairs are excluded.
+      parameters:
+        - name: limit
+          in: query
+          schema:
+            type: integer
+            default: 20
+            minimum: 1
+            maximum: 50
+          description: Maximum number of clusters to return
+        - name: offset
+          in: query
+          schema:
+            type: integer
+            default: 0
+            minimum: 0
+      responses:
+        '200':
+          description: A page of duplicate clusters
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DuplicateClusterPage'
+
+  /files/duplicates/dismiss:
+    post:
+      tags: [Files]
+      summary: Mark two files as not duplicates
+      description: >-
+        Records a global "not a duplicate" decision so the pair stops appearing in
+        the duplicates view (it survives future rescans). The caller must be able
+        to view both files.
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: [file_id_a, file_id_b]
+              properties:
+                file_id_a:
+                  type: string
+                  format: uuid
+                file_id_b:
+                  type: string
+                  format: uuid
+      responses:
+        '204':
+          description: Pair dismissed
+
+  /files/duplicates/resolve:
+    post:
+      tags: [Files]
+      summary: Resolve a duplicate by merging two files
+      description: >-
+        Keeps one file and folds the chosen fields in from the other, then (by
+        default) trashes the other. The caller must be able to edit both files. To
+        simply delete one/both or to keep both, use the bulk-delete and dismiss
+        endpoints instead. Returns the updated survivor.
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/DuplicateResolve'
+      responses:
+        '200':
+          description: The updated surviving file
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/File'
+
   # --- File import ---
   /files/import:
     post:
@@ -1766,6 +1849,19 @@ components:
     # --- File ---
     File:
       type: object
+      required:
+        - id
+        - mime_type
+        - mime_extension
+        - content_datetime
+        - exif
+        - creator_id
+        - creator_name
+        - is_public
+        - is_deleted
+        - needs_review
+        - created_at
+        - tags
       properties:
         id:
           type: string
@@ -1814,6 +1910,11 @@ components:
           type: string
           format: date-time
           description: Extracted from UUID v7
+        tags:
+          type: array
+          description: Tags assigned to the file
+          items:
+            $ref: '#/components/schemas/Tag'
 
     FileUpdate:
       type: object
@@ -1846,6 +1947,83 @@ components:
           nullable: true
           description: Cursor for loading previous (backward) page; null if at the beginning
 
+    # --- Duplicates ---
+    DuplicateCluster:
+      type: object
+      properties:
+        files:
+          type: array
+          description: Two or more mutually similar files
+          items:
+            $ref: '#/components/schemas/File'
+
+    DuplicateClusterPage:
+      type: object
+      properties:
+        items:
+          type: array
+          items:
+            $ref: '#/components/schemas/DuplicateCluster'
+        total:
+          type: integer
+          description: Total number of clusters (not files)
+        offset:
+          type: integer
+        limit:
+          type: integer
+
+    MergeScalarChoice:
+      type: string
+      enum: [keep, discard]
+      default: keep
+      description: Take this field's value from the kept file or the discarded one
+
+    MergeRelationChoice:
+      type: string
+      enum: [keep, both]
+      default: keep
+      description: Keep only the survivor's relations, or union both files' relations
+
+    DuplicateResolve:
+      type: object
+      required: [keep, discard]
+      properties:
+        keep:
+          type: string
+          format: uuid
+          description: The file to keep (the survivor)
+        discard:
+          type: string
+          format: uuid
+          description: The other file in the pair
+        delete_discarded:
+          type: boolean
+          default: true
+          description: Move the discarded file to trash after merging
+        fields:
+          type: object
+          description: Per-field source for the merge; omitted fields default to "keep"
+          properties:
+            original_name:
+              $ref: '#/components/schemas/MergeScalarChoice'
+            notes:
+              $ref: '#/components/schemas/MergeScalarChoice'
+            content_datetime:
+              $ref: '#/components/schemas/MergeScalarChoice'
+            is_public:
+              $ref: '#/components/schemas/MergeScalarChoice'
+            metadata:
+              type: string
+              enum: [keep, discard, merge]
+              default: keep
+              description: >-
+                Keep or take the discarded file's metadata object, or shallow-merge
+                them with the survivor winning on key conflicts
+            tags:
+              $ref: '#/components/schemas/MergeRelationChoice'
+            pools:
+              $ref: '#/components/schemas/MergeRelationChoice'
+
     # --- Tag ---
     Tag:
       type: object