Compare commits
2 Commits
281358ff04
...
a16accf443
| Author | SHA1 | Date | |
|---|---|---|---|
| a16accf443 | |||
| 78b5e86dd6 |
@@ -47,12 +47,16 @@ func (h *DuplicateHandler) List(c *gin.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
items := make([]gin.H, len(clusters))
|
items := make([]gin.H, len(clusters))
|
||||||
for i, files := range clusters {
|
for i, cl := range clusters {
|
||||||
fs := make([]fileJSON, len(files))
|
fs := make([]fileJSON, len(cl.Files))
|
||||||
for j, f := range files {
|
for j, f := range cl.Files {
|
||||||
fs[j] = toFileJSON(f)
|
fs[j] = toFileJSON(f)
|
||||||
}
|
}
|
||||||
items[i] = gin.H{"files": fs}
|
dists := make([]gin.H, len(cl.Distances))
|
||||||
|
for j, d := range cl.Distances {
|
||||||
|
dists[j] = gin.H{"a": d.A, "b": d.B, "distance": d.Distance}
|
||||||
|
}
|
||||||
|
items[i] = gin.H{"files": fs, "distances": dists}
|
||||||
}
|
}
|
||||||
respondJSON(c, http.StatusOK, gin.H{
|
respondJSON(c, http.StatusOK, gin.H{
|
||||||
"items": items,
|
"items": items,
|
||||||
|
|||||||
@@ -1661,6 +1661,11 @@ type dupListResponse struct {
|
|||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
} `json:"tags"`
|
} `json:"tags"`
|
||||||
} `json:"files"`
|
} `json:"files"`
|
||||||
|
Distances []struct {
|
||||||
|
A string `json:"a"`
|
||||||
|
B string `json:"b"`
|
||||||
|
Distance int `json:"distance"`
|
||||||
|
} `json:"distances"`
|
||||||
} `json:"items"`
|
} `json:"items"`
|
||||||
Total int `json:"total"`
|
Total int `json:"total"`
|
||||||
}
|
}
|
||||||
@@ -1703,6 +1708,9 @@ func TestDuplicateDetection(t *testing.T) {
|
|||||||
require.Equal(t, 1, list.Total, "expected one duplicate cluster: %s", resp)
|
require.Equal(t, 1, list.Total, "expected one duplicate cluster: %s", resp)
|
||||||
require.Len(t, list.Items, 1)
|
require.Len(t, list.Items, 1)
|
||||||
require.Len(t, list.Items[0].Files, 2)
|
require.Len(t, list.Items[0].Files, 2)
|
||||||
|
// The pair's stored distance rides along; identical 1×1 uploads are distance 0.
|
||||||
|
require.Len(t, list.Items[0].Distances, 1, "the pair's distance should be reported")
|
||||||
|
assert.Equal(t, 0, list.Items[0].Distances[0].Distance)
|
||||||
|
|
||||||
// --- resolve: keep f1, union tags from f2, trash f2 ----------------------
|
// --- resolve: keep f1, union tags from f2, trash f2 ----------------------
|
||||||
resp = h.doJSON("POST", "/files/duplicates/resolve", map[string]any{
|
resp = h.doJSON("POST", "/files/duplicates/resolve", map[string]any{
|
||||||
|
|||||||
@@ -103,6 +103,31 @@ func buildPairs(entries []domain.PHashEntry, threshold int, onProgress func(done
|
|||||||
return pairs
|
return pairs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// orderedPair returns the two ids in canonical (a < b by UUID byte order) order,
|
||||||
|
// matching how the pairs table keys a distance so a lookup hits regardless of the
|
||||||
|
// argument order.
|
||||||
|
func orderedPair(a, b uuid.UUID) [2]uuid.UUID {
|
||||||
|
if bytes.Compare(a[:], b[:]) > 0 {
|
||||||
|
return [2]uuid.UUID{b, a}
|
||||||
|
}
|
||||||
|
return [2]uuid.UUID{a, b}
|
||||||
|
}
|
||||||
|
|
||||||
|
// clusterDistances returns the stored Hamming distance for every pair of files in
|
||||||
|
// the cluster that has one. Pairs present only transitively have no stored
|
||||||
|
// distance and are left out.
|
||||||
|
func clusterDistances(files []domain.File, distByPair map[[2]uuid.UUID]int) []PairDistance {
|
||||||
|
var out []PairDistance
|
||||||
|
for i := 0; i < len(files); i++ {
|
||||||
|
for j := i + 1; j < len(files); j++ {
|
||||||
|
if d, ok := distByPair[orderedPair(files[i].ID, files[j].ID)]; ok {
|
||||||
|
out = append(out, PairDistance{A: files[i].ID, B: files[j].ID, Distance: d})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// clusterPairs groups pairs into connected components (transitive closure) via
|
// clusterPairs groups pairs into connected components (transitive closure) via
|
||||||
// union-find. Every returned cluster has at least two files; clusters and the ids
|
// union-find. Every returned cluster has at least two files; clusters and the ids
|
||||||
// within them are sorted by UUID for stable pagination.
|
// within them are sorted by UUID for stable pagination.
|
||||||
|
|||||||
@@ -125,10 +125,27 @@ func NewDuplicateService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cluster is a group of near-duplicate files together with the pairwise Hamming
|
||||||
|
// distances known between them. Distances are read from the stored pairs, so two
|
||||||
|
// files linked into the cluster only transitively (through an intermediate) may
|
||||||
|
// have no direct distance — that pair is simply omitted.
|
||||||
|
type Cluster struct {
|
||||||
|
Files []domain.File
|
||||||
|
Distances []PairDistance
|
||||||
|
}
|
||||||
|
|
||||||
|
// PairDistance is the stored Hamming distance between two files of a cluster.
|
||||||
|
type PairDistance struct {
|
||||||
|
A uuid.UUID
|
||||||
|
B uuid.UUID
|
||||||
|
Distance int
|
||||||
|
}
|
||||||
|
|
||||||
// Clusters returns a page of duplicate clusters visible to the caller. Pairs are
|
// Clusters returns a page of duplicate clusters visible to the caller. Pairs are
|
||||||
// read from the precomputed table (no all-pairs scan here) and grouped into
|
// read from the precomputed table (no all-pairs scan here) and grouped into
|
||||||
// connected components; pagination is over whole clusters.
|
// connected components; pagination is over whole clusters. Each cluster carries
|
||||||
func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clusters [][]domain.File, total int, err error) {
|
// the stored pairwise distances so callers can show how close the files are.
|
||||||
|
func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clusters []Cluster, total int, err error) {
|
||||||
userID, isAdmin, _ := domain.UserFromContext(ctx)
|
userID, isAdmin, _ := domain.UserFromContext(ctx)
|
||||||
|
|
||||||
pairs, err := s.pairs.ListVisible(ctx, userID, isAdmin)
|
pairs, err := s.pairs.ListVisible(ctx, userID, isAdmin)
|
||||||
@@ -142,14 +159,20 @@ func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clu
|
|||||||
offset = 0
|
offset = 0
|
||||||
}
|
}
|
||||||
if offset >= len(groups) {
|
if offset >= len(groups) {
|
||||||
return [][]domain.File{}, total, nil
|
return []Cluster{}, total, nil
|
||||||
}
|
}
|
||||||
end := offset + limit
|
end := offset + limit
|
||||||
if end > len(groups) || limit <= 0 {
|
if end > len(groups) || limit <= 0 {
|
||||||
end = len(groups)
|
end = len(groups)
|
||||||
}
|
}
|
||||||
|
|
||||||
out := make([][]domain.File, 0, end-offset)
|
// Index the stored distances once; each page cluster looks up its own pairs.
|
||||||
|
distByPair := make(map[[2]uuid.UUID]int, len(pairs))
|
||||||
|
for _, p := range pairs {
|
||||||
|
distByPair[orderedPair(p.FileA, p.FileB)] = p.Distance
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]Cluster, 0, end-offset)
|
||||||
for _, ids := range groups[offset:end] {
|
for _, ids := range groups[offset:end] {
|
||||||
files := make([]domain.File, 0, len(ids))
|
files := make([]domain.File, 0, len(ids))
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
@@ -164,7 +187,7 @@ func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clu
|
|||||||
files = append(files, *f)
|
files = append(files, *f)
|
||||||
}
|
}
|
||||||
if len(files) >= 2 {
|
if len(files) >= 2 {
|
||||||
out = append(out, files)
|
out = append(out, Cluster{Files: files, Distances: clusterDistances(files, distByPair)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, total, nil
|
return out, total, nil
|
||||||
|
|||||||
@@ -1,9 +1,19 @@
|
|||||||
import { api } from '$lib/api/client';
|
import { api } from '$lib/api/client';
|
||||||
import type { File } from '$lib/api/types';
|
import type { File } from '$lib/api/types';
|
||||||
|
|
||||||
/** A group of mutually similar files. */
|
/** A stored perceptual-hash (Hamming) distance between two files of a cluster. */
|
||||||
|
export interface DuplicatePairDistance {
|
||||||
|
a: string;
|
||||||
|
b: string;
|
||||||
|
distance: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A group of mutually similar files, with the pairwise distances known between
|
||||||
|
* them. A file linked into the cluster only transitively may lack a direct
|
||||||
|
* distance to some others, so that pair is absent. */
|
||||||
export interface DuplicateCluster {
|
export interface DuplicateCluster {
|
||||||
files: File[];
|
files: File[];
|
||||||
|
distances?: DuplicatePairDistance[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DuplicateClusterPage {
|
export interface DuplicateClusterPage {
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { goto } from '$app/navigation';
|
import { goto } from '$app/navigation';
|
||||||
import { api } from '$lib/api/client';
|
import { api } from '$lib/api/client';
|
||||||
import { getDuplicates, dismissDuplicate } from '$lib/api/duplicates';
|
import {
|
||||||
|
getDuplicates,
|
||||||
|
dismissDuplicate,
|
||||||
|
type DuplicatePairDistance
|
||||||
|
} from '$lib/api/duplicates';
|
||||||
import Thumb from '$lib/components/file/Thumb.svelte';
|
import Thumb from '$lib/components/file/Thumb.svelte';
|
||||||
import DuplicateMergeDialog from '$lib/components/file/DuplicateMergeDialog.svelte';
|
import DuplicateMergeDialog from '$lib/components/file/DuplicateMergeDialog.svelte';
|
||||||
import FileViewer from '$lib/components/file/FileViewer.svelte';
|
import FileViewer from '$lib/components/file/FileViewer.svelte';
|
||||||
@@ -14,6 +18,7 @@
|
|||||||
interface Cluster {
|
interface Cluster {
|
||||||
key: number;
|
key: number;
|
||||||
files: File[];
|
files: File[];
|
||||||
|
distances: DuplicatePairDistance[];
|
||||||
}
|
}
|
||||||
let nextKey = 0;
|
let nextKey = 0;
|
||||||
|
|
||||||
@@ -48,13 +53,26 @@
|
|||||||
return keepers[c.key] ?? c.files[0]?.id ?? '';
|
return keepers[c.key] ?? c.files[0]?.id ?? '';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stored perceptual distance between the kept file and another, or null when
|
||||||
|
// the two are linked only transitively (no direct stored pair).
|
||||||
|
function distanceFromKeep(c: Cluster, keep: string, other: string): number | null {
|
||||||
|
for (const d of c.distances) {
|
||||||
|
if ((d.a === keep && d.b === other) || (d.a === other && d.b === keep)) return d.distance;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
async function load() {
|
async function load() {
|
||||||
if (loading) return;
|
if (loading) return;
|
||||||
loading = true;
|
loading = true;
|
||||||
error = '';
|
error = '';
|
||||||
try {
|
try {
|
||||||
const res = await getDuplicates(LIMIT, offset);
|
const res = await getDuplicates(LIMIT, offset);
|
||||||
const incoming = (res.items ?? []).map((c) => ({ key: nextKey++, files: c.files }));
|
const incoming = (res.items ?? []).map((c) => ({
|
||||||
|
key: nextKey++,
|
||||||
|
files: c.files,
|
||||||
|
distances: c.distances ?? []
|
||||||
|
}));
|
||||||
total = res.total ?? total;
|
total = res.total ?? total;
|
||||||
// The server paginates by group index and may drop groups that fell below
|
// The server paginates by group index and may drop groups that fell below
|
||||||
// two live files, so advance by the page size (clamped), not items returned.
|
// two live files, so advance by the page size (clamped), not items returned.
|
||||||
@@ -276,8 +294,18 @@
|
|||||||
|
|
||||||
<div class="actions">
|
<div class="actions">
|
||||||
{#each c.files.filter((f) => f.id !== keep) as other (other.id)}
|
{#each c.files.filter((f) => f.id !== keep) as other (other.id)}
|
||||||
|
{@const dist = distanceFromKeep(c, keep, other.id)}
|
||||||
<div class="actrow">
|
<div class="actrow">
|
||||||
<span class="aname" title={other.original_name ?? ''}>{other.original_name ?? '—'}</span>
|
<span class="aname" title={other.original_name ?? ''}>{other.original_name ?? '—'}</span>
|
||||||
|
<span
|
||||||
|
class="dist"
|
||||||
|
class:unknown={dist === null}
|
||||||
|
title={dist === null
|
||||||
|
? 'No direct match — linked through another file'
|
||||||
|
: 'Perceptual distance from the kept file (lower = more similar)'}
|
||||||
|
>
|
||||||
|
Δ{dist ?? '—'}
|
||||||
|
</span>
|
||||||
<button class="abtn" onclick={() => openMerge(c, other)}>Merge</button>
|
<button class="abtn" onclick={() => openMerge(c, other)}>Merge</button>
|
||||||
<button class="abtn" onclick={() => deleteFile(c, other.id)}>Delete</button>
|
<button class="abtn" onclick={() => deleteFile(c, other.id)}>Delete</button>
|
||||||
<button class="abtn ghost" onclick={() => notDuplicate(c, other)}>Not a dup</button>
|
<button class="abtn ghost" onclick={() => notDuplicate(c, other)}>Not a dup</button>
|
||||||
@@ -493,6 +521,20 @@
|
|||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
text-overflow: ellipsis;
|
text-overflow: ellipsis;
|
||||||
}
|
}
|
||||||
|
.dist {
|
||||||
|
flex-shrink: 0;
|
||||||
|
font-size: 0.72rem;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
color: var(--color-accent);
|
||||||
|
background-color: color-mix(in srgb, var(--color-accent) 14%, transparent);
|
||||||
|
border-radius: 5px;
|
||||||
|
padding: 1px 6px;
|
||||||
|
cursor: help;
|
||||||
|
}
|
||||||
|
.dist.unknown {
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
background-color: var(--color-bg-elevated);
|
||||||
|
}
|
||||||
.abtn {
|
.abtn {
|
||||||
padding: 5px 10px;
|
padding: 5px 10px;
|
||||||
border-radius: 7px;
|
border-radius: 7px;
|
||||||
|
|||||||
@@ -1956,6 +1956,28 @@ components:
|
|||||||
description: Two or more mutually similar files
|
description: Two or more mutually similar files
|
||||||
items:
|
items:
|
||||||
$ref: '#/components/schemas/File'
|
$ref: '#/components/schemas/File'
|
||||||
|
distances:
|
||||||
|
type: array
|
||||||
|
description: >-
|
||||||
|
Stored perceptual-hash (Hamming) distances between pairs of files in
|
||||||
|
the cluster. A pair linked only transitively (through an intermediate
|
||||||
|
file) has no stored distance and is omitted.
|
||||||
|
items:
|
||||||
|
$ref: '#/components/schemas/DuplicatePairDistance'
|
||||||
|
|
||||||
|
DuplicatePairDistance:
|
||||||
|
type: object
|
||||||
|
required: [a, b, distance]
|
||||||
|
properties:
|
||||||
|
a:
|
||||||
|
type: string
|
||||||
|
format: uuid
|
||||||
|
b:
|
||||||
|
type: string
|
||||||
|
format: uuid
|
||||||
|
distance:
|
||||||
|
type: integer
|
||||||
|
description: Hamming distance (0–64) between the two files' perceptual hashes
|
||||||
|
|
||||||
DuplicateClusterPage:
|
DuplicateClusterPage:
|
||||||
type: object
|
type: object
|
||||||
|
|||||||
Reference in New Issue
Block a user