feat(backend): report stored pairwise distances in the duplicates response
ListVisible already loads each pair's Hamming distance, but clusterPairs threw
it away. Thread it through: Clusters now returns a Cluster carrying the stored
pairwise distances (indexed once per page), and the list endpoint emits them as
{a, b, distance}. Pairs linked only transitively have no stored distance and are
omitted. Lets the UI show how close each file is to the kept one without a
client-side hash compare (phash exceeds JS's safe-integer range).
This commit is contained in:
@@ -47,12 +47,16 @@ func (h *DuplicateHandler) List(c *gin.Context) {
|
||||
}
|
||||
|
||||
items := make([]gin.H, len(clusters))
|
||||
for i, files := range clusters {
|
||||
fs := make([]fileJSON, len(files))
|
||||
for j, f := range files {
|
||||
for i, cl := range clusters {
|
||||
fs := make([]fileJSON, len(cl.Files))
|
||||
for j, f := range cl.Files {
|
||||
fs[j] = toFileJSON(f)
|
||||
}
|
||||
items[i] = gin.H{"files": fs}
|
||||
dists := make([]gin.H, len(cl.Distances))
|
||||
for j, d := range cl.Distances {
|
||||
dists[j] = gin.H{"a": d.A, "b": d.B, "distance": d.Distance}
|
||||
}
|
||||
items[i] = gin.H{"files": fs, "distances": dists}
|
||||
}
|
||||
respondJSON(c, http.StatusOK, gin.H{
|
||||
"items": items,
|
||||
|
||||
@@ -1661,6 +1661,11 @@ type dupListResponse struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"tags"`
|
||||
} `json:"files"`
|
||||
Distances []struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
Distance int `json:"distance"`
|
||||
} `json:"distances"`
|
||||
} `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
@@ -1703,6 +1708,9 @@ func TestDuplicateDetection(t *testing.T) {
|
||||
require.Equal(t, 1, list.Total, "expected one duplicate cluster: %s", resp)
|
||||
require.Len(t, list.Items, 1)
|
||||
require.Len(t, list.Items[0].Files, 2)
|
||||
// The pair's stored distance rides along; identical 1×1 uploads are distance 0.
|
||||
require.Len(t, list.Items[0].Distances, 1, "the pair's distance should be reported")
|
||||
assert.Equal(t, 0, list.Items[0].Distances[0].Distance)
|
||||
|
||||
// --- resolve: keep f1, union tags from f2, trash f2 ----------------------
|
||||
resp = h.doJSON("POST", "/files/duplicates/resolve", map[string]any{
|
||||
|
||||
@@ -103,6 +103,31 @@ func buildPairs(entries []domain.PHashEntry, threshold int, onProgress func(done
|
||||
return pairs
|
||||
}
|
||||
|
||||
// orderedPair returns the two ids in canonical (a < b by UUID byte order) order,
|
||||
// matching how the pairs table keys a distance so a lookup hits regardless of the
|
||||
// argument order.
|
||||
func orderedPair(a, b uuid.UUID) [2]uuid.UUID {
|
||||
if bytes.Compare(a[:], b[:]) > 0 {
|
||||
return [2]uuid.UUID{b, a}
|
||||
}
|
||||
return [2]uuid.UUID{a, b}
|
||||
}
|
||||
|
||||
// clusterDistances returns the stored Hamming distance for every pair of files in
|
||||
// the cluster that has one. Pairs present only transitively have no stored
|
||||
// distance and are left out.
|
||||
func clusterDistances(files []domain.File, distByPair map[[2]uuid.UUID]int) []PairDistance {
|
||||
var out []PairDistance
|
||||
for i := 0; i < len(files); i++ {
|
||||
for j := i + 1; j < len(files); j++ {
|
||||
if d, ok := distByPair[orderedPair(files[i].ID, files[j].ID)]; ok {
|
||||
out = append(out, PairDistance{A: files[i].ID, B: files[j].ID, Distance: d})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clusterPairs groups pairs into connected components (transitive closure) via
|
||||
// union-find. Every returned cluster has at least two files; clusters and the ids
|
||||
// within them are sorted by UUID for stable pagination.
|
||||
|
||||
@@ -125,10 +125,27 @@ func NewDuplicateService(
|
||||
}
|
||||
}
|
||||
|
||||
// Cluster is a group of near-duplicate files together with the pairwise Hamming
|
||||
// distances known between them. Distances are read from the stored pairs, so two
|
||||
// files linked into the cluster only transitively (through an intermediate) may
|
||||
// have no direct distance — that pair is simply omitted.
|
||||
type Cluster struct {
|
||||
Files []domain.File
|
||||
Distances []PairDistance
|
||||
}
|
||||
|
||||
// PairDistance is the stored Hamming distance between two files of a cluster.
|
||||
type PairDistance struct {
|
||||
A uuid.UUID
|
||||
B uuid.UUID
|
||||
Distance int
|
||||
}
|
||||
|
||||
// Clusters returns a page of duplicate clusters visible to the caller. Pairs are
|
||||
// read from the precomputed table (no all-pairs scan here) and grouped into
|
||||
// connected components; pagination is over whole clusters.
|
||||
func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clusters [][]domain.File, total int, err error) {
|
||||
// connected components; pagination is over whole clusters. Each cluster carries
|
||||
// the stored pairwise distances so callers can show how close the files are.
|
||||
func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clusters []Cluster, total int, err error) {
|
||||
userID, isAdmin, _ := domain.UserFromContext(ctx)
|
||||
|
||||
pairs, err := s.pairs.ListVisible(ctx, userID, isAdmin)
|
||||
@@ -142,14 +159,20 @@ func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clu
|
||||
offset = 0
|
||||
}
|
||||
if offset >= len(groups) {
|
||||
return [][]domain.File{}, total, nil
|
||||
return []Cluster{}, total, nil
|
||||
}
|
||||
end := offset + limit
|
||||
if end > len(groups) || limit <= 0 {
|
||||
end = len(groups)
|
||||
}
|
||||
|
||||
out := make([][]domain.File, 0, end-offset)
|
||||
// Index the stored distances once; each page cluster looks up its own pairs.
|
||||
distByPair := make(map[[2]uuid.UUID]int, len(pairs))
|
||||
for _, p := range pairs {
|
||||
distByPair[orderedPair(p.FileA, p.FileB)] = p.Distance
|
||||
}
|
||||
|
||||
out := make([]Cluster, 0, end-offset)
|
||||
for _, ids := range groups[offset:end] {
|
||||
files := make([]domain.File, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
@@ -164,7 +187,7 @@ func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clu
|
||||
files = append(files, *f)
|
||||
}
|
||||
if len(files) >= 2 {
|
||||
out = append(out, files)
|
||||
out = append(out, Cluster{Files: files, Distances: clusterDistances(files, distByPair)})
|
||||
}
|
||||
}
|
||||
return out, total, nil
|
||||
|
||||
Reference in New Issue
Block a user