From 78b5e86dd69e27ca99434f210c19b596d7297150 Mon Sep 17 00:00:00 2001 From: Masahiko AMANO Date: Mon, 22 Jun 2026 22:42:14 +0300 Subject: [PATCH] feat(backend): report stored pairwise distances in the duplicates response ListVisible already loads each pair's Hamming distance, but clusterPairs threw it away. Thread it through: Clusters now returns a Cluster carrying the stored pairwise distances (indexed once per page), and the list endpoint emits them as {a, b, distance}. Pairs linked only transitively have no stored distance and are omitted. Lets the UI show how close each file is to the kept one without a client-side hash compare (phash exceeds JS's safe-integer range). --- backend/internal/handler/duplicate_handler.go | 12 ++++--- backend/internal/integration/server_test.go | 8 +++++ backend/internal/service/duplicate_index.go | 25 ++++++++++++++ backend/internal/service/duplicate_service.go | 33 ++++++++++++++++--- openapi.yaml | 22 +++++++++++++ 5 files changed, 91 insertions(+), 9 deletions(-) diff --git a/backend/internal/handler/duplicate_handler.go b/backend/internal/handler/duplicate_handler.go index 7018e59..05afd4e 100644 --- a/backend/internal/handler/duplicate_handler.go +++ b/backend/internal/handler/duplicate_handler.go @@ -47,12 +47,16 @@ func (h *DuplicateHandler) List(c *gin.Context) { } items := make([]gin.H, len(clusters)) - for i, files := range clusters { - fs := make([]fileJSON, len(files)) - for j, f := range files { + for i, cl := range clusters { + fs := make([]fileJSON, len(cl.Files)) + for j, f := range cl.Files { fs[j] = toFileJSON(f) } - items[i] = gin.H{"files": fs} + dists := make([]gin.H, len(cl.Distances)) + for j, d := range cl.Distances { + dists[j] = gin.H{"a": d.A, "b": d.B, "distance": d.Distance} + } + items[i] = gin.H{"files": fs, "distances": dists} } respondJSON(c, http.StatusOK, gin.H{ "items": items, diff --git a/backend/internal/integration/server_test.go b/backend/internal/integration/server_test.go index ae65a0c..0b102af 100644 --- a/backend/internal/integration/server_test.go +++ b/backend/internal/integration/server_test.go @@ -1661,6 +1661,11 @@ type dupListResponse struct { ID string `json:"id"` } `json:"tags"` } `json:"files"` + Distances []struct { + A string `json:"a"` + B string `json:"b"` + Distance int `json:"distance"` + } `json:"distances"` } `json:"items"` Total int `json:"total"` } @@ -1703,6 +1708,9 @@ func TestDuplicateDetection(t *testing.T) { require.Equal(t, 1, list.Total, "expected one duplicate cluster: %s", resp) require.Len(t, list.Items, 1) require.Len(t, list.Items[0].Files, 2) + // The pair's stored distance rides along; identical 1×1 uploads are distance 0. + require.Len(t, list.Items[0].Distances, 1, "the pair's distance should be reported") + assert.Equal(t, 0, list.Items[0].Distances[0].Distance) // --- resolve: keep f1, union tags from f2, trash f2 ---------------------- resp = h.doJSON("POST", "/files/duplicates/resolve", map[string]any{ diff --git a/backend/internal/service/duplicate_index.go b/backend/internal/service/duplicate_index.go index 475e088..b752bbd 100644 --- a/backend/internal/service/duplicate_index.go +++ b/backend/internal/service/duplicate_index.go @@ -103,6 +103,31 @@ func buildPairs(entries []domain.PHashEntry, threshold int, onProgress func(done return pairs } +// orderedPair returns the two ids in canonical (a < b by UUID byte order) order, +// matching how the pairs table keys a distance so a lookup hits regardless of the +// argument order. +func orderedPair(a, b uuid.UUID) [2]uuid.UUID { + if bytes.Compare(a[:], b[:]) > 0 { + return [2]uuid.UUID{b, a} + } + return [2]uuid.UUID{a, b} +} + +// clusterDistances returns the stored Hamming distance for every pair of files in +// the cluster that has one. Pairs present only transitively have no stored +// distance and are left out. +func clusterDistances(files []domain.File, distByPair map[[2]uuid.UUID]int) []PairDistance { + var out []PairDistance + for i := 0; i < len(files); i++ { + for j := i + 1; j < len(files); j++ { + if d, ok := distByPair[orderedPair(files[i].ID, files[j].ID)]; ok { + out = append(out, PairDistance{A: files[i].ID, B: files[j].ID, Distance: d}) + } + } + } + return out +} + // clusterPairs groups pairs into connected components (transitive closure) via // union-find. Every returned cluster has at least two files; clusters and the ids // within them are sorted by UUID for stable pagination. diff --git a/backend/internal/service/duplicate_service.go b/backend/internal/service/duplicate_service.go index b076ef2..9b5010f 100644 --- a/backend/internal/service/duplicate_service.go +++ b/backend/internal/service/duplicate_service.go @@ -125,10 +125,27 @@ func NewDuplicateService( } } +// Cluster is a group of near-duplicate files together with the pairwise Hamming +// distances known between them. Distances are read from the stored pairs, so two +// files linked into the cluster only transitively (through an intermediate) may +// have no direct distance — that pair is simply omitted. +type Cluster struct { + Files []domain.File + Distances []PairDistance +} + +// PairDistance is the stored Hamming distance between two files of a cluster. +type PairDistance struct { + A uuid.UUID + B uuid.UUID + Distance int +} + // Clusters returns a page of duplicate clusters visible to the caller. Pairs are // read from the precomputed table (no all-pairs scan here) and grouped into -// connected components; pagination is over whole clusters. -func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clusters [][]domain.File, total int, err error) { +// connected components; pagination is over whole clusters. Each cluster carries +// the stored pairwise distances so callers can show how close the files are. +func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clusters []Cluster, total int, err error) { userID, isAdmin, _ := domain.UserFromContext(ctx) pairs, err := s.pairs.ListVisible(ctx, userID, isAdmin) @@ -142,14 +159,20 @@ func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clu offset = 0 } if offset >= len(groups) { - return [][]domain.File{}, total, nil + return []Cluster{}, total, nil } end := offset + limit if end > len(groups) || limit <= 0 { end = len(groups) } - out := make([][]domain.File, 0, end-offset) + // Index the stored distances once; each page cluster looks up its own pairs. + distByPair := make(map[[2]uuid.UUID]int, len(pairs)) + for _, p := range pairs { + distByPair[orderedPair(p.FileA, p.FileB)] = p.Distance + } + + out := make([]Cluster, 0, end-offset) for _, ids := range groups[offset:end] { files := make([]domain.File, 0, len(ids)) for _, id := range ids { @@ -164,7 +187,7 @@ func (s *DuplicateService) Clusters(ctx context.Context, limit, offset int) (clu files = append(files, *f) } if len(files) >= 2 { - out = append(out, files) + out = append(out, Cluster{Files: files, Distances: clusterDistances(files, distByPair)}) } } return out, total, nil diff --git a/openapi.yaml b/openapi.yaml index e518875..a087c4a 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1956,6 +1956,28 @@ components: description: Two or more mutually similar files items: $ref: '#/components/schemas/File' + distances: + type: array + description: >- + Stored perceptual-hash (Hamming) distances between pairs of files in + the cluster. A pair linked only transitively (through an intermediate + file) has no stored distance and is omitted. + items: + $ref: '#/components/schemas/DuplicatePairDistance' + + DuplicatePairDistance: + type: object + required: [a, b, distance] + properties: + a: + type: string + format: uuid + b: + type: string + format: uuid + distance: + type: integer + description: Hamming distance (0–64) between the two files' perceptual hashes DuplicateClusterPage: type: object