feat(backend): import server-folder files oldest-first by mtime
os.ReadDir returns entries in name order; sort them by ascending mtime before importing so each Upload's created_at reflects the files' chronological order. mtimes are cached once for the sort and reused as the content_datetime fallback, dropping the redundant per-file Info() call. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1295,6 +1295,50 @@ func TestImportFromFolder(t *testing.T) {
|
|||||||
assert.True(t, ct.Equal(mtime), "content_datetime %v should equal mtime %v", ct, mtime)
|
assert.True(t, ct.Equal(mtime), "content_datetime %v should equal mtime %v", ct, mtime)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestImportOrdersByMtime verifies that a folder import processes files in
|
||||||
|
// ascending mtime order, regardless of filename order. The three files are named
|
||||||
|
// so that alphabetical order (ReadDir's default) is the reverse of mtime order;
|
||||||
|
// the progress-event indices must follow mtime, oldest first.
|
||||||
|
func TestImportOrdersByMtime(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping integration test in short mode")
|
||||||
|
}
|
||||||
|
h := setupSuite(t)
|
||||||
|
adminToken := h.login("admin", "admin")
|
||||||
|
|
||||||
|
// name → mtime; alphabetical order (a,b,c) is the reverse of chronological.
|
||||||
|
files := []struct {
|
||||||
|
name string
|
||||||
|
mtime time.Time
|
||||||
|
}{
|
||||||
|
{"a_newest.jpg", time.Date(2022, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||||
|
{"b_middle.jpg", time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||||
|
{"c_oldest.jpg", time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||||
|
}
|
||||||
|
for _, f := range files {
|
||||||
|
p := filepath.Join(h.importDir, f.name)
|
||||||
|
require.NoError(t, os.WriteFile(p, minimalJPEG(), 0o644))
|
||||||
|
require.NoError(t, os.Chtimes(p, f.mtime, f.mtime))
|
||||||
|
}
|
||||||
|
|
||||||
|
resp := h.doJSON("POST", "/files/import", map[string]any{}, adminToken)
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode, resp.String())
|
||||||
|
|
||||||
|
events := parseImportEvents(t, resp)
|
||||||
|
idx := map[string]int{}
|
||||||
|
for _, ev := range events {
|
||||||
|
if ev.Type == "file" {
|
||||||
|
require.Equal(t, "imported", ev.Status, "%s: %s", ev.Filename, ev.Reason)
|
||||||
|
idx[ev.Filename] = ev.Index
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.Len(t, idx, 3, resp.String())
|
||||||
|
|
||||||
|
// Oldest first: c (2019) → b (2021) → a (2022).
|
||||||
|
assert.Less(t, idx["c_oldest.jpg"], idx["b_middle.jpg"], "oldest should be processed before middle")
|
||||||
|
assert.Less(t, idx["b_middle.jpg"], idx["a_newest.jpg"], "middle should be processed before newest")
|
||||||
|
}
|
||||||
|
|
||||||
// TestContentRangeRequests verifies the original-content endpoint answers a
|
// TestContentRangeRequests verifies the original-content endpoint answers a
|
||||||
// byte-range request with 206 Partial Content (so the browser can seek within
|
// byte-range request with 206 Partial Content (so the browser can seek within
|
||||||
// audio/video) rather than streaming the whole body.
|
// audio/video) rather than streaming the whole body.
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -589,6 +590,23 @@ func (s *FileService) Import(ctx context.Context, path string, onProgress func(I
|
|||||||
return nil, fmt.Errorf("FileService.Import: read dir %q: %w", dir, err)
|
return nil, fmt.Errorf("FileService.Import: read dir %q: %w", dir, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Import oldest-first: process entries in ascending mtime order so the
|
||||||
|
// resulting records' creation order matches the files' chronological order.
|
||||||
|
// mtimes are cached once (re-stat'ing per comparison would be wasteful) and
|
||||||
|
// reused below as the content_datetime fallback. Entries whose info can't be
|
||||||
|
// read get a zero time (sort first); they surface their error in the loop.
|
||||||
|
modTimes := make(map[string]time.Time, len(entries))
|
||||||
|
for _, e := range entries {
|
||||||
|
if info, infoErr := e.Info(); infoErr == nil {
|
||||||
|
modTimes[e.Name()] = info.ModTime()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// SliceStable preserves ReadDir's name order as a deterministic tiebreak for
|
||||||
|
// entries sharing an mtime.
|
||||||
|
sort.SliceStable(entries, func(a, b int) bool {
|
||||||
|
return modTimes[entries[a].Name()].Before(modTimes[entries[b].Name()])
|
||||||
|
})
|
||||||
|
|
||||||
emit := func(ev ImportEvent) {
|
emit := func(ev ImportEvent) {
|
||||||
if onProgress != nil {
|
if onProgress != nil {
|
||||||
onProgress(ev)
|
onProgress(ev)
|
||||||
@@ -646,10 +664,9 @@ func (s *FileService) Import(ctx context.Context, path string, onProgress func(I
|
|||||||
|
|
||||||
// Preserve the file's mtime as a content_datetime fallback (used only when
|
// Preserve the file's mtime as a content_datetime fallback (used only when
|
||||||
// the file has no EXIF date) — once the source is removed below it's the
|
// the file has no EXIF date) — once the source is removed below it's the
|
||||||
// only date left for non-photo files.
|
// only date left for non-photo files. Reuses the mtime cached for sorting.
|
||||||
var mtime *time.Time
|
var mtime *time.Time
|
||||||
if info, statErr := entry.Info(); statErr == nil {
|
if t, ok := modTimes[name]; ok {
|
||||||
t := info.ModTime()
|
|
||||||
mtime = &t
|
mtime = &t
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user