git/intake/main
internal/execute/execute.go
483 lines15 KB last change raw · history
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
package execute

import (
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"intake/internal/artwork"
	"intake/internal/audio"
	"intake/internal/plan"
	"intake/internal/tags"
)

type Options struct {
	DryRun       bool
	KeepStaging  bool
	VerifyDecode string
	Collision    string
	CoverPath    string
	CoverName    string
	// StagingRoot is the staging directory the album came from. Staging cleanup
	// never removes it, however empty it becomes.
	StagingRoot string
	Logf        func(format string, args ...any)
}

type Result struct {
	ArchiveDir       string
	Files            []string
	RemovedStaging   bool
	Bytes            int64
	StagingLeftovers []string
	// StagingStuck names staged files that could not be removed. The album is
	// archived either way.
	StagingStuck []string
}

func Run(ctx context.Context, p plan.Plan, opts Options) (Result, error) {
	logf := opts.Logf
	if logf == nil {
		logf = func(string, ...any) {}
	}
	var res Result
	if p.ArchiveDir == "" {
		return res, fmt.Errorf("plan has no archive directory")
	}
	res.ArchiveDir = p.ArchiveDir

	if opts.DryRun {
		logf("dry run: would create %s", p.ArchiveDir)
		if opts.CoverPath != "" {
			logf("dry run: would write %s and embed it as the front cover", coverName(opts))
		}
		for _, t := range p.Tracks {
			logf("dry run: copy %s -> %s/%s", filepath.Base(t.Source), filepath.Base(p.ArchiveDir), t.Target)
		}
		return res, nil
	}

	parent := filepath.Dir(p.ArchiveDir)
	createdParent := false
	if _, err := os.Stat(parent); os.IsNotExist(err) {
		createdParent = true
	}
	if err := os.MkdirAll(parent, 0o755); err != nil {
		return res, err
	}
	promoted := false
	defer func() {
		// An artist folder is only created so the album can go inside it. When
		// the album never gets there -- a truncated download, a decode that
		// fails verification -- the empty folder is left behind, and an empty
		// artist directory in the archive is indistinguishable from one whose
		// albums were lost. os.Remove refuses to touch a non-empty directory, so
		// this can only ever undo the creation above.
		if createdParent && !promoted {
			os.Remove(parent)
		}
	}()

	// Last line of defence against writing one track over another. The plan
	// checks this too, but the plan is a value that any caller can assemble, and
	// the cost of being wrong here is a quietly missing track rather than a
	// refusal, so it is worth proving again immediately before the first copy.
	if err := checkTargets(p); err != nil {
		return res, err
	}

	tmp, err := os.MkdirTemp(parent, "."+filepath.Base(p.ArchiveDir)+".intake-")
	if err != nil {
		return res, err
	}
	defer os.RemoveAll(tmp)

	if opts.CoverPath != "" {
		if err := copyFile(opts.CoverPath, filepath.Join(tmp, coverName(opts))); err != nil {
			return res, fmt.Errorf("copying cover: %w", err)
		}
	}

	written := make([]string, 0, len(p.Tracks))
	for _, t := range p.Tracks {
		dst := filepath.Join(tmp, t.Target)
		if err := copyFile(t.Source, dst); err != nil {
			return res, fmt.Errorf("copy %s: %w", filepath.Base(t.Source), err)
		}
		lines, err := tags.Write(t.Tags)
		if err != nil {
			return res, err
		}
		if err := audio.WriteTags(ctx, dst, lines, opts.CoverPath != ""); err != nil {
			return res, err
		}
		if opts.CoverPath != "" {
			if err := artwork.Embed(ctx, dst, opts.CoverPath); err != nil {
				return res, err
			}
		}
		written = append(written, dst)
		// An album is minutes of copying and encoding with nothing to show for
		// it otherwise, which reads as a hang.
		logf("copied %s", t.Target)
	}

	if err := verify(ctx, p, written, opts.VerifyDecode, opts, logf); err != nil {
		return res, err
	}

	finalDir, err := promote(p.ArchiveDir, tmp, opts.Collision, logf)
	if err != nil {
		return res, err
	}
	promoted = true
	// "both" lands the album somewhere other than the path that was asked for,
	// and everything downstream -- the summary, the journal, the library this
	// same run builds -- has to follow the album to where it actually went.
	res.ArchiveDir = finalDir
	finalPaths := make([]string, 0, len(written))
	for _, f := range written {
		rel, err := filepath.Rel(tmp, f)
		if err != nil {
			rel = filepath.Base(f)
		}
		finalPaths = append(finalPaths, filepath.Join(finalDir, rel))
	}
	res.Files = finalPaths

	if !opts.KeepStaging {
		// The album is archived from here on. Failing to tidy up after it is a
		// nuisance, not a failure: returning an error would make the caller
		// treat a complete archive as broken, drop its journal record, and offer
		// to file it all over again.
		for _, t := range p.Tracks {
			if err := os.Remove(t.Source); err != nil && !os.IsNotExist(err) {
				res.StagingStuck = append(res.StagingStuck, filepath.Base(t.Source))
			}
		}
		// The album folder to clear is the one the scan called the album, not the
		// folder the first track happened to sit in: a CD1 / CD2 rip puts the
		// tracks one level below the album, and clearing only CD1 would leave CD2
		// and the album's own sidecars sitting in staging for ever.
		albumDir := p.Album.Dir
		if albumDir == "" {
			albumDir = filepath.Dir(p.Tracks[0].Source)
		}
		leftovers, err := cleanStaging(albumDir, opts.StagingRoot)
		if err != nil {
			logf("note: could not fully clear %s: %v", albumDir, err)
		}
		if len(leftovers) > 0 {
			logf("removed non-audio leftovers from staging: %s", strings.Join(leftovers, ", "))
		}
		res.StagingLeftovers = leftovers
		res.RemovedStaging = len(res.StagingStuck) == 0
		if len(res.StagingStuck) > 0 {
			logf("note: the album is archived, but %d staged file(s) could not be removed: %s",
				len(res.StagingStuck), strings.Join(res.StagingStuck, ", "))
		}
	}
	for _, t := range res.Files {
		if fi, err := os.Stat(t); err == nil {
			res.Bytes += fi.Size()
		}
	}
	return res, nil
}

// checkTargets refuses a plan that would write two tracks to the same name.
func checkTargets(p plan.Plan) error {
	if len(p.Tracks) == 0 {
		return fmt.Errorf("plan has no tracks")
	}
	seen := map[string]string{}
	for _, t := range p.Tracks {
		name := filepath.Base(t.Target)
		if name == "" || name == "." || name == string(filepath.Separator) {
			return fmt.Errorf("plan has a track with no target name (%s)", filepath.Base(t.Source))
		}
		if prev, ok := seen[name]; ok {
			return fmt.Errorf("refusing to file: %s and %s would both be written to %q, which would overwrite one of them",
				prev, filepath.Base(t.Source), name)
		}
		seen[name] = filepath.Base(t.Source)
	}
	return nil
}

func coverName(opts Options) string {
	if opts.CoverName != "" {
		return opts.CoverName
	}
	return "cover.jpg"
}

func verify(ctx context.Context, p plan.Plan, written []string, mode string, opts Options, logf func(string, ...any)) error {
	if len(written) != len(p.Tracks) {
		return fmt.Errorf("verification failed: wrote %d files for %d tracks", len(written), len(p.Tracks))
	}
	for i, f := range written {
		src := p.Tracks[i]
		info, err := audio.Probe(ctx, f)
		if err != nil {
			return fmt.Errorf("verification failed reading %s: %w", filepath.Base(f), err)
		}
		if info.Codec == "" {
			return fmt.Errorf("verification failed: %s has no decodable audio stream", filepath.Base(f))
		}
		if src.Ext == ".flac" && info.Codec != "flac" {
			return fmt.Errorf("verification failed: %s codec is %q, expected flac", filepath.Base(f), info.Codec)
		}
		if src.Duration > 0 && info.Duration > 0 {
			diff := info.Duration - src.Duration
			if diff < 0 {
				diff = -diff
			}
			if diff > 0.25 {
				return fmt.Errorf("verification failed: %s duration changed by %.3fs", filepath.Base(f), diff)
			}
		}
		if got, _ := tags.Get(info.Tags, tags.FieldTitle); strings.TrimSpace(got) == "" {
			return fmt.Errorf("verification failed: %s has no TITLE after tagging", filepath.Base(f))
		}
		if got, _ := tags.Get(info.Tags, tags.FieldAlbumArtist); strings.TrimSpace(got) == "" {
			return fmt.Errorf("verification failed: %s has no ALBUMARTIST after tagging", filepath.Base(f))
		}
		stray := tags.Dropped(info.Tags)
		if len(stray) > 0 {
			return fmt.Errorf("verification failed: %s still carries non-whitelisted tags %v", filepath.Base(f), stray)
		}
		if opts.CoverPath != "" {
			if info.ArtCount == 0 {
				return fmt.Errorf("verification failed: %s has no embedded cover", filepath.Base(f))
			}
		}
	}

	// Every flac is integrity-tested, not only the sampled ones: a partial
	// download can sit anywhere in the album, and the check costs about a fifth
	// of a second per track.
	for _, f := range written {
		if !strings.EqualFold(filepath.Ext(f), ".flac") {
			continue
		}
		if err := audio.TestFLAC(ctx, f); err != nil {
			return fmt.Errorf("verification failed: %w", err)
		}
	}

	switch mode {
	case "all":
		for _, f := range written {
			if err := audio.DecodeOK(ctx, f); err != nil {
				return fmt.Errorf("verification failed: %w", err)
			}
		}
	case "sample":
		idx := []int{0, len(written) / 2, len(written) - 1}
		for _, i := range idx {
			if err := audio.DecodeOK(ctx, written[i]); err != nil {
				return fmt.Errorf("verification failed: %w", err)
			}
		}
	}
	logf("verified %d file(s) (decode: %s)", len(written), mode)
	return nil
}

// promote moves a fully built and verified album directory into place and
// returns the directory it actually landed in.
//
// The return value matters: "both" deliberately puts the album somewhere other
// than the path the plan named, and every later step has to agree about where
// that is.
func promote(target, tmp, collision string, logf func(string, ...any)) (string, error) {
	if _, err := os.Stat(target); err == nil {
		switch collision {
		case "replace":
			// Move the old album aside and only delete it once the new one is in
			// place. Removing it first leaves a window -- a failed rename, a
			// permission problem partway through RemoveAll, a power cut -- in
			// which the album is simply gone, with nothing to restore.
			stale := target + ".intake-replaced"
			if err := os.Rename(target, stale); err != nil {
				return "", fmt.Errorf("moving the existing album aside: %w", err)
			}
			if err := os.Rename(tmp, target); err != nil {
				// Put the old album back rather than leaving neither.
				if back := os.Rename(stale, target); back != nil {
					return "", fmt.Errorf("promote %s: %w (and the previous album is now at %s)", target, err, stale)
				}
				return "", fmt.Errorf("promote %s: %w", target, err)
			}
			if err := os.RemoveAll(stale); err != nil {
				logf("note: the replaced album is still at %s: %v", filepath.Base(stale), err)
			}
			logf("placed %s (replacing what was there)", target)
			return target, nil
		case "both":
			alt := target + " (2)"
			for i := 3; ; i++ {
				_, err := os.Stat(alt)
				if os.IsNotExist(err) {
					break
				}
				if err != nil {
					// Anything other than "it is not there" -- an unreadable
					// parent directory, say -- would otherwise spin here for
					// ever picking names that can never be checked.
					return "", fmt.Errorf("looking for a free name beside %s: %w", target, err)
				}
				if i > 1000 {
					return "", fmt.Errorf("could not find a free name beside %s", target)
				}
				alt = fmt.Sprintf("%s (%d)", target, i)
			}
			logf("destination exists; keeping both as %s", filepath.Base(alt))
			target = alt
		case "merge":
			if err := mergeInto(tmp, target); err != nil {
				return "", err
			}
			logf("merged into existing %s", filepath.Base(target))
			return target, nil
		default:
			return "", fmt.Errorf("destination already exists: %s", target)
		}
	}
	if err := os.Rename(tmp, target); err != nil {
		return "", fmt.Errorf("promote %s: %w", target, err)
	}
	logf("placed %s", target)
	return target, nil
}

// mergeInto moves the contents of src into dst, refusing to overwrite anything.
//
// Every possible conflict is found before the first file moves. Checking as it
// goes left the destination holding half of one album and half of another when
// the second file clashed, which is worse than either album on its own and
// cannot be undone by re-running.
func mergeInto(src, dst string) error {
	entries, err := os.ReadDir(src)
	if err != nil {
		return err
	}
	for _, e := range entries {
		to := filepath.Join(dst, e.Name())
		if _, err := os.Stat(to); err == nil {
			return fmt.Errorf("merge conflict: %s already exists, so nothing was merged", to)
		}
	}
	for _, e := range entries {
		from := filepath.Join(src, e.Name())
		to := filepath.Join(dst, e.Name())
		if err := os.Rename(from, to); err != nil {
			return err
		}
	}
	return nil
}

func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
	if err != nil {
		return err
	}
	defer out.Close()
	if _, err := io.Copy(out, in); err != nil {
		return err
	}
	return out.Sync()
}

// cleanStaging clears a filed album out of staging: its leftover files, its
// now-empty subdirectories (a CD1 / CD2 rip leaves one empty disc folder behind
// for every disc but the last), and then the album folder itself.
//
// It climbs towards stagingRoot, never past or through it. An empty staging
// directory is the normal state after the last album is filed, not clutter to be
// tidied away, and deleting it makes the next scan fail and the next run
// recreate it. With no stagingRoot it stops at the album folder.
func cleanStaging(dir, stagingRoot string) ([]string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	var removed []string
	for _, e := range entries {
		if e.IsDir() || audio.IsAudio(e.Name()) {
			continue
		}
		path := filepath.Join(dir, e.Name())
		if err := os.Remove(path); err != nil {
			return removed, err
		}
		removed = append(removed, e.Name())
	}
	for cur := dir; withinStaging(cur, stagingRoot); cur = filepath.Dir(cur) {
		pruneEmptySubdirs(cur)
		left, err := os.ReadDir(cur)
		if err != nil || len(left) > 0 {
			break
		}
		if err := os.Remove(cur); err != nil {
			break
		}
	}
	return removed, nil
}

// withinStaging reports whether dir is inside stagingRoot rather than equal to
// it or above it. An empty stagingRoot means "do not climb at all".
func withinStaging(dir, stagingRoot string) bool {
	if stagingRoot == "" || dir == "" || dir == "/" || dir == "." {
		return false
	}
	root, err := filepath.Abs(stagingRoot)
	if err != nil {
		return false
	}
	abs, err := filepath.Abs(dir)
	if err != nil {
		return false
	}
	rel, err := filepath.Rel(root, abs)
	if err != nil || rel == "." || rel == ".." {
		return false
	}
	return !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}

// pruneEmptySubdirs removes directories under dir that hold nothing, deepest
// first, so that a folder like CD2 or Scans does not keep an otherwise finished
// album sitting in staging forever. A directory holding anything at all is left
// exactly as it is.
func pruneEmptySubdirs(dir string) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return
	}
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		sub := filepath.Join(dir, e.Name())
		pruneEmptySubdirs(sub)
		if left, err := os.ReadDir(sub); err == nil && len(left) == 0 {
			os.Remove(sub)
		}
	}
}