git/orchard/main
updater/procwatch.py
421 lines13 KB last change raw · history
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
"""Observe a running build without parsing its output.

Arch has no machine-readable build event stream, so anything read out of paru's
or makepkg's human output is brittle by construction: it breaks on version,
locale, terminal width and colour settings.

We spawn paru ourselves, so we can observe the kernel instead:

    /proc/<pid>/cwd      -> which package's clone dir is being built (exact)
    /proc/<pid>/stat     -> starttime, hence an exact per-package elapsed timer
    /proc/<pid>/cmdline  -> which build system is actually running
    stat() on artefacts  -> corroborates completion

No cooperation from paru or makepkg is required, and nothing is scraped.
"""

from __future__ import annotations

import os
import time
from collections import deque
from dataclasses import dataclass
from pathlib import Path

PROC = Path("/proc")


def _clk_tck() -> int:
    try:
        return os.sysconf("SC_CLK_TCK")
    except (ValueError, OSError):  # pragma: no cover
        return 100


def boot_time() -> float:
    """Seconds since epoch at boot, from /proc/stat `btime`."""
    with open("/proc/stat", encoding="ascii", errors="replace") as fh:
        for line in fh:
            if line.startswith("btime "):
                return float(line.split()[1])
    return 0.0


@dataclass(frozen=True)
class Proc:
    pid: int
    ppid: int
    comm: str
    cmdline: tuple[str, ...]
    cwd: str | None
    started: float  # epoch seconds

    @property
    def elapsed(self) -> float:
        return max(0.0, time.time() - self.started)

    @property
    def argv0(self) -> str:
        return os.path.basename(self.cmdline[0]) if self.cmdline else self.comm


def read_proc(pid: int, btime: float, clk: int) -> Proc | None:
    """Read one process. Returns None the moment it dies, which is normal."""
    try:
        raw = (PROC / str(pid) / "stat").read_bytes()
    except OSError:
        return None
    # comm can contain spaces and parentheses, so parse from the last ')'
    end = raw.rfind(b")")
    if end < 0:
        return None
    comm = raw[raw.find(b"(") + 1 : end].decode(errors="replace")
    fields = raw[end + 2 :].split()
    if len(fields) < 20:
        return None
    try:
        ppid = int(fields[1])
        starttime = int(fields[19])
    except ValueError:
        return None

    try:
        cmdline = tuple(
            part.decode(errors="replace")
            for part in (PROC / str(pid) / "cmdline").read_bytes().split(b"\0")
            if part
        )
    except OSError:
        cmdline = ()

    try:
        cwd = os.readlink(PROC / str(pid) / "cwd")
    except OSError:
        cwd = None

    return Proc(
        pid=pid, ppid=ppid, comm=comm, cmdline=cmdline, cwd=cwd, started=btime + starttime / clk
    )


def snapshot() -> dict[int, Proc]:
    """Every readable process, keyed by pid."""
    btime, clk = boot_time(), _clk_tck()
    out: dict[int, Proc] = {}
    for entry in os.listdir(PROC):
        if not entry.isdigit():
            continue
        proc = read_proc(int(entry), btime, clk)
        if proc is not None:
            out[proc.pid] = proc
    return out


def descendants(root_pid: int, procs: dict[int, Proc], *, include_root: bool = False) -> list[Proc]:
    """Every process under `root_pid`, breadth-first.

    The root is excluded by default because the root is the launcher we spawned
    (paru), which does not itself build and whose working directory is wherever
    the user happened to run us. Pass `include_root=True` when the root *is* the
    builder, as it would be if we ever drive makepkg directly for parallel
    builds.
    """
    children: dict[int, list[Proc]] = {}
    for proc in procs.values():
        children.setdefault(proc.ppid, []).append(proc)

    out: list[Proc] = []
    if include_root and root_pid in procs:
        out.append(procs[root_pid])
    queue = deque(children.get(root_pid, []))
    while queue:
        proc = queue.popleft()
        out.append(proc)
        queue.extend(children.get(proc.pid, []))
    return out


# Build systems, most specific first. The first match on a process's argv0 wins.
BUILD_TOOLS: tuple[tuple[str, str], ...] = (
    ("cc1plus", "compile"),
    ("cc1", "compile"),
    ("clang", "compile"),
    ("gcc", "compile"),
    ("g++", "compile"),
    ("rustc", "compile"),
    ("cargo", "cargo"),
    ("go", "go build"),
    ("nim", "nim"),
    ("meson", "meson"),
    ("ninja", "ninja"),
    ("cmake", "cmake"),
    ("make", "make"),
    ("scons", "scons"),
    ("waf", "waf"),
    ("npm", "node"),
    ("pnpm", "node"),
    ("yarn", "node"),
    ("node", "node"),
    ("setup.py", "python"),
    ("pip", "python"),
    ("python", "python"),
    ("patch", "prepare"),
    ("git", "fetch"),
    ("curl", "fetch"),
    ("wget", "fetch"),
    ("fakeroot", "package"),
    ("makepkg", "makepkg"),
)

#: processes that only exist to drive the build; not worth reporting as a phase
DRIVERS = {"makepkg", "fakeroot", "sudo", "paru", "systemd"}


#: higher-level build systems first. When several are running at once the label
#: should describe the phase, and "cargo" is more useful than "compile".
PHASE_PRIORITY: tuple[str, ...] = (
    "cargo",
    "go build",
    "meson",
    "ninja",
    "cmake",
    "node",
    "python",
    "nim",
    "scons",
    "waf",
    "make",
    "compile",
    "package",
    "prepare",
    "fetch",
    "makepkg",
)


def _tool_names(proc: Proc) -> set[str]:
    """Candidate tool names for a process.

    argv0 always counts. Later arguments count only when they look like a path,
    so a shell wrapper invoked as `/bin/bash /usr/bin/cargo` is recognised while
    `cat make` is not mistaken for a make run.
    """
    names: set[str] = set()
    for i, arg in enumerate(proc.cmdline):
        if i == 0 or "/" in arg:
            names.add(os.path.basename(arg))
    return names or {proc.comm}


def _labels(proc: Proc) -> list[str]:
    names = _tool_names(proc)
    return [label for tool, label in BUILD_TOOLS if tool in names]


#: helper processes that are part of building but are not the work itself
_NOISE = frozenset(
    {
        "sh",
        "bash",
        "dash",
        "make",
        "ninja",
        "meson",
        "cmake",
        "python",
        "python3",
        "perl",
        "sed",
        "awk",
        "cat",
        "tee",
        "env",
        "time",
        "xargs",
    }
)


def summarise_workers(procs: list[Proc], *, limit: int = 4) -> str:
    """What the running jobs are, counted by program.

    "5 jobs" says something is happening; "rustc ×4, cc1 ×1" says what, which
    is the only thing that distinguishes a compiling build from a hung one.
    """
    counts: dict[str, int] = {}
    for proc in procs:
        name = proc.argv0 or proc.comm
        if not name or name in _NOISE:
            continue
        counts[name] = counts.get(name, 0) + 1
    if not counts:
        return ""
    ordered = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
    shown = [f"{name} ×{n}" if n > 1 else name for name, n in ordered[:limit]]
    if len(ordered) > limit:
        shown.append(f"+{len(ordered) - limit} more")
    return ", ".join(shown)


def classify(procs: list[Proc]) -> str:
    """Best available label for what a package's process tree is doing.

    Deterministic regardless of process ordering: the highest-priority label
    present wins, rather than whichever process happened to be listed first.
    """
    found = {label for proc in procs for label in _labels(proc)}
    for label in PHASE_PRIORITY:
        if label in found:
            return label
    return ""


def package_of(path: str | None, clone_dir: Path, known: set[str] | None = None) -> str | None:
    """Map a cwd back to the AUR package whose clone directory contains it.

    makepkg runs with cwd == <clone_dir>/<pkg>; compilers run deeper, in
    <clone_dir>/<pkg>/src/... . Either way the first path component is the answer.
    """
    if not path:
        return None
    # /proc/<pid>/cwd is a fully resolved path, so a symlinked or relative
    # clone directory would never match it and every lookup would return None.
    try:
        rel = Path(path).relative_to(Path(clone_dir).resolve())
    except ValueError:
        return None
    if not rel.parts:
        return None
    name = rel.parts[0]
    if name.startswith("."):
        return None
    if known is not None and name not in known:
        return None
    return name


@dataclass
class BuildState:
    package: str
    phase: str
    elapsed: float
    jobs: int = 0
    #: what the jobs actually are, e.g. "rustc ×4, cc1 ×1". A bare count says
    #: something is happening; this says what.
    workers: str = ""

    @property
    def clock(self) -> str:
        return format_duration(self.elapsed)


def format_duration(seconds: float) -> str:
    total = int(seconds)
    if total < 60:
        return f"{total}s"
    if total < 3600:
        return f"{total // 60}m{total % 60:02d}s"
    return f"{total // 3600}h{(total % 3600) // 60:02d}m"


class Watcher:
    """Polls the process tree under a build and reports per-package state.

    Completion is derived from observation rather than artefacts: paru's
    `CleanAfter` deletes the built package, so a package that was seen building
    and now has no processes is finished. Artefacts are used only to corroborate.
    """

    def __init__(
        self, clone_dir: Path, known: set[str] | None = None, *, include_root: bool = False
    ) -> None:
        self.clone_dir = clone_dir
        self.known = known
        self.include_root = include_root
        self.seen: set[str] = set()
        #: pending, drained by `new_finishes`
        self.finished: list[str] = []
        #: everything ever handed to the caller. `finished` alone cannot serve
        #: as both the queue and the record: clearing it to report means the
        #: next poll re-finishes the same package, which shows up as a duplicate
        #: row in the feed.
        self.reported: set[str] = set()

    def poll(self, root_pid: int | None) -> list[BuildState]:
        """Observe the tree. `root_pid=None` means it is gone, which drains.

        This is how a package that was still building when the launcher exited
        still reaches the feed, instead of being silently lost.
        """
        procs = snapshot()
        children = (
            descendants(root_pid, procs, include_root=self.include_root)
            if root_pid is not None
            else []
        )

        grouped: dict[str, list[Proc]] = {}
        for proc in children:
            pkg = package_of(proc.cwd, self.clone_dir, self.known)
            if pkg:
                grouped.setdefault(pkg, []).append(proc)

        # A package we have seen building and that no longer has processes is
        # done. `makepkg` holds the package's clone directory as its working
        # directory for the whole build, so a package only disappears from the
        # tree when it has genuinely finished.
        for pkg in sorted(self.seen - set(grouped)):
            if pkg in self.reported:
                continue
            self.reported.add(pkg)
            self.finished.append(pkg)
        self.seen |= set(grouped)

        states = []
        for pkg, group in sorted(grouped.items()):
            # elapsed is measured from the oldest process still alive for this
            # package, which is the honest lower bound on its build time
            oldest = min(p.started for p in group)
            compilers = sum(1 for p in group if classify([p]) == "compile")
            states.append(
                BuildState(
                    package=pkg,
                    phase=classify(group),
                    elapsed=time.time() - oldest,
                    jobs=compilers or len(group),
                    workers=summarise_workers(group),
                )
            )
        return states

    def new_finishes(self) -> list[str]:
        out = self.finished
        self.finished = []
        return out


# --------------------------------------------------------------------------
# repo-phase download progress (real bytes, no guessing)
# --------------------------------------------------------------------------


def download_progress(cache_dir: Path = Path("/var/cache/pacman/pkg")) -> tuple[int, int]:
    """(bytes currently staged, number of in-flight .part files).

    pacman writes `*.part` while downloading and renames on completion, so the
    sum of those sizes is a real progress figure rather than an animation.
    """
    total = 0
    count = 0
    try:
        for entry in os.scandir(cache_dir):
            if entry.name.endswith(".part"):
                try:
                    total += entry.stat().st_size
                    count += 1
                except OSError:
                    continue
    except OSError:
        return 0, 0
    return total, count