git/orchard/main
updater/srcinfo.py
607 lines22 KB last change raw · history
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
"""Read AUR package metadata, statically.

.SRCINFO is a stable, machine-generated format and is the right interface for
reading AUR metadata.

A PKGBUILD is not: it is executable shell owned by whoever maintains the package.
`source`-ing one to inspect it runs their code at *review* time, before any
decision to trust it has been made. Everything here is static parsing, and the
tool must never grow a "just source it to be sure" shortcut.
"""

from __future__ import annotations

import platform
import re
from dataclasses import dataclass, field
from pathlib import Path

# fields that are lists and may legitimately repeat
_LIST_FIELDS = {
    "arch",
    "license",
    "makedepends",
    "checkdepends",
    "depends",
    "optdepends",
    "provides",
    "conflicts",
    "replaces",
    "source",
    "sha256sums",
    "sha512sums",
    "b2sums",
    "md5sums",
    "validpgpkeys",
    "options",
    "backup",
    "groups",
    "noextract",
}

#: architectures Arch recognises in `.SRCINFO` suffixed keys
_ARCHES = ("x86_64", "aarch64", "armv7h", "i686", "riscv64", "pentium4", "any")
#: the one this machine builds for, so another architecture's deps and sources
#: do not become edges here
_HOST_ARCH = platform.machine()

_VER_CONSTRAINT = re.compile(r"(>=|<=|=|>|<)")


def dep_name(spec: str) -> str:
    """`python>=3.14` -> `python`; `foo: for bar` -> `foo`."""
    spec = spec.split(":", 1)[0].strip()
    return _VER_CONSTRAINT.split(spec, 1)[0].strip()


@dataclass
class SrcInfo:
    pkgbase: str = ""
    pkgname: list[str] = field(default_factory=list)
    pkgver: str = ""
    pkgrel: str = ""
    epoch: str = ""
    url: str = ""
    install: str = ""
    makedepends: list[str] = field(default_factory=list)
    checkdepends: list[str] = field(default_factory=list)
    depends: list[str] = field(default_factory=list)
    optdepends: list[str] = field(default_factory=list)
    provides: list[str] = field(default_factory=list)
    conflicts: list[str] = field(default_factory=list)
    replaces: list[str] = field(default_factory=list)
    source: list[str] = field(default_factory=list)
    sha256sums: list[str] = field(default_factory=list)
    sha512sums: list[str] = field(default_factory=list)
    b2sums: list[str] = field(default_factory=list)
    md5sums: list[str] = field(default_factory=list)
    validpgpkeys: list[str] = field(default_factory=list)
    arch: list[str] = field(default_factory=list)
    license: list[str] = field(default_factory=list)
    options: list[str] = field(default_factory=list)
    backup: list[str] = field(default_factory=list)
    groups: list[str] = field(default_factory=list)
    noextract: list[str] = field(default_factory=list)
    # per-subpackage depends, kept separate so a subpackage's deps are visible
    per_package_depends: dict[str, list[str]] = field(default_factory=dict)

    @property
    def sums(self) -> list[str]:
        """Every checksum list this package declares, in preference order.

        A package may use only b2sums or only sha512sums. Reading just
        sha256sums made those packages look as though they declared no checksums
        at all, and the audit treats "no checksums on a network source" as
        nothing to check -- so a SKIP introduced on a non-VCS source went
        unreported.
        """
        return [*self.sha256sums, *self.sha512sums, *self.b2sums, *self.md5sums]

    @property
    def has_sums(self) -> bool:
        return bool(self.sums)

    @property
    def primary_sums(self) -> list[str]:
        """The checksum list that parallels `source`.

        A PKGBUILD uses one checksum array for its source list, so the first
        non-empty one is the list to pair against `source` by index.
        Concatenating them (`sums`) would misalign the indices for any package
        that declares more than one kind.
        """
        for candidate in (self.sha256sums, self.sha512sums, self.b2sums, self.md5sums):
            if candidate:
                return candidate
        return []

    @property
    def version(self) -> str:
        core = f"{self.pkgver}-{self.pkgrel}" if self.pkgrel else self.pkgver
        return f"{self.epoch}:{core}" if self.epoch else core

    @property
    def build_depends(self) -> list[str]:
        """Everything needed before `package()` runs, deduped, order preserved."""
        seen: dict[str, None] = {}
        for spec in (*self.makedepends, *self.checkdepends, *self.depends):
            seen.setdefault(dep_name(spec), None)
        return list(seen)

    def declared_names(self) -> list[str]:
        return self.pkgname or ([self.pkgbase] if self.pkgbase else [])


def parse_srcinfo(text: str) -> SrcInfo:
    info = SrcInfo()

    for raw in text.splitlines():
        line = raw.strip()
        if not line or "=" not in line:
            continue
        key, _, value = line.partition("=")
        key, value = key.strip(), value.strip()
        if not key or not value:
            continue

        # `.SRCINFO` carries `source_x86_64`, `depends_aarch64` and the like.
        # Without this the arch-specific entries match no field at all and are
        # dropped, so a package whose sources are arch-suffixed looks like it
        # has no sources and no checksums -- a silent false negative on the
        # architecture actually being built for.
        # Matched against the known names rather than split on the last
        # underscore: the architecture names contain one, so `x86_64` would
        # otherwise split into `source_x86` and `64`. Only this machine's
        # architecture is merged -- another one's dependencies and sources do
        # not apply here and would become false edges in the build graph.
        for arch in (_HOST_ARCH, "any"):
            if key.endswith(f"_{arch}") and key != f"_{arch}":
                base = key[: -len(arch) - 1]
                if base in _LIST_FIELDS or hasattr(info, base):
                    key = base
                break

        if key == "pkgbase":
            info.pkgbase = value
            current_pkg = None
            continue
        if key == "pkgname":
            info.pkgname.append(value)
            current_pkg = value
            info.per_package_depends.setdefault(value, [])
            continue

        if key == "depends" and current_pkg is not None:
            info.per_package_depends[current_pkg].append(value)
            info.depends.append(value)
            continue

        if key in _LIST_FIELDS:
            getattr(info, key, None)
            if hasattr(info, key) and isinstance(getattr(info, key), list):
                getattr(info, key).append(value)
            continue

        if hasattr(info, key):
            setattr(info, key, value)

    return info


# --------------------------------------------------------------------------
# static PKGBUILD parsing
# --------------------------------------------------------------------------


@dataclass
class Pkgbuild:
    """A statically parsed PKGBUILD. `raw` is kept for pattern scanning."""

    raw: str
    vars: dict[str, list[str]] = field(default_factory=dict)
    funcs: dict[str, str] = field(default_factory=dict)
    #: `raw` with comments removed. Whole-file rules scan this instead, because a
    #: comment saying "never do `curl ... | sh` here" is not a network call.
    #: Suppression also looks for the pattern anywhere in the previous revision,
    #: so the same comment there suppresses a genuine new finding.
    code: str = ""

    def get(self, name: str) -> list[str]:
        return self.vars.get(name, [])

    def one(self, name: str) -> str:
        v = self.vars.get(name)
        return v[0] if v else ""


def _strip_comment(line: str) -> str:
    """Remove a trailing comment, the way the shell decides what one is.

    A `#` only begins a comment when it starts a word: at the beginning of the
    line, or just after whitespace, and never inside quotes. Shell does not
    treat every hash as a comment and neither may we, because PKGBUILDs are full
    of literal ones -- `git+https://host/r.git#tag=v1.0`, `${#arr[@]}`, `$#`.

    Treating them as comments was expensive. It truncated array elements
    mid-value, which removed a `}` or `{` and skewed brace depth so one function
    swallowed the next; a swallowed function is a function the audit never
    scans. It also silently dropped checksums, leaving the audit with nothing to
    compare.
    """
    quote = ""
    i = 0
    while i < len(line):
        char = line[i]
        if quote:
            if char == "\\" and quote == '"':
                i += 2
                continue
            if char == quote:
                quote = ""
        elif char in "'\"":
            quote = char
        elif char == "#" and (i == 0 or line[i - 1] in " \t"):
            return line[:i]
        i += 1
    return line


def _skip_substitution(text: str, i: int) -> int:
    """Index just past a `$(...)` or `${...}` starting at `i`, or -1 if unclosed.

    Substitutions are their own quoting context. `"$(sed -n "s/x/y/p" f)"` is
    valid shell: the inner quotes belong to the substitution and do not close the
    outer one. Reading them as ordinary quotes left the scanner convinced a quote
    was still open, so a function body never appeared to end and it swallowed
    every function after it -- which meant the audit never saw them.
    """
    if i + 1 >= len(text) or text[i] != "$" or text[i + 1] not in "({":
        return -1
    opener = text[i + 1]
    closer = ")" if opener == "(" else "}"
    depth = 1
    quote = ""
    j = i + 2
    while j < len(text):
        char = text[j]
        if quote:
            if char == "\\" and quote == '"':
                j += 2
                continue
            if char == quote:
                quote = ""
        elif char in "'\"":
            quote = char
        elif char == opener:
            depth += 1
        elif char == closer:
            depth -= 1
            if depth == 0:
                return j + 1
        j += 1
    return -1


def _scan_array_end(body: str) -> int:
    """Index of the `)` that closes an array whose `(` was just consumed.

    Quote-aware, because a quoted element may contain a `)`:
    `'foo: does things (fast)'`. Stopping at the first one truncated the element,
    left the quotes unbalanced, and made the fallback split the rest of the
    array on whitespace -- losing entries outright.

    Command substitutions are skipped over, so `source=($(printf '%s' a))` still
    finds the array's own closing paren.
    """
    quote = ""
    i = 0
    while i < len(body):
        char = body[i]
        if char == "\\" and quote != "'":
            i += 2
            continue
        if quote == "'":
            if char == "'":
                quote = ""
            i += 1
            continue
        if char == '"':
            quote = "" if quote == '"' else '"'
            i += 1
            continue
        if char == "'" and quote == "":
            # only opens a quote outside one; a `'` inside "..." is literal
            quote = "'"
            i += 1
            continue
        # a substitution is its own quoting context even inside double quotes,
        # which is where they usually are: `"$(sed -n "s/x/y/p" f)"`
        if char == "$" and (body.startswith("$(", i) or body.startswith("${", i)):
            # `$(( 1 + 2 ))` nests, so a plain depth counter gets this wrong and
            # mistakes the arithmetic's own closing paren for the array's
            end = _skip_substitution(body, i)
            if end < 0:
                return -1
            i = end
            continue
        if quote == "" and char == ")":
            return i
        i += 1
    return -1


#: `foo{,.sig}` and friends. Non-nested only, and never a `${substitution}` --
#: `https://x/f-${pkgver}.tar.gz{,.sig}` has two brace groups and only the second
#: one is a list.
_BRACE = re.compile(r"(?<!\$)\{([^{}]*)\}")


def _expand_braces(word: str, limit: int = 64) -> list[str]:
    """Expand a bash brace list, e.g. `sdl2-1.0.tar.gz{,.sig}` -> two entries.

    Bash performs this at parse time, and .SRCINFO records the expanded result,
    so a source entry with a brace list is really several sources. Leaving it
    unexpanded made `source` shorter than `sha256sums`, which shifted every
    checksum onto the wrong source -- and the audit pairs them by position.
    """
    match = _BRACE.search(word)
    if not match:
        return [word]
    body = match.group(1)
    if "," not in body:
        return [word]  # `{a}` is literal in bash, not a list
    out: list[str] = []
    for option in body.split(","):
        out.extend(_expand_braces(word[: match.start()] + option + word[match.end() :], limit))
        if len(out) > limit:
            return [word]  # pathological; better to under-expand than explode
    return out


def _split_words(body: str) -> list[str]:
    """Split an array body into elements. No variable expansion.

    Hand-written rather than shlex, because shlex splits on the whitespace
    *inside* a command substitution: `$(_checksum ${_tag})` came back as
    `['$(_checksum', '${_tag})']`, which detaches a source or a checksum from the
    thing that identifies it. Substitutions are kept whole, and quoted text keeps
    its spaces.
    """
    words: list[str] = []
    current: list[str] = []
    quote = ""
    i = 0
    while i < len(body):
        char = body[i]
        if quote:
            if char == "\\" and quote == '"' and i + 1 < len(body):
                current.append(body[i + 1])
                i += 2
                continue
            if char == quote:
                quote = ""
            else:
                current.append(char)
        elif char in "'\"":
            quote = char
        elif char == "\\":
            # a backslash-newline continues the word; otherwise it escapes
            if i + 1 < len(body) and body[i + 1] == "\n":
                i += 2
                continue
            if i + 1 < len(body):
                current.append(body[i + 1])
                i += 2
                continue
        elif body.startswith("$(", i) or body.startswith("${", i):
            opener = body[i + 1]
            closer = ")" if opener == "(" else "}"
            depth = 1
            # consume both `$` and its opener before scanning, or the loop counts
            # the opener a second time and swallows the rest of the array
            current.append(body[i])
            current.append(opener)
            i += 2
            while i < len(body) and depth > 0:
                inner = body[i]
                if inner == opener:
                    depth += 1
                elif inner == closer:
                    depth -= 1
                current.append(inner)
                i += 1
            continue
        elif char.isspace():
            if current:
                words.append("".join(current))
                current = []
        else:
            current.append(char)
        i += 1
    if current:
        words.append("".join(current))
    expanded: list[str] = []
    for word in words:
        expanded.extend(_expand_braces(word))
    return expanded


#: a function body is delimited by one of these pairs
_OPENERS = {"{": "}", "(": ")"}


def _func_head(stripped: str) -> tuple[str, str, str] | None:
    """`(name, opener, remainder)` for a function definition line, else None.

    Function names may contain more than word characters: split packages use
    `package_<pkgname>()`, and real PKGBUILDs have hyphenated names, so the
    character class has to allow them or those bodies are never parsed and never
    scanned.
    """
    match = re.match(r"^([A-Za-z_][A-Za-z0-9_.+-]*)\s*\(\s*\)\s*(.*)$", stripped)
    if not match:
        return None
    name, rest = match.group(1), match.group(2).strip()
    opener = ""
    if rest[:1] in _OPENERS:
        opener, rest = rest[0], rest[1:]
    elif rest:
        return None  # something else on the line: not a definition after all
    return name, opener, rest


def _body_closed(text: str, opener: str, closer: str) -> bool:
    """Whether every unquoted opener in `text` has been closed.

    Quote-aware on purpose. A `}` inside a quoted string used to be counted, so
    one stray brace made a function swallow the next one -- and a swallowed
    function is a function the audit never looks at.
    """
    quote = ""
    depth = 0
    i = 0
    while i < len(text):
        char = text[i]
        if char == "\\" and quote != "'":
            i += 2
            continue
        if quote == "'":
            if char == "'":
                quote = ""
            i += 1
            continue
        if char == '"':
            quote = "" if quote == '"' else '"'
            i += 1
            continue
        if char == "'" and quote == "":
            # only opens a quote outside one; a `'` inside "..." is literal
            quote = "'"
            i += 1
            continue
        # inside double quotes a substitution is still its own context, and the
        # braces of `${...}` must not be counted towards the body's depth
        if char == "$" and (text.startswith("$(", i) or text.startswith("${", i)):
            end = _skip_substitution(text, i)
            if end < 0:
                return False  # unclosed substitution: keep reading lines
            i = end
            continue
        if quote == "":
            if char == opener:
                depth += 1
            elif char == closer:
                depth -= 1
        i += 1
    return depth <= 0 and not quote


def _strip_body(text: str, opener: str, closer: str) -> str:
    """Drop the single enclosing pair of delimiters."""
    text = text.strip()
    if opener and text.startswith(opener):
        text = text[1:]
    text = text.rstrip()
    if closer and text.endswith(closer):
        text = text[:-1]
    return text.strip()


def parse_pkgbuild(text: str) -> Pkgbuild:
    """Parse assignments and function bodies without executing anything.

    Deliberately not a shell. It handles the shapes PKGBUILDs actually take
    (scalar assignments, quoted arrays spanning lines, `name() { ... }`,
    `name() ( ... )`) and records anything it does not understand verbatim in
    `raw`, which is what the audit scans. It never expands variables.
    """
    pb = Pkgbuild(raw=text)
    pb.code = "\n".join(_strip_comment(line) for line in text.splitlines())
    lines = text.splitlines()
    i = 0
    while i < len(lines):
        stripped = _strip_comment(lines[i]).strip()

        head = _func_head(stripped)
        if head is not None:
            name, opener, rest = head
            if not opener:
                # `prepare()` with the brace on the following line
                nxt = _strip_comment(lines[i + 1]).strip() if i + 1 < len(lines) else ""
                if nxt[:1] in _OPENERS:
                    opener, rest = nxt[0], nxt[1:]
                    i += 1
            if opener:
                closer = _OPENERS[opener]
                # keep the opener in the text being balanced: `check() (` has an
                # empty remainder, which would otherwise already look closed and
                # the whole body would be skipped
                body = opener + rest
                while not _body_closed(body, opener, closer) and i + 1 < len(lines):
                    i += 1
                    body += "\n" + _strip_comment(lines[i])
                body = _strip_body(body, opener, closer)
            else:
                body = rest
            pb.funcs[name] = body
            i += 1
            continue

        # `name[3]=value` assigns one element, which is how a package sets a
        # single checksum to SKIP without rewriting the whole array
        indexed = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)\[(\d+)\]\s*=\s*(.+)$", stripped)
        if indexed:
            name, slot, value = (indexed.group(1), int(indexed.group(2)), indexed.group(3).strip())
            if len(value) >= 2 and value[0] in "'\"" and value.endswith(value[0]):
                value = value[1:-1]
            current = pb.vars.setdefault(name, [])
            while len(current) <= slot:
                current.append("")
            current[slot] = value
            i += 1
            continue

        arr = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)\s*(\+?=)\s*\((.*)$", stripped)
        if arr:
            name, operator, rest = arr.group(1), arr.group(2), arr.group(3)
            while _scan_array_end(rest) < 0 and i + 1 < len(lines):
                i += 1
                rest += "\n" + _strip_comment(lines[i])
            end = _scan_array_end(rest)
            if end >= 0:
                rest = rest[:end]
            words = _split_words(rest)
            if operator == "=":
                pb.vars[name] = words
            else:
                pb.vars.setdefault(name, []).extend(words)
            i += 1
            continue

        scalar = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.+)$", stripped)
        if scalar:
            value = scalar.group(2).strip()
            # keep the whole value: `pkgdesc=hello world` is one string, and
            # taking the first word silently truncated every unquoted one
            if len(value) >= 2 and value[0] in "'\"" and value.endswith(value[0]):
                value = value[1:-1]
            pb.vars.setdefault(scalar.group(1), []).append(value)
            i += 1
            continue

        i += 1
    return pb


def load(clone_dir: Path, package: str) -> tuple[SrcInfo | None, Pkgbuild | None]:
    """Read both metadata files for a cloned AUR package. Never executes them."""
    base = clone_dir / package
    srcinfo = pkgbuild = None
    si = base / ".SRCINFO"
    if si.is_file():
        srcinfo = parse_srcinfo(si.read_text(errors="replace"))
    pk = base / "PKGBUILD"
    if pk.is_file():
        pkgbuild = parse_pkgbuild(pk.read_text(errors="replace"))
    return srcinfo, pkgbuild