Building an AI Code Review Agent

PR Event → Diff Extraction → Context Enrichment → LLM Analysis → Comment Generation → GitHub API

from github import Github
import openai

class CodeReviewAgent:
    def __init__(self, gh_token: str, openai_key: str):
        self.gh = Github(gh_token)
        self.llm = openai.AsyncOpenAI(api_key=openai_key)

    async def review_pr(self, owner: str, repo: str, pr_number: int):
        repo_obj = self.gh.get_repo(f"{owner}/{repo}")
        pr = repo_obj.get_pull(pr_number)

        diff = self.extract_diff(pr)
        context = self.enrich_with_context(repo_obj, diff)
        findings = await self.analyze(diff, context)

        for finding in findings:
            if finding.severity >= FindingSeverity.MEDIUM:
                await pr.create_review_comment(
                    body=finding.comment,
                    commit=pr.get_commits().reversed[0],
                    path=finding.file,
                    line=finding.line,
                )

def enrich_with_context(repo, diff: DiffHunk) -> ReviewContext:
    """Gather surrounding code, related files, and project conventions."""
    context = ReviewContext()

    for hunk in diff.hunks:
        # Get 20 lines of surrounding context
        context.surrounding[hunk.file] = repo.get_file_content(
            hunk.file,
            start_line=max(0, hunk.start_line - 20),
            end_line=hunk.end_line + 20
        )

        # Find related test files
        test_file = find_test_file(hunk.file)
        if test_file:
            context.tests[test_file] = repo.get_file_content(test_file)

        # Load project lint rules
        context.lint_config = repo.get_file_content('.eslintrc.json')

    return context

REVIEW_PROMPT = """You are a senior engineer reviewing a pull request.

## Rules
1. Only flag issues that are **actual bugs**, **security vulnerabilities**, or **significant performance problems**
2. Do NOT comment on style preferences (formatting, naming conventions) — the linter handles those
3. Do NOT suggest changes that would require modifying more than 10 lines
4. Always provide a code example showing the fix
5. Rate severity: CRITICAL, HIGH, MEDIUM, LOW

## Codebase Context
{context}

## Diff
```diff
{diff}

Key lessons from prompt iteration:

- **Explicit negative constraints** ("do NOT comment on style") reduced noise by 45%
- **Requiring code examples** forced the model to ground its suggestions in reality
- **Severity classification** let us filter which comments actually get posted to the PR

## Filtering and Noise Reduction

Not every LLM suggestion should reach the developer. We apply a multi-layer filter:

```python
def filter_findings(findings: list[Finding]) -> list[Finding]:
    """Remove low-value findings before posting to PR."""
    filtered = []

    for f in findings:
        # Skip low severity
        if f.severity < FindingSeverity.MEDIUM:
            continue

        # Skip if it's a known pattern (e.g., intentional performance trade-off)
        if is_known_pattern(f.file, f.line):
            continue

        # Skip if a human already commented on this
        if already_commented(f.file, f.line):
            continue

        # Deduplicate similar findings
        if is_duplicate(f, filtered):
            continue

        filtered.append(f)

    return filtered