Black Magic and Voodoo

How PyTest generates its failure reports

by Nathan Alderson

Who am I?

nathanalderson.com

@nathanalderson

facebook.com/nathanalderson

github.com/nathanalderson

Why PyTest

Assert With the Assert Statement

def test_set_comparison():
    set1 = set("1308")
    set2 = set("8035")
    assert set1 == set2
        

Beautiful Failure Reports

» py.test
________________ test_function __________________
    def test_set_comparison():
        set1 = set("1308")
        set2 = set("8035")
>       assert set1 == set2
E       assert set(['0', '1', '3', '8']) == set(['0', '3', '5', '8'])
E         Extra items in the left set:
E         '1'
E         Extra items in the right set:
E         '5'
E         Use -v to get the full diff

src/test/python/test_assert1.py:28: AssertionError
============== 1 failed in 0.07 seconds ===============
        

Magic?

They rewrite your code...

Step 1 - PEP302 Import Hook

# _pytest/assertion/__init__.py
def pytest_configure(config):
  ...
  if mode == "rewrite":
    hook = rewrite.AssertionRewritingHook()
    sys.meta_path.insert(0, hook)
        

Step 2 - Implement Finder and Loader

# _pytest/assertion/rewrite.py
class AssertionRewritingHook(object):
  def find_module(self, name, path=None):
    if this_is_a_test_module:
      co = _read_pyc(fn_pypath, pyc) # check cache
      if co is None:
        co = _rewrite_test(fn_pypath)
        _make_rewritten_pyc(pyc, co)  # cache it
      self.modules[name] = co, pyc
      return self
  def load_module(self, name):
    # ...basically...
    return sys.modules[name]
        

Interlude - What are *.pyc files anyway?

→

→

Step 3 - Parse, Modify, Compile

def _rewrite_test(fn):
  source = fn.read("rb")
  tree = ast.parse(source)
  rewrite_asserts(tree) # o_O
  co = compile(tree, fn.strpath, "exec")
  return co
    

def rewrite_asserts(mod):
  AssertionRewriter().run(mod)

class AssertionRewriter(ast.NodeVisitor):
  def run(self, mod):
    # basically...
    for node in tree:
      if isinstance(node, ast.Assert):
        self.visit(node)

  def generic_visit(self, node):
  def visit_Assert(self, assert_):
  def visit_Name(self, name):
  def visit_BoolOp(self, boolop):
  def visit_UnaryOp(self, unary):
  def visit_BinOp(self, binop):
  def visit_Call(self, call):
  def visit_Attribute(self, attr):
  def visit_Compare(self, comp): 
    

def visit_Assert(self, assert_):
  ...
  # Rewrite assert into a bunch of statements. Build explanation.
  top_condition, explanation = self.visit(assert_.test)
  body = self.on_failure
  # Create the `if(!condition)` statement
  negation = ast.UnaryOp(ast.Not(), top_condition)
  self.statements.append(ast.If(negation, body, []))
  # Format the explanation
  explanation = "assert " + explanation
  template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation))
  msg = self.pop_format_context(template)
  fmt = self.helper("format_explanation", msg)
  # raise the AssertionError
  err_name = ast.Name("AssertionError", ast.Load())
  exc = ast.Call(err_name, [fmt], [], None, None)
  raise_ = ast.Raise(exc, None, None)
  body.append(raise_)
  # Clear temporary variables by setting them to None.
  if self.variables:
    variables = [ast.Name(name, ast.Store())
                 for name in self.variables]
    clear = ast.Assign(variables, _NameConstant(None))
    self.statements.append(clear) 

Before

def test_make_empty_file():
  name = "/tmp/empty_test"
  make_empty_file(name)
  with open(name, "r") as fp:
    assert not fp.read()
    

After

def test_make_empty_file():
  name = '/tmp/empty_test'
  make_empty_file(name)
  with open(name, 'r') as fp:
    @py_assert1 = fp.read
    @py_assert3 = @py_assert1()
    @py_assert5 = (not @py_assert3)
    if (not @py_assert5):
      @py_format6 = ('assert not %(py4)s\n{%(py4)s = %(py2)s\n{%(py2)s = %(py0)s.read\n}()\n}' %
      {'py0': (@pytest_ar._saferepr(fp) if ('fp' in @py_builtins.locals() is not @py_builtins.globals()) else 'fp'),
       'py2': @pytest_ar._saferepr(@py_assert1),
       'py4': @pytest_ar._saferepr(@py_assert3)})
      raise AssertionError(@pytest_ar._format_explanation(@py_format6))
    del @py_assert5, @py_assert1, @py_assert3
    

Resources

• The pytest source code, particularly rewrite.py. This presentation references the code at about the 2.7.1 tag.
• This pybites blogpost
• pytest.org, particularly here.
• PEP 0302
• The Python docs. Particularly ast and compile.
• Presentation written using impress.js. Source code on github.