Python tracemalloc

tracemalloc is a memory profiling tool builtin to python-3.4+.

• get traceback where item was allocated
• where memory was allocated (by line number, which file, ...)
• compare memory snapshots to identify leaks

Tutorials

Sanket intro to memory profiling (youtube)

https://www.youtube.com/watch?v=u0qVRm8Hjb4

Usage

Capture a snapshot

import tracemalloc

# get snapshot
tracemalloc.start()
# .. run things..
snapshot = tracemalloc.take_snapshot()

Extract statistics

# top-10 memory causing lines of code
top_stats = snapshot.statistics('lineno')
pprint.pprint(top_stats[:10])

# compare snapshots, and get top 10 offenders
top_stats = snapshot.compare_to(snapshot, 'lineno')
pprint.pprint(top_stats[:10])

# get traceback of offenders
snaphot.statistics('traceback')

# filter by module path
import fnmatch
import tracemalloc
top_stats = filter(lambda x: fnmatch.fnmatch('*/yourproject/*', x.traceback[0].filename), snapshot.statistics('lineno'))
pprint.pprint(top_stats[10:])

How I solved last issue

snapshot = None
last_snapshot = None


def mainloop():
    while True:
        if i % 20 == 0:
            print_memprofile()
        # ...


def print_memprofile():
    global snapshot
    global last_snapshot
    last_snapshot = snapshot
    snapshot = tracemalloc.take_snapshot()
    if last_snapshot:
        top_stats = snapshot.compare_to(last_snapshot, 'lineno')
        top_stats = list(filter(lambda x: fnmatch.fnmatch(x.traceback[0].filename, '*/multivolumecopy/*'), top_stats))
        top_stats = sorted(top_stats, key=lambda x: x.size)
        print('----------------------')
        pprint.pprint(list(reversed(top_stats[-10:])))
        import pdb;pdb.set_trace()