Linux Applications Debugging Techniques/The call stack

The API

Sometimes we need the call stack at a certain point in the program. These are the API functions to get basic stack information:

#include <execinfo.h>

int backtrace(void **buffer, int size);
char **backtrace_symbols(void *const *buffer, int size);
void backtrace_symbols_fd(void *const *buffer, int size, int fd); 

#include <cxxabi.h>
char* __cxa_demangle(const char* __mangled_name, char* __output_buffer, size_t* __length, int* __status);

#include <dlfcn.h>
int dladdr(void *addr, Dl_info *info);

Notes:

• C++ symbols are still mangled. Use abi::__cxa_demangle() or something similar.
• Some of the these functions do allocate memory - either temporarily either explicitly - and this might be a problem if the program is unstable already.
• Some of the these functions do acquire locks (e.g. dladdr()).
• backtrace* are expensive calls on some platforms (x86_64 for instance), the deeper the call stack is the more expensive the call is.
• dladdr will fail to rsolve any symbol that is not exported. Hence, to get the most out of it:
  • Link with -rdynamic and -ldl

To extract more information (file, line number), use libbfd. Possible alternatives: libbacktrace; http://libcsdbg.sourceforge.net/

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <dlfcn.h>

#include <execinfo.h>
#include <signal.h>
#include <bfd.h>
#include <unistd.h>

/* globals retained across calls to resolve. */
static bfd* abfd = 0;
static asymbol **syms = 0;
static asection *text = 0;

static void resolve(char *address) {
    if (!abfd) {
 	  char ename[1024];
 	  int l = readlink("/proc/self/exe",ename,sizeof(ename));
 	  if (l == -1) {
 	    perror("failed to find executable\n");
 	    return;
 	  }
 	  ename[l] = 0;

 	  bfd_init();

 	  abfd = bfd_openr(ename, 0);
 	  if (!abfd) {
 	      perror("bfd_openr failed: ");
 	      return;
 	  }
	  
 	  /* oddly, this is required for it to work... */
 	  bfd_check_format(abfd,bfd_object);

 	  unsigned storage_needed = bfd_get_symtab_upper_bound(abfd);
 	  syms = (asymbol **) malloc(storage_needed);
 	  unsigned cSymbols = bfd_canonicalize_symtab(abfd, syms);

 	  text = bfd_get_section_by_name(abfd, ".text");
    }
	
    long offset = ((long)address) - text->vma;
    if (offset > 0) {
        const char *file;
        const char *func;
        unsigned line;
        if (bfd_find_nearest_line(abfd, text, syms, offset, &file, &func, &line) && file)
            printf("file: %s, line: %u, func %s\n",file,line,func);
    }
}

A C++ wrapper

With this simple class, getting the stack is one line away:

class call_stack 
{
public:

    static const int depth = 40;
    typedef std::array<void *, depth> stack_t;
    
    class const_iterator;    
    class frame
    {
    public:
    
        frame(void *addr = 0) 
                : _addr(0)
                , _dladdr_ret(false)
                , _binary_name(0) 
                , _func_name(0)
                , _demangled_func_name(0)
                , _delta_sign('+')
                , _delta(0L)
                , _source_file_name(0)
                , _line_number(0)
        {
            resolve(addr);
        }

        // frame(stack_t::iterator& it) : frame(*it) {} //C++0x
        frame(stack_t::const_iterator const& it) 
                : _addr(0)
                , _dladdr_ret(false)
                , _binary_name(0) 
                , _func_name(0)
                , _demangled_func_name(0)
                , _delta_sign('+')
                , _delta(0L)
                , _source_file_name(0)
                , _line_number(0)
        {
            resolve(*it);
        }
        
        frame(frame const& other)
        {
            resolve(other._addr);            
        }
        
        frame& operator=(frame const& other)
        {
            if (this != &other) {
                resolve(other._addr);            
            }
            return *this;
        }
        
        ~frame() 
        {
            resolve(0);
        }
        
        std::string as_string() const
        {
            std::ostringstream s;
            s << "[" << std::hex << _addr << "] " 
              << demangled_function() 
              << " (" << binary_file() << _delta_sign << "0x" << std::hex << _delta << ")"
              << " in " << source_file() << ":" << line_number()
              ;
            return s.str();
        }

        const void* addr() const               { return _addr; }
        const char* binary_file() const        { return safe(_binary_name); }
        const char* function() const           { return safe(_func_name); }
        const char* demangled_function() const { return safe(_demangled_func_name); }
        char        delta_sign() const         { return _delta_sign; }
        long        delta() const              { return _delta; }
        const char* source_file() const        { return safe(_source_file_name); } 
        int         line_number() const        { return  _line_number; }
        
    private:
    
        const char* safe(const char* p) const { return p ? p : "??"; }
        
        friend class const_iterator; // To call resolve()
        void resolve(const void * addr)
        {
            if (_addr == addr)
                return;
                
            _addr = addr;
            _dladdr_ret = false; 
            _binary_name = 0; 
            _func_name = 0;
            if (_demangled_func_name) {
                free(_demangled_func_name);
                _demangled_func_name = 0;
            }
            _delta_sign = '+';
            _delta = 0L;
            _source_file_name = 0; 
            _line_number = 0;
            
            if (!_addr)
                return;
                          
            _dladdr_ret = (::dladdr(_addr, &_info) != 0);
            if (_dladdr_ret)
            {
                _binary_name = safe(_info.dli_fname);
                _func_name   = safe(_info.dli_sname);
                _delta_sign  = (_addr >=  _info.dli_saddr) ? '+' : '-';
                _delta = ::labs(static_cast<const char *>(_addr) - static_cast<const char *>(_info.dli_saddr));
                
                 int status = 0;
                 _demangled_func_name = abi::__cxa_demangle(_func_name, 0, 0, &status);
            }
        }
    
    private:
    
        const void* _addr;
        const char* _binary_name;
        const char* _func_name;
        const char* _demangled_func_name;
        char        _delta_sign;
        long        _delta;
        const char* _source_file_name; //TODO: libbfd
        int         _line_number;
        
        Dl_info     _info;
        bool        _dladdr_ret;
    }; //frame
    
    
    class const_iterator
            : public std::iterator< std::bidirectional_iterator_tag
                                  , ptrdiff_t 
                                  >
    {
    public:
    
        const_iterator(stack_t::const_iterator const& it)
                : _it(it)
                , _frame(it)
        {}
    
        bool operator==(const const_iterator& other) const 
        {
            return _frame.addr() == other._frame.addr(); 
        }

        bool operator!=(const const_iterator& x) const 
        {
            return !(*this == x);
        }

        const frame& operator*() const 
        {
            return _frame;
        }
        const frame* operator->() const 
        {
            return &_frame;
        }

        const_iterator& operator++() 
        {
            ++_it;
            _frame.resolve(*_it);
            return *this;
        }
        const_iterator operator++(int) 
        {
            const_iterator tmp = *this;
            ++_it;  
            _frame.resolve(*_it);
            return tmp;
        }

        const_iterator& operator--() 
        {
            --_it;  
            _frame.resolve(*_it);
            return *this;
        }
        const_iterator operator--(int) 
        {
            const_iterator tmp = *this;
            --_it;  
            _frame.resolve(*_it);
            return tmp;
        }
    
    private:
    
        const_iterator();
        
    private:
    
        frame                    _frame;
        stack_t::const_iterator  _it;
    }; //const_iterator

    
    call_stack() : _num_frames(0)
    {
        _num_frames = ::backtrace(_stack.data(), depth);
        assert(_num_frames >= 0 && _num_frames <= depth);
    }
    
    std::string as_string()
    {
        std::string s;
        const_iterator itEnd = end();
        for (const_iterator it = begin(); it != itEnd; ++it) {
            s += it->as_string();
            s += "\n";
        }
        return std::move(s);
    }
    
    virtual ~call_stack()
    {
    }
    
    const_iterator begin() const { return _stack.cbegin(); }
    const_iterator end() const   { return stack_t::const_iterator(&_stack[_num_frames]); }

private: 

    stack_t  _stack;
    int      _num_frames;
};

This class can be used with assertions, logging or exceptions.

The full code can be found at the LPT site and it offers three symbol resolver varieties. This code will require a C++11 compiler.

• A "basic" address-only resolver, without memory side-effects
• A libc resolver
• A libbfd resolver

A boost-based version of the call stack utilities collection can be found at https://github.com/melintea/Boost-Call_stack.

A stack resolved with the libc resolver, embedded in an exception, with debug-compiled code would look like:

Exception PAPI_add_event: Event exists, but cannot be counted due to hardware resource limits
At: 
0x41e11f ??+0x41e11f
        At ??:0
        In binaries/bin/papitest
0x421ae0 lpt::stack::call_stack<40ul>::call_stack(bool)+0x52
        At ??:0
        In binaries/bin/papitest
0x420653 lpt::papi::papi_error::papi_error(std::string const&, int)+0x6f
        At ??:0
        In binaries/bin/papitest
0x426ffe lpt::papi::library<lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::init()+0x722
        At ??:0
        In binaries/bin/papitest
0x426815 lpt::papi::library<lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::library()+0x5b
        At ??:0
        In binaries/bin/papitest
0x4265d4 lpt::singleton<lpt::papi::library<lpt::papi::counting_per_thread, lpt::papi::multiplex_none> >::singleton()+0x18
        At ??:0
        In binaries/bin/papitest
0x42624e lpt::singleton<lpt::papi::library<lpt::papi::counting_per_thread, lpt::papi::multiplex_none> >::instance()+0x42
        At ??:0
        In binaries/bin/papitest
0x425b47 lpt::papi::thread<lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::init()+0x25
        At ??:0
        In binaries/bin/papitest
0x425502 lpt::papi::thread<lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::thread()+0x2e
        At ??:0
        In binaries/bin/papitest
0x424554 lpt::singleton<lpt::papi::thread<lpt::papi::counting_per_thread, lpt::papi::multiplex_none> >::singleton()+0x18
        At ??:0
        In binaries/bin/papitest
0x4230a5 lpt::singleton<lpt::papi::thread<lpt::papi::counting_per_thread, lpt::papi::multiplex_none> >::instance()+0x42
        At ??:0
        In binaries/bin/papitest
0x421d33 lpt::papi::counters<lpt::papi::stdout_print, lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::counters(std::string const&)+0xc9
        At ??:0
        In binaries/bin/papitest
0x41e206 tests()+0x67
        At ??:0
        In binaries/bin/papitest
0x41edb8 ??+0x11
        At ??:0
        In binaries/bin/papitest
0x2ad66a7b9ead ??+0xfd
        At ??:0
        In /lib/x86_64-linux-gnu/libc.so.6
0x41dea9 ??+0x41dea9
        At ??:0
        In binaries/bin/papitest

Note how dladdr() fails to resolve some of the frames. Whereas a stack resolved with the bfd resolver would show file and line numbers and would resolve all frames:

Exception PAPI_add_event: Event exists, but cannot be counted due to hardware resource limits
At: 
[0x41e20f] backtrace+0x41e20f
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/detail/glibc_call_stack.hpp:61
        In binaries/bin/papitest
[0x421c4e] lpt::stack::call_stack<40ul>::call_stack(bool)+0x52
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/call_stack.hpp:74
        In binaries/bin/papitest
[0x4207a1] lpt::papi::papi_error::papi_error(std::string const&, int)+0x6f
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/papi.hpp:112
        In binaries/bin/papitest
[0x42716c] lpt::papi::library<lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::init()+0x722
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/papi.hpp:371
        In binaries/bin/papitest
[0x426983] lpt::papi::library<lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::library()+0x5b
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/papi.hpp:289
        In binaries/bin/papitest
[0x426742] lpt::singleton<lpt::papi::library<lpt::papi::counting_per_thread, lpt::papi::multiplex_none> >::singleton()+0x18
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/singleton.hpp:19
        In binaries/bin/papitest
[0x4263bc] lpt::singleton<lpt::papi::library<lpt::papi::counting_per_thread, lpt::papi::multiplex_none> >::instance()+0x42
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/singleton.hpp:28
        In binaries/bin/papitest
[0x425cb5] lpt::papi::thread<lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::init()+0x25
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/papi.hpp:433
        In binaries/bin/papitest
[0x425670] lpt::papi::thread<lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::thread()+0x2e
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/papi.hpp:409
        In binaries/bin/papitest
[0x4246c2] lpt::singleton<lpt::papi::thread<lpt::papi::counting_per_thread, lpt::papi::multiplex_none> >::singleton()+0x18
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/singleton.hpp:19
        In binaries/bin/papitest
[0x423213] lpt::singleton<lpt::papi::thread<lpt::papi::counting_per_thread, lpt::papi::multiplex_none> >::instance()+0x42
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/singleton.hpp:28
        In binaries/bin/papitest
[0x421ea1] lpt::papi::counters<lpt::papi::stdout_print, lpt::papi::counting_per_thread, lpt::papi::multiplex_none>::counters(std::string const&)+0xc9
        At /home/amelinte/projects/lpt/lpt/lpt/include/lpt/papi.hpp:646
        In binaries/bin/papitest
[0x41e2f6] tests()+0x67
        At /home/amelinte/projects/lpt/lpt/tests/papitest.cpp:51
        In binaries/bin/papitest
[0x41eea8] main+0x11
        At /home/amelinte/projects/lpt/lpt/tests/papitest.cpp:176
        In binaries/bin/papitest
[0x2b5f7aa1bead] __libc_start_main+0xfd
        At ??:0
        In /lib/x86_64-linux-gnu/libc.so.6
[0x41df99] _start+0x41df99
        At ??:0
        In binaries/bin/papitest

Caveats

• Do NOT use in asynchronous interrupts such as signal handlers.
• It might or it might not work when the program is unstable - prefer a core dump if possible. Same for memory issues handlers .
• Performance hit varies greatly with platform and compiler version.

From within gdb

A canned command to resolve a stack address from within gdb:

define addrtosym
    if $argc == 1
        printf "[%u]: ", $arg0
        #whatis/ptype EXPR
        #info frame ADDR
        info symbol $arg0
    end
end
document addrtosym
Resolve the address (e.g. of one stack frame). Usage: addrtosym addr0
end

Links

• Calling conventions