Index: Makefile
===================================================================
--- Makefile	(revision 510af0182efca6719f01850d71de97710acab1bf)
+++ Makefile	(revision 3c9fc9433c0641f662db8ee0e63c4e26c5fb07ba)
@@ -2,5 +2,7 @@
 CXXFLAGS = -std=gnu++0x -O3 -g
 
-CPPFLAGS =
+# When compiling with CXX=powerpc64-bgq-linux-g++, we need these:
+CPPFLAGS = -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk
+
 LDFLAGS = -lpthread -ldl
 
@@ -16,6 +18,7 @@
 	$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) -fPIC -shared -o libmemlog.so memlog.cpp
 
-install: all memlog2dot README
-	cp -a libmemlog.so memlog_s.o memlog2dot README $(DESTDIR)/
+install: all memlog_analyze README
+	cp -a libmemlog.so memlog_s.o memlog_analyze README $(DESTDIR)/
+	echo '-Wl,--wrap,malloc,--wrap,valloc,--wrap,realloc,--wrap,calloc,--wrap,memalign,--wrap,free,--wrap,posix_memalign,--wrap,mmap,--wrap,mmap64,--wrap,munmap $(DESTDIR)/memlog_s.o -lpthread -ldl' > $(DESTDIR)/memlog_s_ld_cmds
 
 clean:
Index: README
===================================================================
--- README	(revision 24aa7345b4768a61f07b49d5c3b9ba51bc17a75e)
+++ README	(revision 3c9fc9433c0641f662db8ee0e63c4e26c5fb07ba)
@@ -16,8 +16,20 @@
       -L/path/to/memlog -Wl,-rpath,/path/to/memlog -lmemlog
 
-For statically-linked applications, add the following to your linker flags:
+For statically-linked applications ld's automatic wrapping functionality is
+employed, and the exact set of necessary flags is large, so a file named
+memlog_s_ld_cmds has been provided containing the necessary flags.
 
-  -Wl,--wrap,malloc,--wrap,free,--wrap,realloc,--wrap,calloc,--wrap,memalign \
-    /path/to/memlog/memlog_s.o -lpthread -ldl
+To your linker flags add:
+
+  `cat /path/to/memlog/memlog_s_ld_cmds`
+
+or, if your compiler and wrappers support response files (gcc and clang do, for
+example), simply:
+
+  @/path/to/memlog/memlog_s_ld_cmds
+
+so your overall linking command might look something like this:
+
+  mpic++ -O3 -g -o my_program my_obj1.o my_obj2.o @/path/to/memlog/memlog_s_ld_cmds
 
 ** RUNNING **
@@ -29,5 +41,5 @@
 use by running:
 
-  /path/to/memlog/memlog2dot /path/to/HOST.PID.memlog
+  /path/to/memlog/memlog_analyze /path/to/HOST.PID.memlog
 
 this will generate files named HOST.PID.memlog.dot, HOST.PID.memlog.ps and
@@ -36,8 +48,28 @@
 in textual form.
 
+If you pass the --leaks option to memlog_analyze, it will provide data on
+allocations active at the end of the program (leaks) instead of those active
+when the peak memory usage is first reached.
+
+You might have many runs of the same application (or output from many ranks of
+an MPI job), and you'd like to pick the one for analysis with the highest
+memory usage. If you provide a glob pattern to memlog_analyze it will do this
+for you. Make sure you quote the glob pattern so that your shell does not
+expand it.
+
+  /path/to/memlog/memlog_analyze "/path/to/*.memlog"
+
+When running under common batch systems, the files are named
+JOB_ID.HOST.PID.memlog, and when running under the BG/Q CNK, the process's rank
+is used instead of the node-local PID.
+
 Note that te peak memory usage is determined by monitoring the processes's
 maximum resident set size, not just the total allocated heap memory.
 
-memlog2dot depends on dot (from the graphviz package) and ps2pdf (from the
+memlog_analyze takes, as a second optional parameter, the name of the output
+directory (the current directory is the default). If the directory does not
+exist, it will be created.
+
+memlog_analyze depends on dot (from the graphviz package) and ps2pdf (from the
 ghostscript package), plus various tools from the binutils package.
 
Index: memlog.cpp
===================================================================
--- memlog.cpp	(revision 09f3093f1ff5e0b2766bed869fb54bcbd95c780d)
+++ memlog.cpp	(revision 2d9e75dc909b47f1f3e2e8170e5720df03470d62)
@@ -45,4 +45,5 @@
 #include <cstdio>
 #include <cstring>
+#include <cstdint>
 
 // NOTE: This source makes very minimal use of C++11 features. It can still be
@@ -52,6 +53,8 @@
 
 #include <limits.h>
+#include <errno.h>
 #include <malloc.h>
 #include <execinfo.h>
+#include <sys/mman.h>
 #include <sys/syscall.h>
 #include <sys/time.h>
@@ -66,4 +69,9 @@
 #include <dlfcn.h>
 
+#ifdef __bgq__
+#include <spi/include/kernel/location.h>
+#include <spi/include/kernel/memory.h>
+#endif
+
 using namespace std;
 
@@ -72,5 +80,5 @@
 //   -Wl,--wrap,malloc,--wrap,free,--wrap,realloc,--wrap,calloc,--wrap,memalign /path/to/memlog_s.o -lpthread -ldl
 
-FILE *log_file = NULL;
+static FILE *log_file = 0;
 static pthread_mutex_t log_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -80,4 +88,12 @@
 static char self_path[PATH_MAX+1] = { '\0' };
 
+#ifdef __bgq__
+static int on_bgq = 0;
+#endif
+
+static void *initial_brk = 0;
+
+static unordered_map<void *, Dl_info> *dladdr_cache = 0;
+
 __attribute__((__constructor__))
 static void record_init() {
@@ -85,6 +101,31 @@
   uname(&u);
 
+  int id = (int) getpid();
+#ifdef __bgq__
+  // If we're really running on a BG/Q compute node, use the job rank instead
+  // of the pid because the node name might not really be globally unique.
+  if (!strcmp(u.sysname, "CNK") && !strcmp(u.machine, "BGQ")) {
+    id = (int) Kernel_GetRank();
+    on_bgq = 1;
+  }
+#endif
+
+  // If we're running under a common batch system, add the job id to the output
+  // file names (add it as a prefix so that sorting the files will sort by job
+  // first).
+  char *job_id = 0;
+  const char *job_id_vars[] =
+    { "COBALT_JOBID", "PBS_JOBID", "SLURM_JOB_ID", "JOB_ID" };
+  for (int i = 0; i < sizeof(job_id_vars)/sizeof(job_id_vars[0]); ++i) {
+    job_id = getenv(job_id_vars[i]);
+    if (job_id)
+      break;
+  }
+
   char log_name[PATH_MAX+1];
-  snprintf(log_name, PATH_MAX+1, "%s.%d.memlog", u.nodename, getpid());
+  if (job_id)
+    snprintf(log_name, PATH_MAX+1, "%s.%s.%d.memlog", job_id, u.nodename, id);
+  else
+    snprintf(log_name, PATH_MAX+1, "%s.%d.memlog", u.nodename, id);
   log_file = fopen(log_name, "w");
   if (!log_file)
@@ -93,4 +134,6 @@
   const char *link_name = "/proc/self/exe";
   readlink(link_name, self_path, PATH_MAX);
+
+  initial_brk = sbrk(0);
 }
 
@@ -110,4 +153,7 @@
   (void) fflush(log_file);
   (void) fclose(log_file);
+
+  if (dladdr_cache)
+    delete dladdr_cache;
 }
 
@@ -115,13 +161,14 @@
 // we need to cache the lookup results.
 static int dladdr_cached(void * addr, Dl_info *info) {
-  static unordered_map<void *, Dl_info> dladdr_cache;
-
-  auto I = dladdr_cache.find(addr);
-  if (I == dladdr_cache.end()) {
+  if (!dladdr_cache)
+    dladdr_cache = new unordered_map<void *, Dl_info>;
+
+  auto I = dladdr_cache->find(addr);
+  if (I == dladdr_cache->end()) {
     int r;
     if (!(r = dladdr(addr, info)))
       memset(info, 0, sizeof(Dl_info));
 
-    dladdr_cache.insert(make_pair(addr, *info));
+    dladdr_cache->insert(make_pair(addr, *info));
     return r;
   }
@@ -140,4 +187,15 @@
   fprintf(log_file, "\t%ld.%06ld %ld %ld", usage.ru_utime.tv_sec,
           usage.ru_utime.tv_usec, usage.ru_maxrss, syscall(SYS_gettid));
+
+  // Some other memory stats (like with maxrss, report these in KB).
+  size_t arena_size = ((size_t) sbrk(0)) - (size_t) initial_brk;
+
+  uint64_t mmap_size = 0;
+#ifdef __bgq__
+  if (on_bgq)
+    (void) Kernel_GetMemorySize(KERNEL_MEMSIZE_MMAP, &mmap_size);
+#endif
+
+  fprintf(log_file, " %ld %ld", arena_size >> 10, mmap_size >> 10);
 
   if (!show_backtrace)
@@ -233,8 +291,30 @@
 }
 
+#ifdef __PIC__
+static int (*__real_posix_memalign)(void **memptr, size_t alignment,
+                                    size_t size) = 0;
+
+static void *(*__real_mmap)(void *addr, size_t length, int prot, int flags,
+                            int fd, off_t offset) = 0;
+static void *(*__real_mmap64)(void *addr, size_t length, int prot, int flags,
+                              int fd, off64_t offset) = 0;
+static int (*__real_munmap)(void *addr, size_t length) = 0;
+#else
+extern "C" {
+extern int __real_posix_memalign(void **memptr, size_t alignment, size_t size);
+
+extern void *__real_mmap(void *addr, size_t length, int prot, int flags,
+                         int fd, off_t offset);
+extern void *__real_mmap64(void *addr, size_t length, int prot, int flags,
+                           int fd, off64_t offset);
+extern int __real_munmap(void *addr, size_t length);
+}
+#endif
+
 // glibc exports its underlying malloc implementation under the name
 // __libc_malloc so that hooks like this can use it.
 extern "C" {
 extern void *__libc_malloc(size_t size);
+extern void *__libc_valloc(size_t size);
 extern void *__libc_realloc(void *ptr, size_t size);
 extern void *__libc_calloc(size_t nmemb, size_t size);
@@ -258,6 +338,23 @@
 
   void *ptr = __libc_malloc(size);
-
-  record_malloc(size, ptr, caller);
+  if (ptr)
+    record_malloc(size, ptr, caller);
+
+  in_malloc = 0;
+  return ptr;
+}
+
+void *FUNC(valloc)(size_t size) {
+  const void *caller =
+    __builtin_extract_return_addr(__builtin_return_address(0));
+
+  if (in_malloc)
+    return __libc_valloc(size);
+
+  in_malloc = 1;
+
+  void *ptr = __libc_valloc(size);
+  if (ptr)
+    record_malloc(size, ptr, caller);
 
   in_malloc = 0;
@@ -278,5 +375,6 @@
   if (ptr)
     record_free(ptr, caller);
-  record_malloc(size, nptr, caller);
+  if (nptr)
+    record_malloc(size, nptr, caller);
 
   in_malloc = 0;
@@ -296,5 +394,6 @@
   void *ptr = __libc_calloc(nmemb, size);
 
-  record_malloc(nmemb*size, ptr, caller);
+  if (ptr)
+    record_malloc(nmemb*size, ptr, caller);
 
   in_malloc = 0;
@@ -314,5 +413,6 @@
   void *ptr = __libc_memalign(boundary, size);
 
-  record_malloc(size, ptr, caller);
+  if (ptr)
+    record_malloc(size, ptr, caller);
 
   in_malloc = 0;
@@ -337,4 +437,113 @@
 }
 
+int FUNC(posix_memalign)(void **memptr, size_t alignment, size_t size) {
+  const void *caller =
+    __builtin_extract_return_addr(__builtin_return_address(0));
+
+#ifdef __PIC__
+  if (!__real_posix_memalign)
+    if (!(*(void **) (&__real_posix_memalign) =
+        dlsym(RTLD_NEXT, "posix_memalign"))) {
+      return ELIBACC;
+    }
+#endif
+
+  if (in_malloc)
+    return __real_posix_memalign(memptr, alignment, size);
+
+  in_malloc = 1;
+
+  int r = __real_posix_memalign(memptr, alignment, size);
+
+  if (!r)
+    record_malloc(size, *memptr, caller);
+
+  in_malloc = 0;
+
+  return r;
+}
+
+void *FUNC(mmap)(void *addr, size_t length, int prot, int flags,
+                 int fd, off_t offset) {
+  const void *caller =
+    __builtin_extract_return_addr(__builtin_return_address(0));
+
+#ifdef __PIC__
+  if (!__real_mmap)
+    if (!(*(void **) (&__real_mmap) = dlsym(RTLD_NEXT, "mmap"))) {
+      errno = ELIBACC;
+      return MAP_FAILED;
+    }
+#endif
+
+  if (in_malloc)
+    return __real_mmap(addr, length, prot, flags, fd, offset);
+
+  in_malloc = 1;
+
+  void *ptr = __real_mmap(addr, length, prot, flags, fd, offset);
+
+  if (ptr != MAP_FAILED)
+    record_malloc(length, ptr, caller);
+
+  in_malloc = 0;
+
+  return ptr;
+}
+
+void *FUNC(mmap64)(void *addr, size_t length, int prot, int flags,
+                   int fd, off64_t offset) {
+  const void *caller =
+    __builtin_extract_return_addr(__builtin_return_address(0));
+
+#ifdef __PIC__
+  if (!__real_mmap64)
+    if (!(*(void **) (&__real_mmap64) = dlsym(RTLD_NEXT, "mmap64"))) {
+      errno = ELIBACC;
+      return MAP_FAILED;
+    }
+#endif
+
+  if (in_malloc)
+    return __real_mmap64(addr, length, prot, flags, fd, offset);
+
+  in_malloc = 1;
+
+  void *ptr = __real_mmap64(addr, length, prot, flags, fd, offset);
+
+  if (ptr != MAP_FAILED)
+    record_malloc(length, ptr, caller);
+
+  in_malloc = 0;
+
+  return ptr;
+}
+
+int FUNC(munmap)(void *addr, size_t length) {
+  const void *caller =
+    __builtin_extract_return_addr(__builtin_return_address(0));
+
+#ifdef __PIC__
+  if (!__real_munmap)
+    if (!(*(void **) (&__real_munmap) = dlsym(RTLD_NEXT, "munmap"))) {
+      errno = ELIBACC;
+      return -1;
+    }
+#endif
+
+  if (in_malloc)
+    return __real_munmap(addr, length);
+
+  in_malloc = 1;
+
+  record_free(addr, caller);
+
+  int r = __real_munmap(addr, length);
+
+  in_malloc = 0;
+
+  return r;
+}
+
 } // extern "C"
 
Index: mlog2dot
===================================================================
--- memlog2dot	(revision 09f3093f1ff5e0b2766bed869fb54bcbd95c780d)
+++ 	(revision )
@@ -1,401 +1,0 @@
-#! /usr/bin/env perl
-
-# *****************************************************************************
-#                   Copyright (C) 2015, UChicago Argonne, LLC
-#                              All Rights Reserved
-#                            memlog (ANL-SF-15-081)
-#                    Hal Finkel, Argonne National Laboratory
-# 
-#                              OPEN SOURCE LICENSE
-# 
-# Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne, LLC,
-# the U.S. Government retains certain rights in this software.
-# 
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-# 
-# 1. Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer.
-# 
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-# 
-# 3. Neither the names of UChicago Argonne, LLC or the Department of Energy nor
-#    the names of its contributors may be used to endorse or promote products
-#    derived from this software without specific prior written permission. 
-#  
-# *****************************************************************************
-#                                  DISCLAIMER
-# 
-# THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND.
-# 
-# NEITHER THE UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF
-# ENERGY, NOR UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY
-# WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY
-# FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA,
-# APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT
-# INFRINGE PRIVATELY OWNED RIGHTS.
-# 
-# *****************************************************************************
-
-use strict;
-use File::Basename;
-
-my $memlog_fn = $ARGV[0];
-my $out_dir = $ARGV[1] || '.';
-
-my $print_raw_proc_name = 0;
-
-if (! -f $memlog_fn) {
-  print "Usage: $0 <memlog file> [<output directory>]\n";
-  exit 1;
-}
-
-# The version of addr2line and friends that you use can make a big difference,
-# especially on BE ppc64, where older versions of addr2line did not account
-# correctly for the function descriptor setup. At ALCF, on the BG/Q, we have
-# newer versions not in the default search path (from bgclang).
-my $alcf_bu_dir = '/soft/compilers/bgclang/current/binutils/bin';
-if (-d $alcf_bu_dir) {
-  $ENV{'PATH'} = $alcf_bu_dir . ':' . $ENV{'PATH'};
-}
-
-open(MEMLOG, $memlog_fn) || die "Can't open $memlog_fn: $!";
-
-# The first step is to determine the high-water mark.
-my $max_rss = 0;
-foreach my $line (<MEMLOG>) {
-  chomp($line);
-  my @parts = split(/\t/, $line);
-
-  my $op = shift(@parts);
-  my $state = shift(@parts);
-
-  my ($time, $then_max_rss, $tid) = split(/\s+/, $state);
-  if ($max_rss < $then_max_rss) {
-    $max_rss = $then_max_rss;
-  }
-}
-
-seek(MEMLOG, 0, 0);
-
-# Scan the log for malloc/free pairings. We're interested only in active
-# allocations at the time when the rss reaches the final maxrss.
-my $max_rss_time = 0;
-my %malloc_lines;
-foreach my $line (<MEMLOG>) {
-  chomp($line);
-  my @parts = split(/\t/, $line);
-
-  my $op = shift(@parts);
-  my $state = shift(@parts);
-
-  if ($op =~ /^M:/) {
-    my ($size, $ptr) = ($op =~ /^M: (\d+) 0x(\w+)/);
-    $malloc_lines{$ptr} = $line;
-  } elsif ($op =~ /^F:/) {
-    my ($ptr) = ($op =~ /^F: 0x(\w+)/);
-    delete $malloc_lines{$ptr};
-  } else {
-    next;
-  }
-
-  # If we've reached the max rss, we've seen all we need to see.
-  my ($time, $then_max_rss, $tid) = split(/\s+/, $state);
-  $max_rss_time = $time;
-  if ($then_max_rss == $max_rss) {
-    last;
-  }
-}
-
-close(MEMLOG);
-
-# Convert maxrss, currently in KB, to bytes.
-$max_rss *= 1024;
-
-my $total_size = 0;
-my %roots;
-my %all_nodes;
-foreach my $line (values %malloc_lines) {
-  my @parts = split(/\t/, $line);
-
-  my $op = shift(@parts);
-  my $state = shift(@parts);
-
-  # Only dealing with allocations here...
-  if ($op !~ /^M:/) {
-    next;
-  }
-
-  my ($size, $ptr) = ($op =~ /^M: (\d+) 0x(\w+)/);
-  my ($time, $then_max_rss, $tid) = split(/\s+/, $state);
-
-  $total_size += $size;
-
-  sub level_parts($) {
-    my $level = @_[0];
-    my ($file_name, $proc_name, $off, $pc, $relpc) =
-      ($level =~ /^(.*) \((.*)\+0x(\w+)\) \[0x(\w+) \(0x(\w+)\)\]/);
-
-    return ($file_name, $proc_name, $off, $pc, $relpc);
-  }
-
-  # Put the top of the stack first.
-  @parts = reverse(@parts);
-
-  my $parent = \%roots;
-  for (my $i = 0; $i < scalar(@parts); ++$i) {
-    my $level = $parts[$i];
-    my ($file_name, $proc_name, $off, $pc, $relpc) = level_parts($level);
-
-    # Skip this level if we don't even know from what file it came.
-    if ($file_name eq '?') {
-      next;
-    }
-
-    # print STDERR "parsed: $file_name, $proc_name, $off, $pc, $relpc\n";
-
-    if (!exists $all_nodes{$pc}) {
-      $all_nodes{$pc}->{'file_name'} = $file_name;
-      $all_nodes{$pc}->{'proc_name'} = $proc_name;
-      $all_nodes{$pc}->{'off'} = $off;
-      $all_nodes{$pc}->{'pc'} = $pc;
-      $all_nodes{$pc}->{'relpc'} = $relpc;
-    }
-
-    if (!exists $parent->{$pc}) {
-      $parent->{$pc} = $all_nodes{$pc};
-    }
-
-    $parent->{$pc}->{'size'} += $size;
-
-    my ($next_file_name, $next_proc_name, $next_off, $next_pc, $next_relpc);
-    if ($i < scalar(@parts)-1) {
-     my $next_level = $parts[$i+1];
-     ($next_file_name, $next_proc_name, $next_off, $next_pc, $next_relpc) =
-       level_parts($next_level);
-      $parent->{$pc}->{'child_sizes'}->{$next_pc} += $size;
-    }
-
-    if (!exists $parent->{'children'}) {
-      $parent->{'children'} = {};
-    }
-
-    $parent = $parent->{'children'};
-  }
-}
-
-my $txt_fn = "$out_dir/" . basename($memlog_fn) . ".txt";
-my $dot_fn = "$out_dir/" . basename($memlog_fn) . ".dot";
-my $ps_fn = "$out_dir/" . basename($memlog_fn) . ".ps";
-my $pdf_fn = "$out_dir/" . basename($memlog_fn) . ".pdf";
-
-open(TXT, ">$txt_fn") || die "Can't open $txt_fn: $!";
-open(DOT, ">$dot_fn") || die "Can't open $dot_fn: $!";
-
-sub format_bytes($) {
-  my @sizes = qw( B KB MB GB TB PB );
-  my $size = $_[0];
-
-  my $i = 0;
-  while ($size > 1024) {
-    $size /= 1024;
-    ++$i;
-  }
-
-  return sprintf("%.3f $sizes[$i]", $size);
-}
-
-print DOT ("digraph \"memlog\" {\n");
-print DOT ("size=\"8,11\";\n");
-print DOT ("node [width=0.375,height=0.25];\n");
-
-printf DOT ("Legend [shape=box, fontsize=100, shape=oval," .
-            "label=\"Total: %s active at maxrss = %s after %s s\"];\n",
-            format_bytes($total_size), format_bytes($max_rss), $max_rss_time);
-
-printf TXT ("memlog: Total: %s active at maxrss = %s after %s s\n\n",
-            format_bytes($total_size), format_bytes($max_rss), $max_rss_time);
-
-my %cached_names;
-sub get_name($) {
-  my $node = $_[0];
-  my $pc = $node->{'pc'};
-
-  if (exists $cached_names{$pc}) {
-    return $cached_names{$pc};
-  }
-
-  my $ret = '';
-
-  # Prefer the relative offset (that is what we want for shared libraries), but
-  # if is not available, use the full offset (which is what we want for the
-  # base executable).
-  my $exe_off = $node->{'relpc'};
-  if (!$exe_off) {
-    $exe_off = $pc;
-  }
-
-  my $file_name = $node->{'file_name'};
-
-  # If we don't have an absolute path, this is probably the base dynamic
-  # executable, so ask the shell which one it used (not foolproof because we
-  # might not have the same PATH now, but hopefully is generally the right
-  # thing).
-  if ($file_name !~ /^\//) {
-    $file_name = `which '$file_name'`;
-    chomp($file_name);
-  }
-
-  my ($func, $loc) = `addr2line -e '$file_name' -f 0x$exe_off`;
-  chomp($func);
-  chomp($loc);
-
-  if ($func !~ /^\?/) {
-    # In general, this function name might look something like:
-    #   00000329.plt_call.wcsnrtombs@@GLIBC_2.3+0
-    $func =~ s/@.*//; # Remove trailing symbol version strings
-    $func =~ s/.*\.//;
-    $func = `c++filt '$func'`;
-    chomp($func);
-
-    # It sometimes happens that addr2line is a bit too smart: when debugging
-    # information is available, it might print a local alias for the
-    # function instead of the full name (for example, printing 'List'
-    # instead of 'Foo::List<int>::List(int, int const&)').
-    if ($node->{'proc_name'} ne '?') {
-      my $proc_name = $node->{'proc_name'};
-      $proc_name = `c++filt '$proc_name'`;
-      chomp($proc_name);
-
-      if (length($proc_name) > length($func)) {
-        $func = $proc_name;
-      }
-    }
-
-    $ret .= $func . '\n';
-
-    if ($loc !~ /^\?/) {
-      $ret .= $loc . '\n';
-    }
-  } elsif ($node->{'proc_name'} ne '?') {
-    my $proc_name = $node->{'proc_name'};
-    $proc_name = `c++filt '$proc_name'`;
-    chomp($proc_name);
-
-    $ret .= $proc_name . '\n';
-  }
-
-  $ret .= $node->{'file_name'};
-  if ($print_raw_proc_name and $node->{'proc_name'} ne '?') {
-    $ret .= '\n' . $node->{'proc_name'} . '+0x' . $node->{'off'};
-  }
-
-  $cached_names{$pc} = $ret;
-  return $ret;
-}
-
-my $skip_frac = 0.01;
-my %skipped;
-
-foreach my $pc (keys %all_nodes) {
-  my $node = $all_nodes{$pc};
-
-  my $local_size = $node->{'size'};
-  if ($local_size * 1.0 / $total_size < $skip_frac) {
-    $skipped{$pc} = 1;
-    next;
-  }
-
-  my $fs = 8.0;
-  if ($local_size > 0) {
-    $fs = 50.0 * (abs($local_size * 1.0 / $total_size))**0.125;
-  }
-
-  my $name = get_name($node);
-
-  printf DOT ("N%s [label=\"%s\\n%s\", shape=box, fontsize=%.1f%s];\n",
-    $pc, $name, format_bytes($local_size), $fs);
-}
-
-foreach my $pc (keys %all_nodes) {
-  if ($skipped{$pc}) {
-    next;
-  }
-
-  my $node = $all_nodes{$pc};
-  my $local_size = $node->{'size'};
-
-  foreach my $cpc (keys %{$node->{'child_sizes'}}) {
-    if ($skipped{$cpc}) {
-      next;
-    }
-
-    my $child_size = $node->{'child_sizes'}->{$cpc};
-    my $frac = $child_size * 1.0 / $local_size;
-
-    my $weight = 100.0 * sqrt($frac);
-    my $style = sprintf("setlinewidth(%f)", 8.0 * sqrt($frac));
-
-    my $fs = 40.0 * $frac**0.125;
-
-    printf DOT ("N%s -> N%s [label=\"%s\", weight=%d, style=\"%s\", fontsize=%.1f];\n",
-      $pc, $cpc, format_bytes($child_size), $weight, $style, $fs);
-  }
-}
-
-print DOT ("}\n");
-
-foreach my $pc (sort { $all_nodes{$::b}->{'size'} <=>
-                       $all_nodes{$::a}->{'size'} } keys %all_nodes) {
-  if ($skipped{$pc}) {
-    next;
-  }
-
-  print TXT ('*' x 80) . "\n\n";
-
-  my $node = $all_nodes{$pc};
-
-  my $local_size = $node->{'size'};
-  printf TXT ("%s - %.1f%%\n", format_bytes($local_size),
-              $local_size * 100.0 / $total_size);
-
-  my $name = get_name($node);
-  $name =~ s/\\n/\n/g;
-
-  print TXT "$name\n$pc\n";
-
-  print TXT "\nMEMORY ALLOCATED BY CALLEES:\n";
-
-  foreach my $cpc (sort { $node->{'child_sizes'}->{$::b} <=>
-                          $node->{'child_sizes'}->{$::a} }
-                     keys %{$node->{'child_sizes'}}) {
-    if ($skipped{$cpc}) {
-      next;
-    }
-
-    my $child_node = $all_nodes{$cpc};
-    my $child_size = $node->{'child_sizes'}->{$cpc};
-
-    printf TXT ("\t%s - %.1f%%\n", format_bytes($child_size),
-                $child_size * 100.0 / $local_size);
-
-    my $child_name = get_name($child_node);
-    $child_name =~ s/\\n/\n\t/g;
-    print TXT "\t$child_name\n\t$cpc\n";
-
-    print TXT "\n";
-  }
-
-  print TXT "\n";
-}
-
-close(TXT);
-close(DOT);
-
-system("dot -Tps2 < '$dot_fn' > '$ps_fn'");
-system("ps2pdf '$ps_fn' '$pdf_fn'");
-
-exit 0;
-
Index: memlog_analyze
===================================================================
--- memlog_analyze	(revision 2d9e75dc909b47f1f3e2e8170e5720df03470d62)
+++ memlog_analyze	(revision 2d9e75dc909b47f1f3e2e8170e5720df03470d62)
@@ -0,0 +1,521 @@
+#! /usr/bin/env perl
+
+# *****************************************************************************
+#                   Copyright (C) 2015, UChicago Argonne, LLC
+#                              All Rights Reserved
+#                            memlog (ANL-SF-15-081)
+#                    Hal Finkel, Argonne National Laboratory
+# 
+#                              OPEN SOURCE LICENSE
+# 
+# Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne, LLC,
+# the U.S. Government retains certain rights in this software.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 
+# 3. Neither the names of UChicago Argonne, LLC or the Department of Energy nor
+#    the names of its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission. 
+#  
+# *****************************************************************************
+#                                  DISCLAIMER
+# 
+# THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND.
+# 
+# NEITHER THE UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF
+# ENERGY, NOR UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY
+# WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY
+# FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA,
+# APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT
+# INFRINGE PRIVATELY OWNED RIGHTS.
+# 
+# *****************************************************************************
+
+use strict;
+use File::Basename;
+use File::Path qw(make_path);
+use Getopt::Long;
+
+my $find_leaks = 0;
+my $print_raw_proc_name = 0;
+my $quiet = 0;
+my $help = 0;
+
+sub print_usage($) {
+  my $ec = $_[0];
+  my $usage = <<EOM;
+Usage: $0 [options] <memlog file or glob> [<output directory>]
+  options:
+    --leaks
+      Provide information on leaks instead of peak usage
+    --print-symbol-names
+      Include symbol names and offsets in the output
+    --quiet or -q
+      Don't print status messages while running
+EOM
+
+  print $usage;
+  exit($ec);
+}
+
+GetOptions("help|h|?" => \$help,
+           "leaks" => \$find_leaks,
+           "print-symbol-names" => \$print_raw_proc_name,
+           "quiet|q" => \$quiet)
+  or print_usage(1);
+
+if ($help) {
+  print_usage(0);
+}
+
+if (scalar(@ARGV) > 2) {
+  print_usage(2);
+}
+
+my $memlog_fn = $ARGV[0];
+my $out_dir = $ARGV[1] || '.';
+
+make_path($out_dir);
+
+if (! -f $memlog_fn) {
+  my @pot_fns = glob($memlog_fn);
+  if (scalar(@pot_fns)) {
+    if (!$quiet) {
+      print "Searching all files matching '$memlog_fn'\n";
+    }
+
+    my $pot_max_rss = 0;
+    my $pos_max_rss_fn;
+    foreach my $pot_fn (@pot_fns) {
+      my $last_line = `tail -n 1 '$pot_fn'`;
+      chomp($last_line);
+
+      my @parts = split(/\t/, $last_line);
+
+      my $op = shift(@parts);
+      my $state = shift(@parts);
+
+      my ($time, $then_max_rss, $tid, $then_arena, $then_mmap) =
+        split(/\s+/, $state);
+      if ($pot_max_rss < $then_max_rss) {
+        $pot_max_rss = $then_max_rss;
+        $pos_max_rss_fn = $pot_fn;
+      }
+    }
+
+    if (defined $pos_max_rss_fn) {
+      $memlog_fn = $pos_max_rss_fn;
+      goto have_memlog_fn;
+    }
+  }
+
+  print_usage(1);
+}
+have_memlog_fn:
+
+# The version of addr2line and friends that you use can make a big difference,
+# especially on BE ppc64, where older versions of addr2line did not account
+# correctly for the function descriptor setup. At ALCF, on the BG/Q, we have
+# newer versions not in the default search path (from bgclang).
+my $alcf_bu_dir = '/soft/compilers/bgclang/current/binutils/bin';
+if (-d $alcf_bu_dir) {
+  $ENV{'PATH'} = $alcf_bu_dir . ':' . $ENV{'PATH'};
+}
+
+open(MEMLOG, $memlog_fn) || die "Can't open $memlog_fn: $!";
+
+# The first step is to determine the high-water mark.
+my $max_rss = 0;
+my $arena = 0;
+my $mmap = 0;
+foreach my $line (<MEMLOG>) {
+  chomp($line);
+  my @parts = split(/\t/, $line);
+
+  my $op = shift(@parts);
+  my $state = shift(@parts);
+
+  my ($time, $then_max_rss, $tid, $then_arena, $then_mmap) =
+    split(/\s+/, $state);
+  if ($max_rss < $then_max_rss) {
+    $max_rss = $then_max_rss;
+    $arena = $then_arena;
+    $mmap = $then_mmap;
+  }
+}
+
+seek(MEMLOG, 0, 0);
+
+# Scan the log for malloc/free pairings. We're interested only in active
+# allocations at the time when the rss reaches the final maxrss.
+# If we're finding leaks, then go to the very end.
+my $active_alloc_time = 0;
+my %malloc_lines;
+foreach my $line (<MEMLOG>) {
+  chomp($line);
+  my @parts = split(/\t/, $line);
+
+  my $op = shift(@parts);
+  my $state = shift(@parts);
+
+  if ($op =~ /^M:/) {
+    my ($size, $ptr) = ($op =~ /^M: (\d+) 0x(\w+)/);
+    $malloc_lines{$ptr} = $line;
+  } elsif ($op =~ /^F:/) {
+    my ($ptr) = ($op =~ /^F: 0x(\w+)/);
+    delete $malloc_lines{$ptr};
+  } else {
+    next;
+  }
+
+  my ($time, $then_max_rss, $tid, $then_arena, $then_mmap) =
+    split(/\s+/, $state);
+  $active_alloc_time = $time;
+
+  if (!$find_leaks) {
+    # If we've reached the max rss, we've seen all we need to see.
+    if ($then_max_rss == $max_rss) {
+      last;
+    }
+  }
+}
+
+close(MEMLOG);
+
+# Convert maxrss, currently in KB, to bytes.
+$max_rss *= 1024;
+if (defined $arena) {
+  $arena *= 1024;
+}
+if (defined $mmap) {
+  $mmap *= 1024;
+}
+
+my $total_size = 0;
+my %roots;
+my %all_nodes;
+foreach my $line (values %malloc_lines) {
+  my @parts = split(/\t/, $line);
+
+  my $op = shift(@parts);
+  my $state = shift(@parts);
+
+  # Only dealing with allocations here...
+  if ($op !~ /^M:/) {
+    next;
+  }
+
+  my ($size, $ptr) = ($op =~ /^M: (\d+) 0x(\w+)/);
+  my ($time, $then_max_rss, $tid, $then_arena, $then_mmap) =
+    split(/\s+/, $state);
+
+  $total_size += $size;
+
+  sub level_parts($) {
+    my $level = @_[0];
+    my ($file_name, $proc_name, $off, $pc, $relpc) =
+      ($level =~ /^(.*) \((.*)\+0x(\w+)\) \[0x(\w+) \(0x(\w+)\)\]/);
+
+    return ($file_name, $proc_name, $off, $pc, $relpc);
+  }
+
+  # Put the top of the stack first.
+  @parts = reverse(@parts);
+
+  my $parent = \%roots;
+  for (my $i = 0; $i < scalar(@parts); ++$i) {
+    my $level = $parts[$i];
+    my ($file_name, $proc_name, $off, $pc, $relpc) = level_parts($level);
+
+    # Skip this level if we don't even know from what file it came.
+    if ($file_name eq '?') {
+      next;
+    }
+
+    # print STDERR "parsed: $file_name, $proc_name, $off, $pc, $relpc\n";
+
+    if (!exists $all_nodes{$pc}) {
+      $all_nodes{$pc}->{'file_name'} = $file_name;
+      $all_nodes{$pc}->{'proc_name'} = $proc_name;
+      $all_nodes{$pc}->{'off'} = $off;
+      $all_nodes{$pc}->{'pc'} = $pc;
+      $all_nodes{$pc}->{'relpc'} = $relpc;
+    }
+
+    if (!exists $parent->{$pc}) {
+      $parent->{$pc} = $all_nodes{$pc};
+    }
+
+    $parent->{$pc}->{'size'} += $size;
+
+    my ($next_file_name, $next_proc_name, $next_off, $next_pc, $next_relpc);
+    if ($i < scalar(@parts)-1) {
+     my $next_level = $parts[$i+1];
+     ($next_file_name, $next_proc_name, $next_off, $next_pc, $next_relpc) =
+       level_parts($next_level);
+      $parent->{$pc}->{'child_sizes'}->{$next_pc} += $size;
+    }
+
+    if (!exists $parent->{'children'}) {
+      $parent->{'children'} = {};
+    }
+
+    $parent = $parent->{'children'};
+  }
+}
+
+my $txt_fn = "$out_dir/" . basename($memlog_fn) . ".txt";
+my $dot_fn = "$out_dir/" . basename($memlog_fn) . ".dot";
+my $ps_fn = "$out_dir/" . basename($memlog_fn) . ".ps";
+my $pdf_fn = "$out_dir/" . basename($memlog_fn) . ".pdf";
+
+if (!$quiet) {
+  print "Creating $txt_fn\n";
+}
+
+open(TXT, ">$txt_fn") || die "Can't open $txt_fn: $!";
+
+if (!$quiet) {
+  print "Creating $dot_fn\n";
+}
+
+open(DOT, ">$dot_fn") || die "Can't open $dot_fn: $!";
+
+sub format_bytes($) {
+  my @sizes = qw( B KB MB GB TB PB );
+  my $size = $_[0];
+
+  my $i = 0;
+  while ($size > 1024) {
+    $size /= 1024;
+    ++$i;
+  }
+
+  return sprintf("%.3f $sizes[$i]", $size);
+}
+
+sub format_bytes_or_unk($) {
+  my $b = $_[0];
+  return defined($b) ? format_bytes($b) : "(unknown)";
+}
+
+print DOT ("digraph \"memlog\" {\n");
+print DOT ("size=\"8,11\";\n");
+print DOT ("node [width=0.375,height=0.25];\n");
+
+my $find_type = $find_leaks ? " (leaks)" : "";
+print DOT "subgraph cluster_key {\n";
+print DOT "\trank=min;\n";
+print DOT "\tlabel=\"memlog\";\n";
+print DOT "\tfontsize=100;\n";
+print DOT "\trankdir=UR;\n";
+printf DOT ("Legend [shape=box, fontsize=100, shape=plaintext," .
+            "label=\"Total: %s active$find_type at maxrss = %s after %s s\\narena: %s\\nmmap: %s\"];\n",
+            format_bytes($total_size), format_bytes($max_rss),
+            $active_alloc_time, format_bytes_or_unk($arena),
+            format_bytes_or_unk($mmap));
+print DOT "}\n";
+
+printf TXT ("memlog: Total: %s active$find_type at maxrss = %s after %s s\n\tarena: %s\tmmap: %s\n\n",
+            format_bytes($total_size), format_bytes($max_rss),
+            $active_alloc_time, format_bytes_or_unk($arena),
+            format_bytes_or_unk($mmap));
+
+my %cached_names;
+sub get_name($) {
+  my $node = $_[0];
+  my $pc = $node->{'pc'};
+
+  if (exists $cached_names{$pc}) {
+    return $cached_names{$pc};
+  }
+
+  my $ret = '';
+
+  # Prefer the relative offset (that is what we want for shared libraries), but
+  # if is not available, use the full offset (which is what we want for the
+  # base executable).
+  my $exe_off = $node->{'relpc'};
+  if (!$exe_off) {
+    $exe_off = $pc;
+  }
+
+  my $file_name = $node->{'file_name'};
+
+  # If we don't have an absolute path, this is probably the base dynamic
+  # executable, so ask the shell which one it used (not foolproof because we
+  # might not have the same PATH now, but hopefully is generally the right
+  # thing).
+  if ($file_name !~ /^\//) {
+    $file_name = `which '$file_name'`;
+    chomp($file_name);
+  }
+
+  my ($func, $loc) = `addr2line -e '$file_name' -f 0x$exe_off`;
+  chomp($func);
+  chomp($loc);
+
+  if ($func !~ /^\?/) {
+    # In general, this function name might look something like:
+    #   00000329.plt_call.wcsnrtombs@@GLIBC_2.3+0
+    $func =~ s/@.*//; # Remove trailing symbol version strings
+    $func =~ s/.*\.//;
+    $func = `c++filt '$func'`;
+    chomp($func);
+
+    # It sometimes happens that addr2line is a bit too smart: when debugging
+    # information is available, it might print a local alias for the
+    # function instead of the full name (for example, printing 'List'
+    # instead of 'Foo::List<int>::List(int, int const&)').
+    if ($node->{'proc_name'} ne '?') {
+      my $proc_name = $node->{'proc_name'};
+      $proc_name = `c++filt '$proc_name'`;
+      chomp($proc_name);
+
+      if (length($proc_name) > length($func)) {
+        $func = $proc_name;
+      }
+    }
+
+    $ret .= $func . '\n';
+
+    if ($loc !~ /^[:?]/) {
+      $ret .= $loc . '\n';
+    }
+  } elsif ($node->{'proc_name'} ne '?') {
+    my $proc_name = $node->{'proc_name'};
+    $proc_name = `c++filt '$proc_name'`;
+    chomp($proc_name);
+
+    $ret .= $proc_name . '\n';
+  }
+
+  $ret .= $node->{'file_name'};
+  if ($print_raw_proc_name and $node->{'proc_name'} ne '?') {
+    $ret .= '\n' . $node->{'proc_name'} . '+0x' . $node->{'off'};
+  }
+
+  $cached_names{$pc} = $ret;
+  return $ret;
+}
+
+my $skip_frac = 0.01;
+my %skipped;
+
+foreach my $pc (keys %all_nodes) {
+  my $node = $all_nodes{$pc};
+
+  my $local_size = $node->{'size'};
+  if ($local_size * 1.0 / $total_size < $skip_frac) {
+    $skipped{$pc} = 1;
+    next;
+  }
+
+  my $fs = 8.0;
+  if ($local_size > 0) {
+    $fs = 50.0 * (abs($local_size * 1.0 / $total_size))**0.125;
+  }
+
+  my $name = get_name($node);
+
+  printf DOT ("N%s [label=\"%s\\n%s\", shape=box, fontsize=%.1f%s];\n",
+    $pc, $name, format_bytes($local_size), $fs);
+}
+
+foreach my $pc (keys %all_nodes) {
+  if ($skipped{$pc}) {
+    next;
+  }
+
+  my $node = $all_nodes{$pc};
+  my $local_size = $node->{'size'};
+
+  foreach my $cpc (keys %{$node->{'child_sizes'}}) {
+    if ($skipped{$cpc}) {
+      next;
+    }
+
+    my $child_size = $node->{'child_sizes'}->{$cpc};
+    my $frac = $child_size * 1.0 / $local_size;
+
+    my $weight = 100.0 * sqrt($frac);
+    my $style = sprintf("setlinewidth(%f)", 8.0 * sqrt($frac));
+
+    my $fs = 40.0 * $frac**0.125;
+
+    printf DOT ("N%s -> N%s [label=\"%s\", weight=%d, style=\"%s\", fontsize=%.1f];\n",
+      $pc, $cpc, format_bytes($child_size), $weight, $style, $fs);
+  }
+}
+
+print DOT ("}\n");
+
+foreach my $pc (sort { $all_nodes{$::b}->{'size'} <=>
+                       $all_nodes{$::a}->{'size'} } keys %all_nodes) {
+  if ($skipped{$pc}) {
+    next;
+  }
+
+  print TXT ('*' x 80) . "\n\n";
+
+  my $node = $all_nodes{$pc};
+
+  my $local_size = $node->{'size'};
+  printf TXT ("%s - %.1f%%\n", format_bytes($local_size),
+              $local_size * 100.0 / $total_size);
+
+  my $name = get_name($node);
+  $name =~ s/\\n/\n/g;
+
+  print TXT "$name\n$pc\n";
+
+  print TXT "\nMEMORY ALLOCATED BY CALLEES:\n";
+
+  foreach my $cpc (sort { $node->{'child_sizes'}->{$::b} <=>
+                          $node->{'child_sizes'}->{$::a} }
+                     keys %{$node->{'child_sizes'}}) {
+    if ($skipped{$cpc}) {
+      next;
+    }
+
+    my $child_node = $all_nodes{$cpc};
+    my $child_size = $node->{'child_sizes'}->{$cpc};
+
+    printf TXT ("\t%s - %.1f%%\n", format_bytes($child_size),
+                $child_size * 100.0 / $local_size);
+
+    my $child_name = get_name($child_node);
+    $child_name =~ s/\\n/\n\t/g;
+    print TXT "\t$child_name\n\t$cpc\n";
+
+    print TXT "\n";
+  }
+
+  print TXT "\n";
+}
+
+close(TXT);
+close(DOT);
+
+if (!$quiet) {
+  print "Creating $ps_fn\n";
+}
+
+system("dot -Tps2 < '$dot_fn' > '$ps_fn'");
+
+if (!$quiet) {
+  print "Creating $pdf_fn\n";
+}
+
+system("ps2pdf '$ps_fn' '$pdf_fn'");
+
+exit 0;
+
