Re: [Bug-tar] memory exhausted

[John Hein]

Dr. Jürgen Vollmer wrote at 15:50 +0200 on Oct 17, 2008:
 > Am Freitag 17 Oktober 2008 15:40:11 schrieb Sergey Poznyakoff:
 > > Hi Jürgen,
 > >
 > > > Not I'm not able to display (list) the tape contants, since tar aborts
 > > > with the message:
 > > >
 > > >  /usr/local/bin/tar: memory exhausted
 > > >  /usr/local/bin/tar: Error is not recoverable: exiting now
 > >
 > > Did it display any partial listing before that, or it failed right after
 > > startup?
 > 
 > the written archive contains ca. 1.900.000 files.
 > tar aborted afzer listing of ca. 1.700.000 files.

This happens for me, too, when I used --listed-incremental on
a large directory (~400 GB).  Here is how tar is invoked...

gtar --create --file /dev/null --directory /foo --listed-incremental 
/tmp/tarlist .

You can make this happen quicker if you set the data segment size
limit lower than the normal default.
It dies when tar's memory usage (as reported by ps) reaches the
'datasize' limit.

FWIW, this happens with gtar-1.15.1 & gtar-1.18, too.

I ran gtar in gdb and set a breakpoint in xalloc_die.  Here's the
backtrace when it reached the breakpoint.  I believe the large size in
frame #2 is a symptom, not the cause (due to issues when datasize is
exceeded).

(gdb) bt full
#0  xalloc_die () at extract.c:1377
No locals.
#1  0x0807f011 in xmalloc ()
No symbol table info available.
#2  0x08055dcf in make_directory (
    name=0x1d488800 "./foo/bar/bar/bar/bar/bar/bar/bar/bar/bar/")
    at incremen.c:251
        namelen = 63
        size = 673749152
        directory = (struct directory *) 0x2808bf9c
#3  0x08055f52 in find_directory (name=0x0) at incremen.c:312
        dir = (struct directory *) 0xbfbfe1b0
        ret = (struct directory *) 0x2808bf9c
#4  0x08056064 in procdir (
    name_buffer=0x1d488800 "./foo/bar/bar/bar/bar/bar/bar/bar/bar/bar/", 
stat_data=0xbfbfe1b0, device=0, children=NO_CHILDREN, verbose=false, entry=0x0)
    at incremen.c:367
        directory = (struct directory *) 0x2808bf9c
        nfs = false
#5  0x0805667f in get_directory_contents (
    dir=0x2808bf9c "./foo/bar/bar/bar/bar/bar/bar/bar/bar/bar",
    device=100) at incremen.c:609
No locals.
#6  0x0805be6f in add_hierarchy_to_namelist (name=0x2808bf80, device=100) at 
names.c:737
        file_name = 0x2808bf9c "./foo/bar/bar/bar/bar/bar/bar/bar/bar/bar"
        buffer = 0x4 <Error reading address 0x4: Bad address>
#7  0x0805bf4a in add_hierarchy_to_namelist (name=0x28083198, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 59
        namebuf = 0x28087c00 "./foo/bar/bar/bar/bar/bar/bar/bar/bar/bar"
        string = 0x28083198 "Dbar"
        string_length = 4
        file_name = 0x2808299c "./foo/bar/bar/bar/bar/bar/bar/bar/bar"
        buffer = 0x4 <Error reading address 0x4: Bad address>
#8  0x0805bf4a in add_hierarchy_to_namelist (name=0x27dd4ce2, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 54
        namebuf = 0x27e7f480 "./foo/bar/bar/bar/bar/bar/bar/bar/bar/bar"
        string = 0x27dd4ce2 "Dbar"
        string_length = 5
        file_name = 0x27da921c "./foo/bar/bar/bar/bar/bar/bar/bar/bar"
        buffer = 0x5 <Error reading address 0x5: Bad address>
#9  0x0805bf4a in add_hierarchy_to_namelist (name=0x27d9f4a3, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 50
        namebuf = 0x27da4c80 "./foo/bar/bar/bar/bar/bar/bar/bar/bar"
        string = 0x27d9f4a3 "Dbar"
        string_length = 4
        file_name = 0x27c2399c "./foo/bar/bar/bar/bar/bar/bar/bar"
        buffer = 0x4 <Error reading address 0x4: Bad address>
#10 0x0805bf4a in add_hierarchy_to_namelist (name=0x27c1f715, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 46
        namebuf = 0x27c23780 "./foo/bar/bar/bar/bar/bar/bar"
        string = 0x27c1f715 "Dbar"
        string_length = 4
        file_name = 0x27c2361c "./foo/bar/bar/bar/bar/bar"
        buffer = 0x4 <Error reading address 0x4: Bad address>
#11 0x0805bf4a in add_hierarchy_to_namelist (name=0x27c21e1f, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 39
        namebuf = 0x27c23500 "./foo/bar/bar/bar/bar/bar"
        string = 0x27c21e1f "Dbar"
        string_length = 7
        file_name = 0x27c2169c "./foo/bar/bar/bar/bar"
        buffer = 0x7 <Error reading address 0x7: Bad address>
#12 0x0805bf4a in add_hierarchy_to_namelist (name=0x25f7ae76, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 31
        namebuf = 0x25fafb00 "./foo/bar/bar/bar/bar"
        string = 0x25f7ae76 "Dbar"
        string_length = 8
        file_name = 0x25fa9c5c "./foo/bar/bar/bar"
        buffer = 0x8 <Error reading address 0x8: Bad address>
#13 0x0805bf4a in add_hierarchy_to_namelist (name=0x25fa2e70, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 23
        namebuf = 0x253c4280 "./foo/bar/bar/bar"
        string = 0x25fa2e70 "Dbar"
        string_length = 8
        file_name = 0x25fa9c1c "./foo/bar/bar"
        buffer = 0x8 <Error reading address 0x8: Bad address>
#14 0x0805bf4a in add_hierarchy_to_namelist (name=0x253c3036, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 17
        namebuf = 0x95ab080 "./foo/bar/bar"
        string = 0x253c3036 "Dbar"
        string_length = 6
        file_name = 0x24b5845c "./foo/bar"
        buffer = 0x6 <Error reading address 0x6: Bad address>
#15 0x0805bf4a in add_hierarchy_to_namelist (name=0x958b239, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 6
        namebuf = 0x8094180 "./foo/bar"
        string = 0x958b239 "Dbar"
        string_length = 11
        file_name = 0x95ac31c "./foo"
        buffer = 0xb <Error reading address 0xb: Bad address>
#16 0x0805bf4a in add_hierarchy_to_namelist (name=0x8091162, device=100) at 
names.c:782
        np = (struct name *) 0x0
        allocated_length = 100
        change_dir = 1
        name_length = 2
        namebuf = 0x8094100 "./foo"
        string = 0x8091162 "Dfoo"
        string_length = 4
        file_name = 0x808f0dc "."
        buffer = 0x4 <Error reading address 0x4: Bad address>
#17 0x0805c588 in collect_and_sort_names () at names.c:831
        name = (struct name *) 0x808f0c0
        next_name = (struct name *) 0x0
        num_names = 0
        statbuf = {st_dev = 100, st_ino = 2, st_mode = 16877, st_nlink = 6, 
st_uid = 0,
  st_gid = 0, st_rdev = 3040, st_atimespec = {tv_sec = 1225036680, tv_nsec = 0},
  st_mtimespec = {tv_sec = 1208447795, tv_nsec = 0}, st_ctimespec = {
    tv_sec = 1208447795, tv_nsec = 0}, st_size = 512, st_blocks = 4, st_blksize 
= 4096,
  st_flags = 0, st_gen = 578958771, st_lspare = 0, st_birthtimespec = {
    tv_sec = 1202319701, tv_nsec = 0}}
#18 0x080513c5 in create_archive () at create.c:1284
        buffer_size = 1000
        buffer = 0x8093000 "8"
        q = 0x0
        p = 0x0
#19 0x0806268b in main (argc=0, argv=0x0) at tar.c:2445
No locals.