threads: deadlock occured on creation of a thread while joining others

[p5pRT]

Migrated from rt.perl.org#40382 (status was 'open')

Searchable as RT40382$

[p5pRT]

From t-ikegami@aist.go.jp

Created by ikegami@p08svc000.asc.hpcc.jp

There seems to be a potential deadlock situation on creating a new
ithread while destructing (or joining) others. The attached sample
code occasionally freezed on dual Opteron machine running under SuSE
Linux 9.3 / perl5.8.8. Stack traces obtained by gdb are appended.

Perl5.9.4 is much robust, but it rarely failed with SEGV when the
constant N was increased to 10. Investigation with gdb indicated that
the SEGV happened in a similar situation, that is, one ithread is
under destruction and another under creation.

  IKEGAMI, Tsutomu
  AIST
  t-ikegami@​aist.go.jp

----- 8< ---------- 8< ----- snip, snip ----- 8< ---------- 8< -----
#!/usr/bin/perl
#
# Perl thread creation stalles on collision with perl thread
# destruction.
#
use strict;
use warnings;
use threads;
use threads​::shared;
use constant N => 1;

my $flag : shared = 1;

my (@​b, @​d);

for (1..N) {
  push @​b, async {
  lock $flag;
  while($flag) { cond_wait($flag); }
  };
}

for (1..N) {
  push @​d, async {
  lock $flag;
  while($flag) { cond_wait($flag); }
  async { }->detach;
  };
}

{
  lock $flag;
  $flag = 0;
  cond_broadcast($flag);
}
$_->join for @​b;
$_->join for @​d;

----- >8 ---------- >8 ----- snip, snip ----- >8 ---------- >8 -----
(gdb) info thread
  2 Thread 1077942624 (LWP 10954) 0x0000002a95c25fdb in __lll_mutex_lock_wait ()
  from /lib64/tls/libpthread.so.0
  1 Thread 182904516864 (LWP 10952) 0x0000002a95c25fdb in __lll_mutex_lock_wait ()
  from /lib64/tls/libpthread.so.0
(gdb) thread 1
[Switching to thread 1 (Thread 182904516864 (LWP 10952))]#0 0x0000002a95c25fdb in __lll_mutex_lock_wait ()
  from /lib64/tls/libpthread.so.0
(gdb) where
#0 0x0000002a95c25fdb in __lll_mutex_lock_wait () from /lib64/tls/libpthread.so.0
#1 0x000000000074e5f0 in ?? ()
#2 0xffffffffffffffff in ?? ()
#3 0x0000002a95c22eec in pthread_mutex_lock () from /lib64/tls/libpthread.so.0
#4 0x0000000000763450 in ?? ()
#5 0x0000000000000004 in ?? ()
#6 0x0000000000000000 in ?? ()
#7 0x000000000074e5f0 in ?? ()
#8 0x0000000000763430 in ?? ()
#9 0x000000000074e5f0 in ?? ()
#10 0x00000000004dbe7b in Perl_PerlIO_flush (my_perl=0x62d010, f=<value optimized out>) at perlio.c​:1616
#11 0x00000000004dbed4 in Perl_PerlIO_flush (my_perl=0x74e5f0, f=0x763450) at perlio.c​:1640
#12 0x000000000074e5f0 in ?? ()
#13 0x00000000006d5230 in ?? ()
#14 0x000000000041eced in perl_destruct (my_perl=0x74e5f0) at perl.c​:651
#15 0x0000002a95f7c55e in S_ithread_clear (my_perl=0x62d010, thread=0x65b590) at threads.xs​:117
#16 0x0000002a95f7d83e in Perl_ithread_join (my_perl=0x62d010, obj=<value optimized out>) at threads.xs​:663
#17 0x0000002a95f7df2d in XS_threads_join (my_perl=0x62d010, cv=<value optimized out>) at threads.xs​:739
#18 0x000000000047e3ad in Perl_pp_entersub (my_perl=0x62d010) at pp_hot.c​:2877
#19 0x0000000000464b4a in Perl_runops_debug (my_perl=0x62d010) at dump.c​:1459
#20 0x00000000004217e2 in S_run_body (my_perl=0x62d010, oldscope=1) at perl.c​:2366
#21 0x000000000042139c in perl_run (my_perl=0x62d010) at perl.c​:2283
#22 0x000000000041d701 in main (argc=2, argv=0x7fbfffee78, env=<value optimized out>) at perlmain.c​:99
(gdb) thread 2
[Switching to thread 2 (Thread 1077942624 (LWP 10954))]#0 0x0000002a95c25fdb in __lll_mutex_lock_wait ()
  from /lib64/tls/libpthread.so.0
(gdb) where
#0 0x0000002a95c25fdb in __lll_mutex_lock_wait () from /lib64/tls/libpthread.so.0
#1 0x0000000040400c00 in ?? ()
#2 0x0000000000000041 in ?? ()
#3 0x0000002a95c22eec in pthread_mutex_lock () from /lib64/tls/libpthread.so.0
#4 0x0000000000000041 in ?? ()
#5 0x0000000000000045 in ?? ()
#6 0x0000002a95f51d80 in __malloc_initialize_hook () from /lib64/tls/libc.so.6
#7 0x00000000008bc600 in ?? ()
#8 0x0000000000000030 in ?? ()
#9 0x0000000000895130 in ?? ()
#10 0x0000000040400c00 in ?? ()
#11 0x000000000097c398 in ?? ()
#12 0x0000002a95d9a179 in malloc () from /lib64/tls/libc.so.6
#13 0x0000000000000000 in ?? ()
#14 0x00000000008bc600 in ?? ()
#15 0x00000000008c2ff0 in ?? ()
#16 0x000000000097c3c0 in ?? ()
#17 0x0000002a95f7ca12 in ithread_mg_dup (my_perl=<value optimized out>, mg=<value optimized out>,
  param=<value optimized out>) at threads.xs​:253
#18 0x000000000048f425 in Perl_mg_dup (my_perl=0x895130, mg=0x30, param=0x40400c00) at sv.c​:9795
#19 0x000000000048fd66 in Perl_sv_dup (my_perl=0x895130, sstr=0x8bc600, param=0x40400c00) at sv.c​:10129
#20 0x000000000048f8a0 in Perl_rvpv_dup (my_perl=<value optimized out>, dstr=0x970ea0, sstr=0x8bc5f0,
  param=<value optimized out>) at sv.c​:9995
#21 0x000000000048fd9e in Perl_sv_dup (my_perl=0x895130, sstr=0x8bc5f0, param=0x40400c00) at sv.c​:10131
#22 0x00000000004906b1 in Perl_sv_dup (my_perl=0x895130, sstr=0x8bc5e0, param=0x40400c00) at sv.c​:10247
#23 0x00000000004906b1 in Perl_sv_dup (my_perl=0x895130, sstr=0x8bc5b0, param=0x40400c00) at sv.c​:10247
#24 0x00000000004906b1 in Perl_sv_dup (my_perl=0x895130, sstr=0x8bc540, param=0x40400c00) at sv.c​:10247
#25 0x0000000000490bc8 in Perl_sv_dup (my_perl=0x895130, sstr=0x8bc530, param=0x40400c00) at sv.c​:10334
#26 0x0000000000492789 in perl_clone (proto_perl=0x7dea50, flags=2) at sv.c​:11199
#27 0x0000002a95f7d2f9 in Perl_ithread_create (my_perl=0x7dea50, obj=0x0, classname=0x740520 "threads",
  init_function=0x8bc8a0, params=0x8bcf00) at threads.xs​:438
#28 0x0000002a95f7da06 in XS_threads_new (my_perl=0x7dea50, cv=<value optimized out>) at threads.xs​:698
#29 0x00000000004ab9ff in Perl_pp_goto (my_perl=0x7dea50) at pp_ctl.c​:2292
#30 0x0000000000464b4a in Perl_runops_debug (my_perl=0x7dea50) at dump.c​:1459
#31 0x000000000042242c in S_call_body (my_perl=0x7dea50, myop=0x0, is_eval=2 '\002') at perl.c​:2731
#32 0x0000000000422083 in Perl_call_sv (my_perl=0x7dea50, sv=<value optimized out>,
  flags=<value optimized out>) at perl.c​:2646
#33 0x0000002a95f7cc29 in Perl_ithread_run (arg=<value optimized out>) at threads.xs​:312
#34 0x0000002a95c21aff in start_thread () from /lib64/tls/libpthread.so.0
#35 0x0000002a95de94b3 in clone () from /lib64/tls/libc.so.6
#36 0x0000000000000000 in ?? ()
#37 0x0000000000000000 in ?? ()
#38 0x0000000000000000 in ?? ()
  :
  :

----- >8 ---------- >8 ----- snip, snip ----- >8 ---------- >8 -----

Perl Info

Flags:
    category=core
    severity=low

Site configuration information for perl v5.8.8:

Configured by ikegami at Thu Sep 21 15:06:47 JST 2006.

Summary of my perl5 (revision 5 version 8 subversion 8) configuration:
  Platform:
    osname=linux, osvers=2.6.5-7.257-smp, archname=x86_64-linux-thread-multi
    uname='linux p08svc000.asc.hpcc.jp 2.6.5-7.257-smp #1 smp mon may 15 23:14:14 jst 2006 x86_64 x86_64 x86_64 gnulinux '
    config_args='-ds -e -Dprefix=/work/perl-5.8.8 -Dusethreads -Doptimize=-O -g'
    hint=recommended, useposix=true, d_sigaction=define
    usethreads=define use5005threads=undef useithreads=define usemultiplicity=define
    useperlio=define d_sfio=undef uselargefiles=define usesocks=undef
    use64bitint=define use64bitall=define uselongdouble=undef
    usemymalloc=n, bincompat5005=undef
  Compiler:
    cc='cc', ccflags ='-D_REENTRANT -D_GNU_SOURCE -DTHREADS_HAVE_PIDS -DDEBUGGING -fno-strict-aliasing -pipe -I/usr/local/include -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64',
    optimize='-O -g',
    cppflags='-D_REENTRANT -D_GNU_SOURCE -DTHREADS_HAVE_PIDS -DDEBUGGING -fno-strict-aliasing -pipe -I/usr/local/include'
    ccversion='', gccversion='3.3.3 (SuSE Linux)', gccosandvers=''
    intsize=4, longsize=8, ptrsize=8, doublesize=8, byteorder=12345678
    d_longlong=define, longlongsize=8, d_longdbl=define, longdblsize=16
    ivtype='long', ivsize=8, nvtype='double', nvsize=8, Off_t='off_t', lseeksize=8
    alignbytes=8, prototype=define
  Linker and Libraries:
    ld='cc', ldflags =' -L/usr/local/lib'
    libpth=/usr/local/lib /lib /usr/lib
    libs=-lnsl -lgdbm -ldl -lm -lcrypt -lutil -lpthread -lc
    perllibs=-lnsl -ldl -lm -lcrypt -lutil -lpthread -lc
    libc=, so=so, useshrplib=false, libperl=libperl.a
    gnulibc_version='2.3.5'
  Dynamic Linking:
    dlsrc=dl_dlopen.xs, dlext=so, d_dlsymun=undef, ccdlflags='-Wl,-E'
    cccdlflags='-fpic', lddlflags='-shared -L/usr/local/lib'

Locally applied patches:
    


@INC for perl v5.8.8:
    /work/perl-5.8.8/lib/5.8.8/x86_64-linux-thread-multi
    /work/perl-5.8.8/lib/5.8.8
    /work/perl-5.8.8/lib/site_perl/5.8.8/x86_64-linux-thread-multi
    /work/perl-5.8.8/lib/site_perl/5.8.8
    /work/perl-5.8.8/lib/site_perl
    .


Environment for perl v5.8.8:
    HOME=/home/ikegami
    LANG=ja_JP.ujis
    LANGUAGE (unset)
    LD_LIBRARY_PATH=/opt/intel/fce/9.1.033/lib:/opt/intel/cce/9.1.039/lib:/opt/gt4/lib:/home/ikegami/local/lib
    LOGDIR (unset)
    PATH=/home/ikegami/bin:/home/ikegami/util:/home/ikegami/local/bin:/home/ikegami/local/perl/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/X11R6/bin:/opt/MPICH/psc/bin:/home/ikegami/local/ninfg-2.4.1/bin:/usr/pbs/bin:/opt/intel/fce/9.1.033/bin:/opt/intel/idbe/9.1.039/bin:/opt/intel/cce/9.1.039/bin:/opt/gt4/bin:/opt/gt4/sbin:/usr/games:/opt/gnome/bin:/opt/mpi/bin:/opt/pathscale/bin:/opt/pgi/linux86-64/6.0/bin:/opt/pgi/linux86/6.0/bin:.
    PERL_BADLANG (unset)
    SHELL=/bin/bash

[p5pRT]

From jdhedden@yahoo.com

Tsutomu IKEGAMI reported​:

There seems to be a potential deadlock situation on creating a new
ithread while destructing (or joining) others. The attached sample
code occasionally freezed on dual Opteron machine running under SuSE
Linux 9.3 / perl5.8.8. Stack traces obtained by gdb are appended.

I was not able to reproduce the problem using ActivePerl 5.8.8
(original threads version and 1.42), or Perl 5.8.8 under Cygwin or
Solaris (threads 1.42).

It would be helpful to know if you can repeat this problem using the
lastest versions of 'threads' (v1.42) and 'threads​::shared' (v1.03)
from CPAN. threads 1.41 contained changes that tried to eliminate
deadlock/race conditions.

If deadlock occurs with 1.42, gdb stack traces would be helpful.
Thanks.

Perl5.9.4 is much robust, but it rarely failed with SEGV when the
constant N was increased to 10. Investigation with gdb indicated
that
the SEGV happened in a similar situation, that is, one ithread is
under destruction and another under creation.

I was able to use N=50 under ActivePerl, N=350 (!) under Cyginw
and N=40 on Solaris. Again, with no deadlocks.

__________________________________________________
Do You Yahoo!?
Tired of spam? Yahoo! Mail has the best spam protection around
http​://mail.yahoo.com

[p5pRT]

The RT System itself - Status changed from 'new' to 'open'

[p5pRT]

From @Hugmeir

On Fri Sep 22 08​:20​:59 2006, jdhedden@​yahoo.com wrote​:

Tsutomu IKEGAMI reported​:

There seems to be a potential deadlock situation on creating a new
ithread while destructing (or joining) others. The attached sample
code occasionally freezed on dual Opteron machine running under SuSE
Linux 9.3 / perl5.8.8. Stack traces obtained by gdb are appended.

I was not able to reproduce the problem using ActivePerl 5.8.8
(original threads version and 1.42), or Perl 5.8.8 under Cygwin or
Solaris (threads 1.42).

It would be helpful to know if you can repeat this problem using the
lastest versions of 'threads' (v1.42) and 'threads​::shared' (v1.03)
from CPAN. threads 1.41 contained changes that tried to eliminate
deadlock/race conditions.

If deadlock occurs with 1.42, gdb stack traces would be helpful.
Thanks.

Perl5.9.4 is much robust, but it rarely failed with SEGV when the
constant N was increased to 10. Investigation with gdb indicated
that
the SEGV happened in a similar situation, that is, one ithread is
under destruction and another under creation.

I was able to use N=50 under ActivePerl, N=350 (!) under Cyginw
and N=40 on Solaris. Again, with no deadlocks.

__________________________________________________
Do You Yahoo!?
Tired of spam? Yahoo! Mail has the best spam protection around
http​://mail.yahoo.com

I can reproduce the deadlock on 5.8.8, but not on anything newer; I even
tried upping N to 1000, and nothing. Should this ticket be closed?

--hugmeir

[p5pRT]

From @jkeenan

On Fri May 25 16​:36​:02 2012, Hugmeir wrote​:

On Fri Sep 22 08​:20​:59 2006, jdhedden@​yahoo.com wrote​:

Tsutomu IKEGAMI reported​:

There seems to be a potential deadlock situation on creating a new
ithread while destructing (or joining) others. The attached sample
code occasionally freezed on dual Opteron machine running under SuSE
Linux 9.3 / perl5.8.8. Stack traces obtained by gdb are appended.

I was not able to reproduce the problem using ActivePerl 5.8.8
(original threads version and 1.42), or Perl 5.8.8 under Cygwin or
Solaris (threads 1.42).

It would be helpful to know if you can repeat this problem using the
lastest versions of 'threads' (v1.42) and 'threads​::shared' (v1.03)
from CPAN. threads 1.41 contained changes that tried to eliminate
deadlock/race conditions.

If deadlock occurs with 1.42, gdb stack traces would be helpful.
Thanks.

Perl5.9.4 is much robust, but it rarely failed with SEGV when the
constant N was increased to 10. Investigation with gdb indicated
that
the SEGV happened in a similar situation, that is, one ithread is
under destruction and another under creation.

I was able to use N=50 under ActivePerl, N=350 (!) under Cyginw
and N=40 on Solaris. Again, with no deadlocks.

__________________________________________________
Do You Yahoo!?
Tired of spam? Yahoo! Mail has the best spam protection around
http​://mail.yahoo.com

I can reproduce the deadlock on 5.8.8, but not on anything newer; I even
tried upping N to 1000, and nothing. Should this ticket be closed?

--hugmeir

When I built a threaded perl on the Dromedary server and then tried out
the OP's program with various values of 'N', I got no errors up to 510.
But at N => 511, I got the following error​:

#########
$ ./perl -Ilib ../p5p/40382-threads.pl
Thread creation failed​: pthread_create returned 11 at
../p5p/40382-threads.pl line 23.
Can't call method "join" on an undefined value at
../p5p/40382-threads.pl line 32.
##########

[p5pRT]

From @jkeenan

$ ./perl -Ilib -V
Summary of my perl5 (revision 5 version 19 subversion 4) configuration​:
  Commit id​: cb4b14d
  Platform​:
  osname=linux, osvers=2.6.32-358.el6.x86_64, archname=x86_64-linux-thread-multi
  uname='linux dromedary-001.ams6.corp.booking.com 2.6.32-358.el6.x86_64 #1 smp fri feb 22 00​:31​:26 utc 2013 x86_64 x86_64 x86_64 gnulinux '
  config_args='-des -Dusedevel -Dusethreads'
  hint=recommended, useposix=true, d_sigaction=define
  useithreads=define, usemultiplicity=define
  useperlio=define, d_sfio=undef, uselargefiles=define, usesocks=undef
  use64bitint=define, use64bitall=define, uselongdouble=undef
  usemymalloc=n, bincompat5005=undef
  Compiler​:
  cc='cc', ccflags ='-D_REENTRANT -D_GNU_SOURCE -fno-strict-aliasing -pipe -fstack-protector -I/usr/local/include -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64',
  optimize='-O2',
  cppflags='-D_REENTRANT -D_GNU_SOURCE -fno-strict-aliasing -pipe -fstack-protector -I/usr/local/include'
  ccversion='', gccversion='4.4.7 20120313 (Red Hat 4.4.7-3)', gccosandvers=''
  intsize=4, longsize=8, ptrsize=8, doublesize=8, byteorder=12345678
  d_longlong=define, longlongsize=8, d_longdbl=define, longdblsize=16
  ivtype='long', ivsize=8, nvtype='double', nvsize=8, Off_t='off_t', lseeksize=8
  alignbytes=8, prototype=define
  Linker and Libraries​:
  ld='cc', ldflags =' -fstack-protector -L/usr/local/lib'
  libpth=/usr/local/lib /lib/../lib64 /usr/lib/../lib64 /lib /usr/lib /lib64 /usr/lib64 /usr/local/lib64
  libs=-lnsl -ldl -lm -lcrypt -lutil -lpthread -lc
  perllibs=-lnsl -ldl -lm -lcrypt -lutil -lpthread -lc
  libc=/lib/libc-2.12.so, so=so, useshrplib=false, libperl=libperl.a
  gnulibc_version='2.12'
  Dynamic Linking​:
  dlsrc=dl_dlopen.xs, dlext=so, d_dlsymun=undef, ccdlflags='-Wl,-E'
  cccdlflags='-fPIC', lddlflags='-shared -O2 -L/usr/local/lib -fstack-protector'

Characteristics of this binary (from libperl)​:
  Compile-time options​: HAS_TIMES MULTIPLICITY PERLIO_LAYERS
  PERL_DONT_CREATE_GVSV
  PERL_HASH_FUNC_ONE_AT_A_TIME_HARD
  PERL_IMPLICIT_CONTEXT PERL_MALLOC_WRAP
  PERL_NEW_COPY_ON_WRITE PERL_PRESERVE_IVUV
  PERL_USE_DEVEL USE_64_BIT_ALL USE_64_BIT_INT
  USE_ITHREADS USE_LARGE_FILES USE_LOCALE
  USE_LOCALE_COLLATE USE_LOCALE_CTYPE
  USE_LOCALE_NUMERIC USE_PERLIO USE_PERL_ATOF
  USE_REENTRANT_API
  Built under linux
  Compiled at Sep 19 2013 01​:23​:08
  %ENV​:
  PERLBREW_BASHRC_VERSION="0.63"
  PERLBREW_HOME="/home/jkeenan/.perlbrew"
  PERLBREW_ROOT="/home/jkeenan/perl5/perlbrew"
  @​INC​:
  lib
  /usr/local/lib/perl5/site_perl/5.19.4/x86_64-linux-thread-multi
  /usr/local/lib/perl5/site_perl/5.19.4
  /usr/local/lib/perl5/5.19.4/x86_64-linux-thread-multi
  /usr/local/lib/perl5/5.19.4
  /usr/local/lib/perl5/site_perl
  .

[p5pRT]

From @jkeenan

40382-threads.pl