Flaky crash on iso-stress-linux bot

[alexmarkov]

[/b/s/w/ir/cache/builder/sdk/runtime/tests/concurrency/../../../tests/standalone_2/io/io_sink_test.dart] starting ...

===== CRASH =====
si_signo=Segmentation fault(11), si_code=1, si_addr=0x23
version=2.14.0-edge.846af57d996d5d07758043772f4445b42ecc1ddd (be) (Thu Aug 5 17:41:30 2021 +0000) on "linux_x64"
pid=9063, thread=9078, isolate_group=main(0x55dee9b51c00), isolate=wrapper3215(0x55deed300000)
isolate_instructions=55dee3c531e0, vm_instructions=55dee3c531e0
  pc 0x000055dee3f2eec7 fp 0x00007f727227d460 dart::StackTraceUtils::UnwindAwaiterChain(dart::Zone*, dart::GrowableObjectArray const&, dart::GrowableArray<unsigned long>*, dart::CallerClosureFinder*, dart::Closure const&)+0x127
  pc 0x000055dee3f2f256 fp 0x00007f727227d670 dart::StackTraceUtils::CollectFramesLazy(dart::Thread*, dart::GrowableObjectArray const&, dart::GrowableArray<unsigned long>*, int, std::__2::function<void (dart::StackFrame*)>*, bool*)+0x1e6
  pc 0x000055dee3d7a2a6 fp 0x00007f727227d6d0 out/ReleaseX64/dart+0x1e192a6
  pc 0x000055dee3e2f0e4 fp 0x00007f727227d750 dart::NativeEntry::BootstrapNativeCallWrapper(_Dart_NativeArguments*, void (*)(_Dart_NativeArguments*))+0xb4
  pc 0x00007f7278e018e4 fp 0x00007f727227d790 Unknown symbol
  pc 0x00007f72713100b0 fp 0x00007f727227d7c0 Unknown symbol
  pc 0x00007f720f9b6b18 fp 0x00007f727227d808 Unknown symbol
  pc 0x00007f720f9f44c3 fp 0x00007f727227d8a8 Unknown symbol
  pc 0x00007f720f9d95f1 fp 0x00007f727227d920 Unknown symbol
  pc 0x00007f720f9fdfb0 fp 0x00007f727227d978 Unknown symbol
  pc 0x00007f7214d4135d fp 0x00007f727227da00 Unknown symbol
  pc 0x00007f725282ab04 fp 0x00007f727227da48 Unknown symbol
  pc 0x00007f7277e69aac fp 0x00007f727227dac0 Unknown symbol
  pc 0x00007f7277e6988c fp 0x00007f727227db28 Unknown symbol
  pc 0x00007f7277e68ad9 fp 0x00007f727227dbc0 Unknown symbol
  pc 0x00007f7277e3f2ee fp 0x00007f727227dc10 Unknown symbol
  pc 0x00007f7277e3f00e fp 0x00007f727227dc70 Unknown symbol
  pc 0x00007f7277e3e2af fp 0x00007f727227dcf0 Unknown symbol
  pc 0x00007f7277e3d003 fp 0x00007f727227dd38 Unknown symbol
  pc 0x00007f7261549bbd fp 0x00007f727227dd80 Unknown symbol
  pc 0x00007f7214d2abcd fp 0x00007f727227ddd0 Unknown symbol
  pc 0x00007f7277e480b9 fp 0x00007f727227de40 Unknown symbol
  pc 0x00007f7277e47f59 fp 0x00007f727227dea0 Unknown symbol
  pc 0x00007f7277e47856 fp 0x00007f727227df30 Unknown symbol
  pc 0x00007f7277e66f5f fp 0x00007f727227dfa0 Unknown symbol
  pc 0x00007f727051b6f3 fp 0x00007f727227dfe0 Unknown symbol
  pc 0x00007f7270519652 fp 0x00007f727227e030 Unknown symbol
  pc 0x00007f726ed8a99c fp 0x00007f727227e068 Unknown symbol
  pc 0x00007f7249fb87f1 fp 0x00007f727227e0a0 Unknown symbol
  pc 0x00007f7249fc66e7 fp 0x00007f727227e0e0 Unknown symbol
  pc 0x00007f72476b144a fp 0x00007f727227e118 Unknown symbol
  pc 0x00007f7277e480b9 fp 0x00007f727227e188 Unknown symbol
  pc 0x00007f7277e47f59 fp 0x00007f727227e1e8 Unknown symbol
  pc 0x00007f7277e47856 fp 0x00007f727227e278 Unknown symbol
  pc 0x00007f7277e66f5f fp 0x00007f727227e2e8 Unknown symbol
  pc 0x00007f727051b6f3 fp 0x00007f727227e328 Unknown symbol
  pc 0x00007f7270519652 fp 0x00007f727227e378 Unknown symbol
  pc 0x00007f72705180df fp 0x00007f727227e3b0 Unknown symbol
  pc 0x00007f727137bf4f fp 0x00007f727227e400 Unknown symbol
  pc 0x00007f7271378bbb fp 0x00007f727227e448 Unknown symbol
  pc 0x00007f7277e480b9 fp 0x00007f727227e4b8 Unknown symbol
  pc 0x00007f7277e47f59 fp 0x00007f727227e518 Unknown symbol
  pc 0x00007f7277e47856 fp 0x00007f727227e5a8 Unknown symbol
  pc 0x00007f7277e66f5f fp 0x00007f727227e618 Unknown symbol
  pc 0x00007f7277e66cfd fp 0x00007f727227e658 Unknown symbol
  pc 0x00007f7277e481d4 fp 0x00007f727227e6c8 Unknown symbol
  pc 0x00007f7277e47f59 fp 0x00007f727227e728 Unknown symbol
  pc 0x00007f7277e47856 fp 0x00007f727227e7b8 Unknown symbol
  pc 0x00007f7277e66f5f fp 0x00007f727227e828 Unknown symbol
  pc 0x00007f7277e66cfd fp 0x00007f727227e868 Unknown symbol
  pc 0x00007f7277e64fe3 fp 0x00007f727227e8b0 Unknown symbol
  pc 0x00007f7277e649ab fp 0x00007f727227e8f0 Unknown symbol
  pc 0x00007f7277e648d9 fp 0x00007f727227e918 Unknown symbol
  pc 0x00007f7277e3b581 fp 0x00007f727227e958 Unknown symbol
  pc 0x00007f7277e21a6c fp 0x00007f727227e980 Unknown symbol
  pc 0x00007f7278e01c4f fp 0x00007f727227e9f8 Unknown symbol
  pc 0x000055dee3dca133 fp 0x00007f727227eaa0 dart::DartEntry::InvokeCode(dart::Code const&, unsigned long, dart::Array const&, dart::Array const&, dart::Thread*)+0x153
  pc 0x000055dee3dc9f85 fp 0x00007f727227eb00 dart::DartEntry::InvokeFunction(dart::Function const&, dart::Array const&, dart::Array const&, unsigned long)+0x165
  pc 0x000055dee3dcc56d fp 0x00007f727227eb50 dart::DartLibraryCalls::HandleMessage(dart::Object const&, dart::Instance const&)+0x15d
  pc 0x000055dee3df3dd6 fp 0x00007f727227ec30 dart::IsolateMessageHandler::HandleMessage(std::__2::unique_ptr<dart::Message, std::__2::default_delete<dart::Message> >)+0x596
  pc 0x000055dee3e1e76c fp 0x00007f727227eca0 dart::MessageHandler::HandleMessages(dart::MonitorLocker*, bool, bool)+0x14c
  pc 0x000055dee3e1ee8f fp 0x00007f727227ed00 dart::MessageHandler::TaskCallback()+0x1df
  pc 0x000055dee3f3db98 fp 0x00007f727227ed80 dart::ThreadPool::WorkerLoop(dart::ThreadPool::Worker*)+0x148
  pc 0x000055dee3f3dfcc fp 0x00007f727227edb0 dart::ThreadPool::Worker::Main(unsigned long)+0x5c
  pc 0x000055dee3eb78e8 fp 0x00007f727227ee70 out/ReleaseX64/dart+0x1f568e8
-- End of DumpStackTrace
[exit     : sp(0) fp(0x7f727227d790) pc(0)]
[dart     : sp(0x7f727227d7a0) fp(0x7f727227d7c0) pc(0x7f72713100b0) dart:core_StackTrace_get_current ]
[dart     : sp(0x7f727227d7d0) fp(0x7f727227d808) pc(0x7f720f9b6b18) dart:_http__HttpClient@16463476__getConnection@16463476 ]
[dart     : sp(0x7f727227d818) fp(0x7f727227d8a8) pc(0x7f720f9f44c3) dart:_http__HttpClient@16463476__openUrl@16463476 ]
[dart     : sp(0x7f727227d8b8) fp(0x7f727227d920) pc(0x7f720f9d95f1) dart:_http__HttpClient@16463476_open ]
[dart     : sp(0x7f727227d930) fp(0x7f727227d978) pc(0x7f720f9fdfb0) dart:_http__HttpClient@16463476_get ]
[dart     : sp(0x7f727227d988) fp(0x7f727227da00) pc(0x7f7214d4135d) file:///b/s/w/ir/cache/builder/sdk/tests/standalone_2/io/http_session_test.dart_::_connectGetSession ]
[dart     : sp(0x7f727227da10) fp(0x7f727227da48) pc(0x7f725282ab04) file:///b/s/w/ir/cache/builder/sdk/tests/standalone_2/io/http_session_test.dart_::_testSessions_<anonymous closure>_<anonymous closure> ]
[dart     : sp(0x7f727227da58) fp(0x7f727227dac0) pc(0x7f7277e69aac) dart:async_::__rootRunUnary@4048458 ]
[dart     : sp(0x7f727227dad0) fp(0x7f727227db28) pc(0x7f7277e6988c) dart:async_::__rootRunUnary@4048458__rootRunUnary@4048458 ]
[dart     : sp(0x7f727227db38) fp(0x7f727227dbc0) pc(0x7f7277e68ad9) dart:async__CustomZone@4048458_runUnary ]
[dart     : sp(0x7f727227dbd0) fp(0x7f727227dc10) pc(0x7f7277e3f2ee) dart:async__FutureListener@4048458_handleValue ]
[dart     : sp(0x7f727227dc20) fp(0x7f727227dc70) pc(0x7f7277e3f00e) dart:async__Future@4048458__propagateToListeners@4048458_handleValueCallback ]
[dart     : sp(0x7f727227dc80) fp(0x7f727227dcf0) pc(0x7f7277e3e2af) dart:async__Future@4048458__propagateToListeners@4048458 ]
[dart     : sp(0x7f727227dd00) fp(0x7f727227dd38) pc(0x7f7277e3d003) dart:async__Future@4048458__complete@4048458 ]
[dart     : sp(0x7f727227dd48) fp(0x7f727227dd80) pc(0x7f7261549bbd) dart:async_Stream_fold_<anonymous closure> ]
[dart     : sp(0x7f727227dd90) fp(0x7f727227ddd0) pc(0x7f7214d2abcd) dart:_http__HttpClientResponse@16463476_listen_<anonymous closure> ]
[dart     : sp(0x7f727227dde0) fp(0x7f727227de40) pc(0x7f7277e480b9) dart:async_::__rootRun@4048458 ]
[dart     : sp(0x7f727227de50) fp(0x7f727227dea0) pc(0x7f7277e47f59) dart:async_::__rootRun@4048458__rootRun@4048458 ]
[dart     : sp(0x7f727227deb0) fp(0x7f727227df30) pc(0x7f7277e47856) dart:async__CustomZone@4048458_run ]
[dart     : sp(0x7f727227df40) fp(0x7f727227dfa0) pc(0x7f7277e66f5f) dart:async__CustomZone@4048458_runGuarded ]
[dart     : sp(0x7f727227dfb0) fp(0x7f727227dfe0) pc(0x7f727051b6f3) dart:async__BufferingStreamSubscription@4048458__sendDone@4048458_sendDone ]
[dart     : sp(0x7f727227dff0) fp(0x7f727227e030) pc(0x7f7270519652) dart:async__BufferingStreamSubscription@4048458__sendDone@4048458 ]
[dart     : sp(0x7f727227e040) fp(0x7f727227e068) pc(0x7f726ed8a99c) dart:async__BufferingStreamSubscription@4048458__close@4048458 ]
[dart     : sp(0x7f727227e078) fp(0x7f727227e0a0) pc(0x7f7249fb87f1) dart:async__ForwardingStream@4048458__handleDone@4048458 ]
[dart     : sp(0x7f727227e0b0) fp(0x7f727227e0e0) pc(0x7f7249fc66e7) dart:async__ForwardingStreamSubscription@4048458__handleDone@4048458 ]
[dart     : sp(0x7f727227e0f0) fp(0x7f727227e118) pc(0x7f72476b144a) dart:async__ForwardingStreamSubscription@4048458__handleDone@4048458__handleDone@4048458 ]
[dart     : sp(0x7f727227e128) fp(0x7f727227e188) pc(0x7f7277e480b9) dart:async_::__rootRun@4048458 ]
[dart     : sp(0x7f727227e198) fp(0x7f727227e1e8) pc(0x7f7277e47f59) dart:async_::__rootRun@4048458__rootRun@4048458 ]
[dart     : sp(0x7f727227e1f8) fp(0x7f727227e278) pc(0x7f7277e47856) dart:async__CustomZone@4048458_run ]
[dart     : sp(0x7f727227e288) fp(0x7f727227e2e8) pc(0x7f7277e66f5f) dart:async__CustomZone@4048458_runGuarded ]
[dart     : sp(0x7f727227e2f8) fp(0x7f727227e328) pc(0x7f727051b6f3) dart:async__BufferingStreamSubscription@4048458__sendDone@4048458_sendDone ]
[dart     : sp(0x7f727227e338) fp(0x7f727227e378) pc(0x7f7270519652) dart:async__BufferingStreamSubscription@4048458__sendDone@4048458 ]
[dart     : sp(0x7f727227e388) fp(0x7f727227e3b0) pc(0x7f72705180df) dart:async__DelayedDone@4048458_perform ]
[dart     : sp(0x7f727227e3c0) fp(0x7f727227e400) pc(0x7f727137bf4f) dart:async__StreamImplEvents@4048458_handleNext ]
[dart     : sp(0x7f727227e410) fp(0x7f727227e448) pc(0x7f7271378bbb) dart:async__PendingEvents@4048458_schedule_<anonymous closure> ]
[dart     : sp(0x7f727227e458) fp(0x7f727227e4b8) pc(0x7f7277e480b9) dart:async_::__rootRun@4048458 ]
[dart     : sp(0x7f727227e4c8) fp(0x7f727227e518) pc(0x7f7277e47f59) dart:async_::__rootRun@4048458__rootRun@4048458 ]
[dart     : sp(0x7f727227e528) fp(0x7f727227e5a8) pc(0x7f7277e47856) dart:async__CustomZone@4048458_run ]
[dart     : sp(0x7f727227e5b8) fp(0x7f727227e618) pc(0x7f7277e66f5f) dart:async__CustomZone@4048458_runGuarded ]
[dart     : sp(0x7f727227e628) fp(0x7f727227e658) pc(0x7f7277e66cfd) dart:async__CustomZone@4048458_bindCallbackGuarded_<anonymous closure> ]
[dart     : sp(0x7f727227e668) fp(0x7f727227e6c8) pc(0x7f7277e481d4) dart:async_::__rootRun@4048458 ]
[dart     : sp(0x7f727227e6d8) fp(0x7f727227e728) pc(0x7f7277e47f59) dart:async_::__rootRun@4048458__rootRun@4048458 ]
[dart     : sp(0x7f727227e738) fp(0x7f727227e7b8) pc(0x7f7277e47856) dart:async__CustomZone@4048458_run ]
[dart     : sp(0x7f727227e7c8) fp(0x7f727227e828) pc(0x7f7277e66f5f) dart:async__CustomZone@4048458_runGuarded ]
[dart     : sp(0x7f727227e838) fp(0x7f727227e868) pc(0x7f7277e66cfd) dart:async__CustomZone@4048458_bindCallbackGuarded_<anonymous closure> ]
[dart     : sp(0x7f727227e878) fp(0x7f727227e8b0) pc(0x7f7277e64fe3) dart:async_::__microtaskLoop@4048458 ]
[dart     : sp(0x7f727227e8c0) fp(0x7f727227e8f0) pc(0x7f7277e649ab) dart:async_::__startMicrotaskLoop@4048458 ]
[dart     : sp(0x7f727227e900) fp(0x7f727227e918) pc(0x7f7277e648d9) dart:async_::__startMicrotaskLoop@4048458__startMicrotaskLoop@4048458 ]
[dart     : sp(0x7f727227e928) fp(0x7f727227e958) pc(0x7f7277e3b581) dart:isolate_::__runPendingImmediateCallback@1026248 ]
[dart     : sp(0x7f727227e968) fp(0x7f727227e980) pc(0x7f7277e21a6c) dart:isolate__RawReceivePortImpl@1026248__handleMessage@1026248 ]
[entry    : sp(0x7f727227e990) fp(0x7f727227e9f8) pc(0x7f7278e01c4f)]

Full log:
https://logs.chromium.org/logs/dart/buildbucket/cr-buildbucket.appspot.com/8839717685687839392/+/u/Run_Isolate_Stress_Tests_shard_2/task_stdout_stderr:_Run_Isolate_Stress_Tests_shard_2

/cc @aam @mkustermann

[alexmarkov]

Still crashes:

===== CRASH =====
si_signo=Segmentation fault(11), si_code=1, si_addr=0x23
version=2.15.0-edge.c51f885613177f18a5516eda4bed62f701e0631d (be) (Thu Sep 16 16:12:17 2021 +0000) on "linux_x64"
pid=30984, thread=31001, isolate_group=main(0x55b593190a00), isolate=wrapper3158(0x55b5981b1800)
isolate_instructions=55b58dbfb920, vm_instructions=55b58dbfb920
  pc 0x000055b58decf9f7 fp 0x00007f360c27dd50 dart::StackTraceUtils::UnwindAwaiterChain(dart::Zone*, dart::GrowableObjectArray const&, dart::GrowableArray<unsigned long>*, dart::CallerClosureFinder*, dart::Closure const&)+0x127
  pc 0x000055b58decfd86 fp 0x00007f360c27df60 dart::StackTraceUtils::CollectFramesLazy(dart::Thread*, dart::GrowableObjectArray const&, dart::GrowableArray<unsigned long>*, int, std::__2::function<void (dart::StackFrame*)>*, bool*)+0x1e6
  pc 0x000055b58dd22646 fp 0x00007f360c27dfc0 out/ReleaseX64/dart+0x1e69646
  pc 0x000055b58dd8c7b0 fp 0x00007f360c27e1a0 out/ReleaseX64/dart+0x1ed37b0
  pc 0x000055b58dd8c174 fp 0x00007f360c27e1c0 out/ReleaseX64/dart+0x1ed3174
  pc 0x000055b58deac1c6 fp 0x00007f360c27e270 out/ReleaseX64/dart+0x1ff31c6
  pc 0x00007f3613201853 fp 0x00007f360c27e2b0 Unknown symbol
  pc 0x00007f3613204ea9 fp 0x00007f360c27e2e0 Unknown symbol
  pc 0x00007f35d4a88d2b fp 0x00007f360c27e310 Unknown symbol
  pc 0x00007f36122715ac fp 0x00007f360c27e388 Unknown symbol
  pc 0x00007f361227144c fp 0x00007f360c27e3f0 Unknown symbol
  pc 0x00007f361226dac9 fp 0x00007f360c27e488 Unknown symbol
  pc 0x00007f36122470ce fp 0x00007f360c27e4d8 Unknown symbol
  pc 0x00007f3612246c1e fp 0x00007f360c27e538 Unknown symbol
  pc 0x00007f36122450df fp 0x00007f360c27e5b8 Unknown symbol
  pc 0x00007f36122483cb fp 0x00007f360c27e600 Unknown symbol
  pc 0x00007f361226d55d fp 0x00007f360c27e640 Unknown symbol
  pc 0x00007f3612246364 fp 0x00007f360c27e6b0 Unknown symbol
  pc 0x00007f3612245f39 fp 0x00007f360c27e710 Unknown symbol
  pc 0x00007f36122445a6 fp 0x00007f360c27e7a0 Unknown symbol
  pc 0x00007f361226cfdf fp 0x00007f360c27e810 Unknown symbol
  pc 0x00007f361226cd9d fp 0x00007f360c27e850 Unknown symbol
  pc 0x00007f361226c833 fp 0x00007f360c27e898 Unknown symbol
  pc 0x00007f361226c4bb fp 0x00007f360c27e8d8 Unknown symbol
  pc 0x00007f361226bcc9 fp 0x00007f360c27e900 Unknown symbol
  pc 0x00007f361223db61 fp 0x00007f360c27e940 Unknown symbol
  pc 0x00007f3612221b68 fp 0x00007f360c27e980 Unknown symbol
  pc 0x00007f3613201c9f fp 0x00007f360c27e9f8 Unknown symbol
  pc 0x000055b58dd72183 fp 0x00007f360c27eaa0 dart::DartEntry::InvokeCode(dart::Code const&, unsigned long, dart::Array const&, dart::Array const&, dart::Thread*)+0x153
  pc 0x000055b58dd71fd5 fp 0x00007f360c27eb00 dart::DartEntry::InvokeFunction(dart::Function const&, dart::Array const&, dart::Array const&, unsigned long)+0x165
  pc 0x000055b58dd7461f fp 0x00007f360c27eb50 dart::DartLibraryCalls::HandleMessage(long, dart::Instance const&)+0x16f
  pc 0x000055b58dd9b4dc fp 0x00007f360c27ec30 dart::IsolateMessageHandler::HandleMessage(std::__2::unique_ptr<dart::Message, std::__2::default_delete<dart::Message> >)+0x2cc
  pc 0x000055b58ddc4a6c fp 0x00007f360c27eca0 dart::MessageHandler::HandleMessages(dart::MonitorLocker*, bool, bool)+0x14c
  pc 0x000055b58ddc518f fp 0x00007f360c27ed00 dart::MessageHandler::TaskCallback()+0x1df
  pc 0x000055b58deddcd8 fp 0x00007f360c27ed80 dart::ThreadPool::WorkerLoop(dart::ThreadPool::Worker*)+0x148
  pc 0x000055b58dede10c fp 0x00007f360c27edb0 dart::ThreadPool::Worker::Main(unsigned long)+0x5c
  pc 0x000055b58de5ae08 fp 0x00007f360c27ee70 out/ReleaseX64/dart+0x1fa1e08
-- End of DumpStackTrace
  pc 0x0000000000000000 fp 0x00007f360c27e2b0 sp 0x0000000000000000 [Stub] CallToRuntime
  pc 0x00007f3613204ea9 fp 0x00007f360c27e2e0 sp 0x00007f360c27e2c0 [Stub] Throw
  pc 0x00007f35d4a88d2b fp 0x00007f360c27e310 sp 0x00007f360c27e2f0 [Unoptimized] _Directory@14069316.create.<anonymous closure>
  pc 0x00007f36122715ac fp 0x00007f360c27e388 sp 0x00007f360c27e320 [Unoptimized] _rootRunUnary@4048458
  pc 0x00007f361227144c fp 0x00007f360c27e3f0 sp 0x00007f360c27e398 [Unoptimized] _rootRunUnary@4048458
  pc 0x00007f361226dac9 fp 0x00007f360c27e488 sp 0x00007f360c27e400 [Unoptimized] _CustomZone@4048458.runUnary
  pc 0x00007f36122470ce fp 0x00007f360c27e4d8 sp 0x00007f360c27e498 [Unoptimized] _FutureListener@4048458.handleValue
  pc 0x00007f3612246c1e fp 0x00007f360c27e538 sp 0x00007f360c27e4e8 [Unoptimized] _Future@4048458._propagateToListeners@4048458.handleValueCallback
  pc 0x00007f36122450df fp 0x00007f360c27e5b8 sp 0x00007f360c27e548 [Unoptimized] _Future@4048458._propagateToListeners@4048458
  pc 0x00007f36122483cb fp 0x00007f360c27e600 sp 0x00007f360c27e5c8 [Unoptimized] _Future@4048458._completeWithValue@4048458
  pc 0x00007f361226d55d fp 0x00007f360c27e640 sp 0x00007f360c27e610 [Unoptimized] _Future@4048458._asyncCompleteWithValue@4048458.<anonymous closure>
  pc 0x00007f3612246364 fp 0x00007f360c27e6b0 sp 0x00007f360c27e650 [Unoptimized] _rootRun@4048458
  pc 0x00007f3612245f39 fp 0x00007f360c27e710 sp 0x00007f360c27e6c0 [Unoptimized] _rootRun@4048458
  pc 0x00007f36122445a6 fp 0x00007f360c27e7a0 sp 0x00007f360c27e720 [Unoptimized] _CustomZone@4048458.run
  pc 0x00007f361226cfdf fp 0x00007f360c27e810 sp 0x00007f360c27e7b0 [Unoptimized] _CustomZone@4048458.runGuarded
  pc 0x00007f361226cd9d fp 0x00007f360c27e850 sp 0x00007f360c27e820 [Unoptimized] _CustomZone@4048458.bindCallbackGuarded.<anonymous closure>
  pc 0x00007f361226c833 fp 0x00007f360c27e898 sp 0x00007f360c27e860 [Unoptimized] _microtaskLoop@4048458
  pc 0x00007f361226c4bb fp 0x00007f360c27e8d8 sp 0x00007f360c27e8a8 [Unoptimized] _startMicrotaskLoop@4048458
  pc 0x00007f361226bcc9 fp 0x00007f360c27e900 sp 0x00007f360c27e8e8 [Unoptimized] _startMicrotaskLoop@4048458
  pc 0x00007f361223db61 fp 0x00007f360c27e940 sp 0x00007f360c27e910 [Unoptimized] _runPendingImmediateCallback@1026248
  pc 0x00007f3612221b68 fp 0x00007f360c27e980 sp 0x00007f360c27e950 [Unoptimized] _RawReceivePortImpl@1026248._handleMessage@1026248
  pc 0x00007f3613201c9f fp 0x00007f360c27e9f8 sp 0x00007f360c27e990 [Stub] InvokeDartCode

https://logs.chromium.org/logs/dart/buildbucket/cr-buildbucket/8835918067259570241/+/u/Run_Isolate_Stress_Tests_shard_2/task_stdout_stderr:_Run_Isolate_Stress_Tests_shard_2

[mkustermann]

It seems multiple such crashes happened at precisely dart::StackTraceUtils::UnwindAwaiterChain+0x127.

Looking at the disassembly of this C++ function, it appears to be crashing when trying to load the UntaggedClosure::function_ here:

void StackTraceUtils::UnwindAwaiterChain(
    Zone* zone,
    const GrowableObjectArray& code_array,
    GrowableArray<uword>* pc_offset_array,
    CallerClosureFinder* caller_closure_finder,
    const Closure& leaf_closure) {
  ...
  for (; !closure.IsNull(); closure = caller_closure_finder->FindCaller(closure)) {
    function = closure.function(); // <-- Crashes here.
    if (function.IsNull()) continue;
    ...
  }
}

Seems like the closure is garbage.

By reading through the code I haven't been able to identify how it can happen. Have also had no luck reproducing this so far.

[mkustermann]

One interesting piece of code is:

  DartFrameIterator future_frames(frames);
  if (function.recognized_kind() == MethodRecognizer::kRootZoneRunUnary) {
    frame = future_frames.NextFrame();
    ...
  }

The DartFrameIterator copy constructor is not initializing the current_frame, see here:

StackFrameIterator::StackFrameIterator(const StackFrameIterator& orig)
    :  ...
      current_frame_(nullptr), // <---
      ... { ... }

So it may not do what is intended there.

[mkustermann]

Landed 7abf6bf which will hopefully make the builder archive coredumps, then we can analyze the archived dumps.

[aam]

another crash on https://logs.chromium.org/logs/dart/buildbucket/cr-buildbucket/8827132115406361905/+/u/Run_Isolate_Stress_Tests_shard_2/task_stdout_stderr:_Run_Isolate_Stress_Tests_shard_2

[mkustermann]

Have been looking at this issue today. It seems what is happening is that in this code:

ClosurePtr StackTraceUtils::ClosureFromFrameFunction(...) {
  ...
  if (function.recognized_kind() == MethodRecognizer::kFutureListenerHandleValue) {
    ...

    // The _FutureListener receiver is at the top of the previous frame, right
    // before the arguments to the call.
    Object& receiver =
        Object::Handle(*(reinterpret_cast<ObjectPtr*>(frame->GetCallerSp()) +
                         kNumArgsFutureListenerHandleValue));

    ...
  }
  ...
}

the receiver (which is supposed to be a _FutureListener object) is <optimized out>.

Even though the _FutureListener.handeValue function itself cannot be inlined (so we can always see it when walking the stack), its arguments can become unreachable and clobbered by deoptimizations.

Looking at the Dart code that seems to make sense:

  @pragma("vm:recognized", "other")
  @pragma("vm:never-inline")
  FutureOr<T> handleValue(S sourceResult) {
    return _zone.runUnary<FutureOr<T>, S>(_onValue, sourceResult);
  }

After evaluating _onValue the receiver is not needed anymore.

Supposedly this is because we don't introduce new LocalVariable proxies for the arguments (like we do for functions that e.g. have optional parameters) and instead use LocalVariables representing the slots in the caller frame (i.e. causing us to modify the caller-pushed arguments on assignment to the variables in the callee).

We could either make the stack trace collection code more robust or change compiler to avoid modifying the slots in the caller frame.

@alexmarkov @mraleph wdyt?

[alexmarkov]

I think we should make stack trace collection more robust. Any variable may become unused after optimizations, so code trying to find variables in the stack frames should be prepared to the absence of the variable or OptimizedOut sentinel.

Although parameters are pushed by caller, they are "owned" by the callee and callee can freely modify their values. Making an artificial copy of parameters in the callee seems to be a waste, as caller never looks at the passed parameters.

[alexmarkov]

/cc @cskau-g