From 7d51b657d1f3b7728a418f15867d30c8561d78a4 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Wed, 30 Apr 2025 15:24:18 +0800 Subject: [PATCH 01/24] [fix] support npu --- .../coati/distributed/consumer.py | 14 +++++++--- .../coati/distributed/producer.py | 10 +++---- .../kernel_meta/buildPidInfo.json | 6 +++++ applications/ColossalChat/rl_example.py | 26 +++++++++---------- 4 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 applications/ColossalChat/kernel_meta/buildPidInfo.json diff --git a/applications/ColossalChat/coati/distributed/consumer.py b/applications/ColossalChat/coati/distributed/consumer.py index 1cebcb40eacb..d04ffae2ff8f 100644 --- a/applications/ColossalChat/coati/distributed/consumer.py +++ b/applications/ColossalChat/coati/distributed/consumer.py @@ -18,7 +18,7 @@ from .comm import ray_broadcast_tensor_dict from .utils import bind_batch, pad_batch, post_recv, unbind_batch - +first_sleep=True class BaseConsumer: def __init__( self, @@ -55,7 +55,8 @@ def __init__( self.model_config = model_config self.plugin_config = plugin_config - self.device = get_current_device() + # self.device = get_current_device() + self.device = 'npu' self.lr_scheduler = None def setup(self) -> None: @@ -86,11 +87,11 @@ def setup(self) -> None: # use hybrid tp + pp if self.tp_rank == 0 and self.dp_rank == 0: cc.init_collective_group( - self.num_producers + 1, self.num_producers, group_name=f"sync_model_{self.pp_rank}" + self.num_producers + 1, self.num_producers, backend='hccl', group_name=f"sync_model_{self.pp_rank}" ) else: if self.rank == 0: - cc.init_collective_group(self.num_producers + 1, self.num_producers, group_name="sync_model") + cc.init_collective_group(self.num_producers + 1, self.num_producers, backend='hccl', group_name="sync_model") self.buffer = [] @@ -114,6 +115,11 @@ def loop(self) -> None: # receive data from producers for r in range(self.num_producers): print(f"[T{dist.get_rank()}] Recv data episode {episode} step {step} from {r}") + global first_sleep + if first_sleep: + import time + time.sleep(180) + first_sleep=False self.buffer.extend( unbind_batch( ray_broadcast_tensor_dict( diff --git a/applications/ColossalChat/coati/distributed/producer.py b/applications/ColossalChat/coati/distributed/producer.py index a2d675870fc2..c45ddd450fb4 100644 --- a/applications/ColossalChat/coati/distributed/producer.py +++ b/applications/ColossalChat/coati/distributed/producer.py @@ -13,7 +13,6 @@ from .inference_backend import BACKEND_MAP from .utils import pre_send - class BaseProducer: def __init__( self, @@ -71,7 +70,8 @@ def __init__( num_workers=4, drop_last=True, ) - self.device = get_current_device() + # self.device = get_current_device() + self.device = 'npu' # init backend if backend in BACKEND_MAP: @@ -82,12 +82,12 @@ def __init__( self.consumer_pp_size = consumer_plugin_config["pp_size"] # consumer pp size def setup(self) -> None: - cc.init_collective_group(1 + self.num_consumer_procs, 0, group_name=f"sync_data_{self.producer_idx}") + cc.init_collective_group(1 + self.num_consumer_procs, 0, backend='hccl', group_name=f"sync_data_{self.producer_idx}") if self.consumer_pp_size > 1: for i in range(self.consumer_pp_size): - cc.init_collective_group(self.num_producers + 1, self.producer_idx, group_name=f"sync_model_{i}") + cc.init_collective_group(self.num_producers + 1, self.producer_idx, backend='hccl', group_name=f"sync_model_{i}") else: - cc.init_collective_group(self.num_producers + 1, self.producer_idx, group_name="sync_model") + cc.init_collective_group(self.num_producers + 1, self.producer_idx, backend='hccl', group_name="sync_model") def rollout(self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwargs) -> Dict[str, torch.Tensor]: raise NotImplementedError diff --git a/applications/ColossalChat/kernel_meta/buildPidInfo.json b/applications/ColossalChat/kernel_meta/buildPidInfo.json new file mode 100644 index 000000000000..7194c917d7ed --- /dev/null +++ b/applications/ColossalChat/kernel_meta/buildPidInfo.json @@ -0,0 +1,6 @@ +[ + [ + 3383334, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_18208839462778721971" + ] +] \ No newline at end of file diff --git a/applications/ColossalChat/rl_example.py b/applications/ColossalChat/rl_example.py index 788e60c2edac..18948a569642 100644 --- a/applications/ColossalChat/rl_example.py +++ b/applications/ColossalChat/rl_example.py @@ -129,7 +129,7 @@ args.top_k = -1 inference_model_config = dict(path=args.model) - train_model_config = dict(path=args.model, use_flash_attention_2=True, use_cache=False) + train_model_config = dict(path=args.model, use_flash_attention_2=False, use_cache=False, attn_implementation="eager") generate_config = dict(top_k=args.top_k, top_p=args.top_p, temperature=args.temperature) if args.backend == "transformers": @@ -155,7 +155,7 @@ enforce_eager=True, enable_chunked_prefill=True, max_model_len=args.max_new_tokens + args.max_prompt_tokens, - tensor_parallel_size=1, + tensor_parallel_size=2, ) ) generate_config.update( @@ -219,18 +219,18 @@ num_generations=args.num_generations, train_model_config=train_model_config, grpo_config=grpo_config, - plugin_config={ - "zero_stage": 2, - }, # for zero # plugin_config={ - # "tp_size": 1, - # "pp_size": 2, - # "microbatch_size": max( - # 1, args.train_microbatch_size // 2 - # ), # microbatch size should be set to train_microbatch_size // pp_size - # "zero_stage": 0, - # "max_norm": 1.0, - # }, # for pp, tp + # "zero_stage": 2, + # }, # for zero + plugin_config={ + "tp_size": 2, + "pp_size": 2, + "microbatch_size": max( + 1, args.train_microbatch_size // 2 + ), # microbatch size should be set to train_microbatch_size // pp_size + "zero_stage": 1, + "max_norm": 1.0, + }, # for pp, tp inference_backend=args.backend, master_addr="localhost", master_port=args.master_port, From f4c1993726b9c9d033548d15f53feeea0df41f66 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Thu, 8 May 2025 17:54:50 +0800 Subject: [PATCH 02/24] [feat] multinode 14B --- .../ColossalChat/.nfs00000000078104b100001d70 | 389 ++++++++++++++++++ .../coati/distributed/consumer.py | 4 +- .../ColossalChat/coati/distributed/launch.py | 124 +++++- .../coati/distributed/producer.py | 3 +- applications/ColossalChat/fusion_result.json | 1 + .../kernel_meta/buildPidInfo.json | 12 +- applications/ColossalChat/rl_example.py | 8 +- .../ColossalChat/tests/test_hybrid.py | 143 +++++++ applications/ColossalChat/tests/test_ray.py | 88 ++++ applications/ColossalChat/tests/test_vllm.py | 27 ++ .../ColossalChat/tests/test_vllm_multinode.py | 108 +++++ 11 files changed, 891 insertions(+), 16 deletions(-) create mode 100755 applications/ColossalChat/.nfs00000000078104b100001d70 create mode 100644 applications/ColossalChat/fusion_result.json create mode 100644 applications/ColossalChat/tests/test_hybrid.py create mode 100644 applications/ColossalChat/tests/test_ray.py create mode 100644 applications/ColossalChat/tests/test_vllm.py create mode 100644 applications/ColossalChat/tests/test_vllm_multinode.py diff --git a/applications/ColossalChat/.nfs00000000078104b100001d70 b/applications/ColossalChat/.nfs00000000078104b100001d70 new file mode 100755 index 000000000000..5db53e4f6063 --- /dev/null +++ b/applications/ColossalChat/.nfs00000000078104b100001d70 @@ -0,0 +1,389 @@ +2025-05-06 22:50:50,843 WARNING collective.py:22 -- NCCL seems unavailable. Please install Cupy following the guide at: https://docs.cupy.dev/en/stable/install.html. +/home/duanjunwen/ColossalAI/colossalai/utils/safetensors.py:13: UserWarning: Please install the latest tensornvme to use async save. pip install git+https://github.com/hpcaitech/TensorNVMe.git + warnings.warn( +/usr/local/python3.10/lib/python3.10/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. + warn("The installed version of bitsandbytes was compiled without GPU support. " +/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/normalization.py:48: UserWarning: Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel + warnings.warn("Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel") +'NoneType' object has no attribute 'cadam32bit_grad_fp32' +2025-05-06 22:51:04,272 INFO worker.py:1654 -- Connecting to existing Ray cluster at address: 10.0.0.3:6379... +2025-05-06 22:51:04,285 INFO worker.py:1841 -- Connected to Ray cluster. +(pid=259440) NCCL seems unavailable. Please install Cupy following the guide at: https://docs.cupy.dev/en/stable/install.html. +(pid=132985, ip=10.0.0.4) /home/duanjunwen/ColossalAI/colossalai/utils/safetensors.py:13: UserWarning: Please install the latest tensornvme to use async save. pip install git+https://github.com/hpcaitech/TensorNVMe.git +(pid=132985, ip=10.0.0.4) warnings.warn( +(pid=259440) /usr/local/python3.10/lib/python3.10/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +(pid=259440) warn("The installed version of bitsandbytes was compiled without GPU support. " +(pid=132987, ip=10.0.0.4) /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/normalization.py:48: UserWarning: Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel +(pid=132987, ip=10.0.0.4) warnings.warn("Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel") +(GRPOConsumer pid=132981, ip=10.0.0.4) Loading checkpoint shards: 0%| | 0/4 [00:00 +(SimpleProducer pid=259440) INFO 05-06 22:51:32 config.py:549] This model supports multiple tasks: {'embed', 'classify', 'reward', 'score', 'generate'}. Defaulting to 'generate'. +(SimpleProducer pid=259435) INFO 05-06 22:51:32 config.py:549] This model supports multiple tasks: {'embed', 'score', 'reward', 'classify', 'generate'}. Defaulting to 'generate'. +(SimpleProducer pid=259436) INFO 05-06 22:51:32 config.py:549] This model supports multiple tasks: {'classify', 'generate', 'score', 'reward', 'embed'}. Defaulting to 'generate'. +(GRPOConsumer pid=132981, ip=10.0.0.4) [extension] Loading the JIT-built cpu_adam_arm kernel during runtime now +(GRPOConsumer pid=132985, ip=10.0.0.4) Using GRPO config: {'lr': 1e-06, 'train_microbatch_size': 8, 'beta': 0.01, 'loss_variation': 'sample_level', 'reward_fn_type': 'boxed'} [repeated 7x across cluster] +(SimpleProducer pid=259436) INFO 05-06 22:51:32 config.py:1555] Chunked prefill is enabled with max_num_batched_tokens=2048. [repeated 7x across cluster] +(SimpleProducer pid=259436) INFO 05-06 22:51:32 llm_engine.py:234] Initializing a V0 LLM engine (v0.7.3) with config: model='/home/duanjunwen/models/Qwen/Qwen2.5-7B', speculative_config=None, tokenizer='/home/duanjunwen/models/Qwen/Qwen2.5-7B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, device_config=npu, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=/home/duanjunwen/models/Qwen/Qwen2.5-7B, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=False, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"splitting_ops":[],"compile_sizes":[],"cudagraph_capture_sizes":[],"max_capture_size":0}, use_cached_outputs=False,  [repeated 7x across cluster] +(SimpleProducer pid=259436) WARNING 05-06 22:51:33 utils.py:2262] Methods add_lora,add_prompt_adapter,cache_config,compilation_config,current_platform,list_loras,list_prompt_adapters,load_config,pin_lora,pin_prompt_adapter,remove_lora,remove_prompt_adapter not implemented in  [repeated 7x across cluster] +(GRPOConsumer pid=132981, ip=10.0.0.4) [extension] Time taken to load cpu_adam_arm op: 0.1460132598876953 seconds +(SimpleProducer pid=259437) INFO 05-06 22:52:06 executor_base.py:111] # npu blocks: 3809, # CPU blocks: 585 +(SimpleProducer pid=259437) INFO 05-06 22:52:06 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.03x +(GRPOConsumer pid=132987, ip=10.0.0.4) [extension] Loading the JIT-built cpu_adam_arm kernel during runtime now [repeated 7x across cluster] +(GRPOConsumer pid=132987, ip=10.0.0.4) [extension] Time taken to load cpu_adam_arm op: 0.16289782524108887 seconds [repeated 7x across cluster] +(SimpleProducer pid=259437) INFO 05-06 22:52:08 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 16.79 seconds +(SimpleProducer pid=259449) INFO 05-06 22:52:12 executor_base.py:111] # npu blocks: 3809, # CPU blocks: 585 +(SimpleProducer pid=259449) INFO 05-06 22:52:12 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.03x +(SimpleProducer pid=259449) INFO 05-06 22:52:14 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 15.55 seconds +(SimpleProducer pid=259440) INFO 05-06 22:52:17 executor_base.py:111] # npu blocks: 3809, # CPU blocks: 585 +(SimpleProducer pid=259440) INFO 05-06 22:52:17 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.03x +(SimpleProducer pid=259440) INFO 05-06 22:52:18 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 16.44 seconds +(SimpleProducer pid=259435) INFO 05-06 22:52:20 executor_base.py:111] # npu blocks: 3810, # CPU blocks: 585 [repeated 2x across cluster] +(SimpleProducer pid=259435) INFO 05-06 22:52:20 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.06x [repeated 2x across cluster] +(SimpleProducer pid=259435) INFO 05-06 22:52:22 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 18.77 seconds [repeated 2x across cluster] +(SimpleProducer pid=259445) INFO 05-06 22:52:27 executor_base.py:111] # npu blocks: 3810, # CPU blocks: 585 +(SimpleProducer pid=259445) INFO 05-06 22:52:27 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.06x +(GRPOConsumer pid=132981, ip=10.0.0.4) [05/06/25 22:52:34] INFO colossalai - colossalai - INFO: +(GRPOConsumer pid=132981, ip=10.0.0.4) /home/duanjunwen/ColossalAI/colossalai/initialize.py:75 launch +(GRPOConsumer pid=132981, ip=10.0.0.4) INFO colossalai - colossalai - INFO: Distributed environment is +(GRPOConsumer pid=132981, ip=10.0.0.4) initialized, world size: 8 +(SimpleProducer pid=259445) INFO 05-06 22:52:29 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 16.97 seconds +(SimpleProducer pid=259436) INFO 05-06 22:52:36 executor_base.py:111] # npu blocks: 3810, # CPU blocks: 585 +(SimpleProducer pid=259436) INFO 05-06 22:52:36 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.06x +(SimpleProducer pid=259436) INFO 05-06 22:52:38 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 17.26 seconds +(SimpleProducer pid=259443) INFO 05-06 22:52:49 executor_base.py:111] # npu blocks: 3810, # CPU blocks: 585 +(SimpleProducer pid=259443) INFO 05-06 22:52:49 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.06x +(SimpleProducer pid=259443) INFO 05-06 22:52:51 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 17.65 seconds +(SimpleProducer pid=259435) [P0] num_valid_microbatches 468, nmb: 4, dl: 468 +(GRPOConsumer pid=132981, ip=10.0.0.4) Consumer0 num_update: 117, num_recv: 4, nmb: 1 +(GRPOConsumer pid=132981, ip=10.0.0.4) [T0] Recv data episode 0 step 0 from 0 +(SimpleProducer pid=259436) [P6] Send data [('input_ids', torch.Size([2, 8, 2654])), ('attention_mask', torch.Size([2, 8, 2654])), ('action_log_probs', torch.Size([2, 8, 2142])), ('action_mask', torch.Size([2, 8, 2142])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(SimpleProducer pid=259445) [P7] num_valid_microbatches 468, nmb: 4, dl: 468 [repeated 7x across cluster] +(GRPOConsumer pid=132988, ip=10.0.0.4) Consumer7 num_update: 117, num_recv: 4, nmb: 1 [repeated 7x across cluster] +(GRPOConsumer pid=132988, ip=10.0.0.4) [T7] Recv data episode 0 step 0 from 0 [repeated 7x across cluster] +(SimpleProducer pid=259440) [P5] Send data [('input_ids', torch.Size([2, 8, 3944])), ('attention_mask', torch.Size([2, 8, 3944])), ('action_log_probs', torch.Size([2, 8, 3432])), ('action_mask', torch.Size([2, 8, 3432])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(SimpleProducer pid=259449) [P2] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(SimpleProducer pid=259435) [P0] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] [repeated 4x across cluster] +(SimpleProducer pid=259434) Rollout example: +(SimpleProducer pid=259434) system +(SimpleProducer pid=259434) Please reason step by step, and put your final answer within \boxed{}. +(SimpleProducer pid=259434) user +(SimpleProducer pid=259434) Regular hexagon $ABCDEF$ is divided into six smaller equilateral triangles, such as $\triangle ABG$, shown in boldface in the diagram. By connecting every other vertex, we obtain a larger equilateral triangle $\triangle ACE$, also shown in boldface. Compute the ratio $[\triangle ABG]/[\triangle ACE]$. [asy] +(SimpleProducer pid=259434) size(150); defaultpen(linewidth(0.8)); dotfactor=5; +(SimpleProducer pid=259434) pair[] hex = new pair[6]; +(SimpleProducer pid=259434) string[] hexlabels = {"$C$","$B$","$A$","$F$","$E$","$D$"}; +(SimpleProducer pid=259434) hexlabels.cyclic=true; +(SimpleProducer pid=259434) hex[0] = dir(0); +(SimpleProducer pid=259434) for(int i = 1; i <= 6; ++i){ +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) hex[i] = dir(60*i); +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) draw(hex[i] -- hex[i-1]); +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) dot(hexlabels[i],hex[i],hex[i]); +(SimpleProducer pid=259434) } +(SimpleProducer pid=259434) draw(hex[0]--hex[3]); draw(hex[1]--hex[4]); draw(hex[2]--hex[5]); +(SimpleProducer pid=259434) draw(hex[0]--hex[2]--hex[4]--cycle,linewidth(1.3)); +(SimpleProducer pid=259434) draw(hex[1]--hex[2]--(0,0)--cycle,linewidth(1.3)); +(SimpleProducer pid=259434) dot("$G$",(0,0),2*S); +(SimpleProducer pid=259434) [/asy] Let's think step by step and output the final answer within \boxed{}. +(SimpleProducer pid=259434) assistant +(SimpleProducer pid=259434) To compute the ratio $[\triangle ABG]/[\triangle ACE]$, we can use the formula for the area of an equilateral triangle. The area of an equilateral triangle with side length $s$ is given by $\frac{\sqrt{3}}{4} s^2$. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) In triangle $ ACE$, the side length is $s_1$, and in triangle $ ABG$, the side length is $s_2$. To find the ratio $\frac{[\triangle ABG]}{[\triangle ACE]}$, we can use the fact that each of the smaller equilateral triangles is similar to the larger triangles. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) Using this similarity relationship, we can express the side lengths of the triangle in terms of the length of the side of the regular hexagon, $s$. Since $AC$ is a diagonal of the hexagon, it is equal to $2s$. Therefore, we have $s_1 = 2s$. Similarly, $BG$ is half the length of $AC$, so $s_2 = \frac{1}{2}s_1 = s$. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) Now we can calculate the areas of the triangles using the formula for the area of an equilateral triangle. We have: +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) $[\triangle ABG] = \frac{\sqrt{3}}{4} s_2^2 = \frac{\sqrt{3}}{4} s^2$ +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) $[\triangle ACE] = \frac{\sqrt{3}}{4} s_1^2 = \frac{\sqrt{3}}{4} (2s)^2 = 4 \frac{\sqrt{3}}{4} s^2 = \sqrt{3} s^2$ +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) Thus, the ratio is: +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) $\frac{[\triangle ABG]}{[\triangle ACE]} = \frac{\frac{\sqrt{3}}{4} s^2}{\sqrt{3} s^2} = \frac{1}{4}$ +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) So the answer is $\boxed{1/4}$. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) jifss - Factoring Example Number 1 (#17)Sheila uses rewrite congruence notation to write.info@... er Exponential notation for. tables and graphs. These notation, number, and algebra to represent information. ppt Download 7-May-2018 positano.it supportive media and humanist a are. . . . transforms to 1000000 people a year.We've made a. moving up to new social and physical challenges.Now the human. world numerical notation for. But the majority of numerals look. It our socio-cultural environment using Supernormality Theory. in essence. taking up weak, competing, ideas; they religious sects), and moreland (religious morality).taking. the Project Team include Ian Goldin. and Mike Hammond.adoption of more rational. . sum of themFind out what. time, location visiting as the day of ReFESA. Numerical notation for. CFCSC Technology Standards Explained. What are they up to?? . rounded to two decimal places, in vertrite.2 ., 34 Numerical notation for. 25 100 306.14. 8.32. 2.725 158.89. Practice Grid Version. 6.taking these developments a step further.And a recent study in a major. economic journal. . but we need to explain what exactly has changed. The core of the differences has been absent,in small doses, in even more places, . . of the U.S.professor with documentation. . . at least that it. counting without notation. for. . Allies’ Strategy PubUniver. Why? advantages. University. of. London. time. RajeshR., KenHyett and Laura. . million deaths in French- . religion." Choose correctly. [ além. pronominal] .CONQUIS. abor setting up for sub-contract, sub- and. become. Numerical notation for. 2 t Separate 4-m systems?. Mental division n. of decimal points, if necessary.The woman has exchanged a. and Certificates of 1. Our own internal. "This ability for a fast moving physical object to. numerical notation for. Original WOFR information. then doing so on. Words using at least one digital. and signals digital signals neer 1.0. . Very soon, they will learn other forms of numerical notation for. A third problem is particularly.More information . NBP09019 - AS Lighthouse Pilot 10 shipping at home. Another example could strive. The Muslim world possibly solu-. . situated. within. . 10: (12.3482 × 2^. muito bem para fazer algo; is a market. . SYD 01:30 attendance women. Learning the types of inactive buscarGood book. PRP 2010 de purchase. Mathematical +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) jifss - Factoring Example Number 1 (#17)Sheila uses rewrite congruence notation to write.info@... er Exponential notation for. tables and graphs. These notation, number, and algebra to represent information. ppt Download 7-May-2018 positano.it supportive media and humanist a are. . . . transforms to 1000000 people a year.We've made a. moving up to new social and physical challenges.Now the human. world numerical notation for. But the majority of numerals look. It our socio-cultural environment using Supernormality Theory. in essence. taking up weak, competing, ideas; they religious sects), and moreland (religious morality).taking. the Project Team include Ian Goldin. and Mike Hammond.adoption of more rational. . sum of themFind out what. time, location visiting as the day of ReFESA. Numerical notation for. CFCSC Technology Standards Explained. What are they up to?? . rounded to two decimal places, in vertrite.2 ., 34 Numerical notation for. 25 100 306.14. 8.32. 2.725 158.89. Practice Grid Version. 6.taking these developments a step further.And a recent study in a major. economic journal. . but we need to explain what exactly has changed. The core of the differences has been absent,in small doses, in even more places, . . on an international learning and distance course Series.line are initially incomplete.Achre kingdoms. providers. . not just prestige. . Our societyeven. . uses this. CAV GROUP ContactsObjectives of this Division have adopted new standards with. the top-down manner -- using. . essay on the scientific and. so achieving longer deserve 100ams in New York City.is far more important' ' . saying' ' . Video Games: Affect can become. Numerical notation for. Calculating costs. THE OR邓小, thanks ... who established city-states. in Italy viz . Notice the pattern:1, 4, 16, 64, 256, . . . Compare this with the place value notation we use in decimal system (based on multiplication with 10s from right to left; and addition in the last column). In binary system, multiplication is with 2s. So, there is a place value notation for binary numbers as well. have few practical uses, and knowledge of binary notation and. standards . The basic number system. The intuitive system of numeration used by . Numerical notation for. procedures for add, subtract, multiply, and divide numbers. needed . . Indeed, teaching of mathematics by Tamil came at the expense of making. . Both use a formal grammar to derive their programs '. ‘0/3’ £lion. . and. . Numerical notation for. while Brunet (2009) identifies a high market demand and use. Proprietary and Quantitative Regulation for, Dominance . if bourgeoise of UniversAlm mathematics. if I is a. 3.x 2² '). If for numbers are encoded. meter close to dollars.. unlimited liability so would have the)This figure includes, for the purposes of initial processing. Numerical notation for. American. For forged documents. Search examples.http://bmpdb.com/22-Misc.html cộng đồngSinhala (12345) is used for numerals (and ' 100 w public ', . establishing a workshop for digitization. . to two devices and use to multiply. file name '03Media_characters.zip'. . and . involving the study of Narcissism or Morality on a global level, ._RANGE._, means '. divided by A Можно to get meaningful results without revisiting methodological questions,. . the next mathematical realization. to 100000 personal finance articles. . to do aoi courtesies, there by. . Supernormality . some never made a spare. number of Maajnini. TERMS.Square[ FL 80. left to half of our 5' playquery ' '. The ideas. . list. of age for their Mental calculations actizing. PR = widthanth GGNnnbr hand racers,. German A EUL . coleg Www.bestpracticeseducatedUSA.eqales IU 70-80. 7 using binary arithmetic. In加拿大经验 that is. CAN operate assembler based MCUs' ' . getting the. concept. On a framework to provide a window . . Numerical notation for. e cite text[ ABP] COL 449 PHY 140R ORTH. Kamlesha point that 1000$ 'mer' meaningless methodological superiority over. " In technology. receive level must first computing converts. quit working, it was named . INTERNATIONAL. COMMUNICATION UNITED KINGDOM. for array, vector, and matrix elements. Plus. . United States anddiary system Insimon . also writes 100 years of history as extraordinary. tended. . corners to find groups of hash marks. . desperation .Phone, Internetand all manner throughout most Southern and essentially рей . searches, is. position '' (the vertical norsmen hafa account of Environmental Evaluation of. Graphed. Environment. for Numerical notation for. making adjustments by subtracting one value from the next, . were. responsible for providing a standard of days, which determination markers coming up in the Bermuda Triangle. FOR SCHOLASTIC ANIMATED GRAPHICSTring to asks letters. This is the. . . mathematical learning from step up the International. Mathematics. Curriculum & , spae, /\ refold BIOFIT Connect. US LT 12,000 Grammar and Williamiving Same solution he came up "I am arguably extrapolated the speed of Our research been graduate... ' . . . . . thousand, billion, trillion, etc. 02. . inv see '01Solution x 2 2 '01x 473所述。solution x other. Related . . Numerical notation for. 2012 The Android Four Horizons 15 18 year requirements numberWriters, Call for a solution, I have abolition of money. .. 등에 의거 후 구매Pagodas symbolism award system which Makin copies tough on Catholics: ... it out in future shipments.Will not wait. not ('; +(SimpleProducer pid=259434) jifss - Verical-line notifications and distribution. for. . CERT. CP #5 000 UP & Down FD 50 @ 400KCCERT. Downloaded at : Monday, March 06, 2023 Examination Timetable Original Request: solicitud at the . EmailBAXCO and CSSCO Certificate of Complexity . their support from总理. 'Customer Response Center3 . www.markistan.com Website of the. Educational Framework and Community Service hours were our greatest contribution to the HEALTH MANAGEMENT . . (Subjective. . In three years. might be conceptually strong.They. focused and . . numeral in Mathematics . born every year: 320,000 couples per. Census does not have. Subsidized children exist in an elite hierarchy, for the IDMS have industrial agreements,ad.fopoly meeting Oct 3, and significant policies and laws have been enacted, mostly in accordance with their. teaching course modules for. . p i erosion . (16;48) and. . for. . . becomes. Adding interactive. . . Marshall, of. Chapter#: All of you top prime delta is the lead when the labor relations of. LOS VERS. Marihead hand, who want the. . Who was. regardless of claims, verification, or in some cases, disputes from Take Technical Applications. young people interested. . ) . I, finally, spent 300 staff head core refracting light under his unit . . . Certificate to' ' . 670 Trading Standard. ANALYSIS provided for Mountains2 with . ' . Numerical notation for. Numbered Surgery_ . Food and Drug Administration (FDA), or. how to play in ATC thinking routinesMaster page ... for money. . . updated. tJ.|t ' . . dependent. . people as working and student residents. The interpersonal skills necessary for leading numeracy. ' .Fibus are all in the spring of same month professiona1 books and . This is the general. content Knowledge measurement feasible. CDC was established. by Congress or the ratio is . conch on in channel. direction. . moderately. elastic Council for Labor Commission technical and operational. . Meaning that words should not be separated by spaces.is characterized. You open . . to help fill the junior. placement service section how to open a file in excel No . inter-school programs a doctoral. humans. is. beginning of daily. Kurdish Women's High School goat dy_backup_updater d g 1 upbase . Basis Registers.There is a high level of responsibility in creating, maintaining, and managing a well-structured ecommerce architecture 163A level to substantially . insecure environment is uniquely crashing the browser, followed. system) and never received pay checks. . regarding videogames in . Numerical notation for. undergraduate course in mathematics. of the textbooks. . . In geographic terms . Arial" " No" " No_hi! In some cases, parents have increased the reference can not take credit for success, for. Benefits Return to . Our class hundred systems Below is a short . Pyinvoke (PIC 2:R/W CLR_FUTURE) . In contemporary mathematics, an orbit is a collection of points related by the group action of a group element. The term is most commonly used in the context of Lie groups and related group actions.The generalization of the simple idea of orbits of group actions is minimal homogeneous spaces. File- . ' . That's sad, ' he said like the activist exceptionally skilled. Board and place holders. . writing sample. and program specifically. . is explained recently. mathematics students. . as part of the. 24 hour, seven-andover. an important cultural role representations in mathematics: this. Office of eNew under Lien/. belief of Vernon and keys in multiplication. and reduce. 08, 119 . Back to Understand Observer.Dataeye . for Suicide prevention.numer stayed quietly trou y out... a Presentation Symbolicnary, Amphibia_230-235 in System_ompute_4 in Word'. Ensure bog State equal nne 4000. Typically, this will 2 be said that productivity is a technical and economic. computer science.. The third era can be better total hours worked per week to 35 hours per week.long help us to to be grabbed by.Temporary Characteristics The current. Numerical notation for. different than the symbols in security; 2.0187 Numeric Key A Espresso HiPro Authentication/ " +(SimpleProducer pid=259434) jifss - Tatiana likes to command all numbers to whisper. their Sumiantile Age GRWYLA + Adolescent subclass number's' 5 in—not surprised —that construe-to-be PT governing powers?Generation of Biorefineries(USDOE-AL ) N numbers 3 to reconstruct .. axes. to Point" by.100$, $300 oro miercion?c. 02ti $q, o zn . Numerical notation for. that these solutions are related. by philosophy. . programme and rated documents. Physics 15. Rice 12.2 Reformulate Theorem around. . . A) 0.118m B) 0.228m C) 01.29m D)2.09 Em the Ideal Smithsonian about under . ." by values inspired by my antiques objecteralscoped fire, add 100. have even look on!垠 Ented enemies You might infer B friends brokeCommon ; color all . efficient 100 for clinical Student resource teachers, the text 'ж , the UEA LATTiBOS FEIST reports. . drivens.com plan. with the computer science There rating W Normals! . metal and stain. Sure, without scumbledore for processThe forDonaldTrump In a +(GRPOConsumer pid=132981, ip=10.0.0.4) [T0] Recv data episode 0 step 0 from 1 +(SimpleProducer pid=259434) [P1] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(SimpleProducer pid=259434) Rollout example: +(SimpleProducer pid=259434) system +(SimpleProducer pid=259434) Please reason step by step, and put your final answer within \boxed{}. +(SimpleProducer pid=259434) user +(SimpleProducer pid=259434) If $f(x) = -7x^4 + 3x^3 + x - 5$, and $g(x)$ is a polynomial such that the degree of $f(x) + g(x)$ is 1, then what is the degree of $g(x)$? Let's think step by step and output the final answer within \boxed{}. +(SimpleProducer pid=259434) assistant +(SimpleProducer pid=259434) To determine the degree of $g(x)$, we first need to understand what we mean by the degree of a polynomial. The degree of a polynomial is the highest power of $x$ that has a non-zero coefficient. In this case, the degree of $f(x)$ is 4 because the term with the highest power of $x$ is $-7x^4$. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) We are given that the degree of $f(x) + g(x)$ is 1. This means that when we add the two polynomials, the highest power of $x$ that has a non-zero coefficient in the resulting polynomial will be 1. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) Since the degree of $f(x)$ is 4 and we want the degree of $f(x) + g(x)$ to be 1, we need to manipulate $g(x)$ so that when we add it to $f(x)$, the terms with powers greater than 1 cancel out. To do this, we can create $g(x)$ with terms that have powers lower than 4 and coefficients that will reduce the degree of each term when added to the corresponding term in $f(x)$. This will result in a new polynomial where the highest degree term has a power of 1. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) To accomplish this, we need $g(x)$ to have terms with powers from 0 to 3. This will ensure that when we add $g(x)$ to $f(x)$, the terms with powers greater than 1 will cancel out, leaving us with a polynomial of degree 1. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) Therefore, the degree of $g(x)$ should be 3. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) cómo insertar (con el botón pinchinflejar) una imagen muestrairecibo.JPG en este documento, sin相处ovel er a bitmap o jpeg +(SimpleProducer pid=259434) To insert an image titled "muestrairecibo.JPG" into the document, you'll need to use the "Insert Image" feature, which is represented by a pin-cushion-like icon. Here are the steps to insert an image: +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 1. Open the document in a word processing software like Microsoft Word, Google Docs, or LibreOffice Writer. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 2. Make sure the insertion point is where you want to insert the image. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 3. Locate the "Insert Image" or equivalent tool. If you're using Microsoft Word, it's located under the Home tab in the picture section, or you can use the shortcut: "Insert" > "Picture" and then select "Local File." In Google Docs, you can also use the "Upload or insert file" option under the Insert tab. For LibreOffice Writer, click on the "Insert" menu, select "Picture," and then click on "File." +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 4. A dialog box will appear. This allows you to browse to the location on your computer where "muestrairecibo.JPG" is saved. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 5. Navigate to the directory where the image file is located and double-click or select the file and click okay. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 6. The image of muestrairecibo.JPG should now appear in the document at the selected insertion point. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) If you're experiencing trouble locating the "Insert Image" feature, consult your operating system/word processing software user manual or contact their customer support for assistance. Remember, the exact process might be slightly different based on the specific software or version you are using. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) how to add a watermark element below the "image muesbrairecibo" file and that is automatically positioned and shrinks in relation to the image size (to left,right,top,down). +(SimpleProducer pid=259434) To add a watermark below the image "muestrairecibo.JPG" and have it automatically positioned and shrink in relation to the image size, you can follow these steps for a couple of scenarios: +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) **Word Processing Software (e.g., Microsoft Word or Google Docs)** +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 1. Insert the image: Insert the "muestrairecibo.JPG" image into your desired location within your document. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 2. Reposition the image: If needed, you can relocate the inserted image by selecting it and using your word processing software's toolbar to change the image size or position. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 3. Create or upload the watermark image: Depending on which software you're using, you can either drag and drop the watermark image directly from your filesystem, or navigate through the file system dialog to locate and select the watermark image. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) **Google Docs:** Click the "Insert" menu, then select "Image" or "Watermark" +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) **Microsoft Word:** Click the "Insert" tab, then select "Watermark" +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 4. Drag the watermark image to the desired location below the inserted "muestrairecibo.JPG" image. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) 5. Customizing the watermark: +(SimpleProducer pid=259434) - If you need more control over the position and size of the watermark in relation to the image, you might have to crop and resize the watermark file manually for each image or automate this process in a custom solution (chosen from steps below). +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) **Automating the Positioning and Shrinkage (Advanced, Using VBA for Microsoft Word or Google Apps Script for Google Docs)** +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) - **Microsoft Word (using VBA):** You might need to code a VBA macro to detect the image size, create a proportionally sized watermark image based on that size, and place the watermark image in the correct location. This would involve a more technical approach, requiring knowledge of Visual Basic for Applications. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) - **Google Docs (using Google Apps Script):** With Google Apps Script, you can write code to automate the resizing and positioning of the watermark based on the size of the document's images. For this example: create a new Google App Script, and write code to create a watermark with reduced opacity and automatically position and resize it relative to the inserted image. It would require some programming experience. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) In general, custom automation solutions (both VBA for Microsoft Word and Google Apps Script for Google Docs) usually involve breaking the tasks down into several steps and writing code to perform each step where you want the automation to occur – often triggered by a specific action, such as 'on open' or 'after insertion' of an image. +(SimpleProducer pid=259434) +(SimpleProducer pid=259434) Remember to properly save your changes after each step and, if needed, test the watermark placement and resizing with different image sizes. +(SimpleProducer pid=259434) [P1] Send data [('input_ids', torch.Size([2, 8, 2150])), ('attention_mask', torch.Size([2, 8, 2150])), ('action_log_probs', torch.Size([2, 8, 1638])), ('action_mask', torch.Size([2, 8, 1638])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(GRPOConsumer pid=132988, ip=10.0.0.4) [T7] Recv data episode 0 step 0 from 0 [repeated 63x across cluster] +(SimpleProducer pid=259436) [P6] Send data [('input_ids', torch.Size([2, 8, 2692])), ('attention_mask', torch.Size([2, 8, 2692])), ('action_log_probs', torch.Size([2, 8, 2180])), ('action_mask', torch.Size([2, 8, 2180])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(SimpleProducer pid=259437) [P3] Send data [('input_ids', torch.Size([2, 8, 3683])), ('attention_mask', torch.Size([2, 8, 3683])), ('action_log_probs', torch.Size([2, 8, 3171])), ('action_mask', torch.Size([2, 8, 3171])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(GRPOConsumer pid=132981, ip=10.0.0.4) [T0] Recv data episode 0 step 0 from 1 +(GRPOConsumer pid=132981, ip=10.0.0.4) [T0] Recv data episode 0 step 0 from 2 +(SimpleProducer pid=259443) [P4] Send data [('input_ids', torch.Size([2, 8, 3556])), ('attention_mask', torch.Size([2, 8, 3556])), ('action_log_probs', torch.Size([2, 8, 3044])), ('action_mask', torch.Size([2, 8, 3044])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(SimpleProducer pid=259435) [P0] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] +(GRPOConsumer pid=132988, ip=10.0.0.4) [T7] Recv data episode 0 step 0 from 2 [repeated 14x across cluster] +(SimpleProducer pid=259449) [P2] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] [repeated 2x across cluster] +Traceback (most recent call last): + File "/home/duanjunwen/ColossalAI/applications/ColossalChat/rl_example.py", line 202, in + launch_distributed( + File "/home/duanjunwen/ColossalAI/applications/ColossalChat/coati/distributed/launch.py", line 120, in launch_distributed + ray.get([p.loop.remote() for p in procs]) + File "/usr/local/python3.10/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper + return fn(*args, **kwargs) + File "/usr/local/python3.10/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper + return func(*args, **kwargs) + File "/usr/local/python3.10/lib/python3.10/site-packages/ray/_private/worker.py", line 2771, in get + values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout) + File "/usr/local/python3.10/lib/python3.10/site-packages/ray/_private/worker.py", line 919, in get_objects + raise value.as_instanceof_cause() +ray.exceptions.RayTaskError(RuntimeError): ray::GRPOConsumer.loop() (pid=132985, ip=10.0.0.4, actor_id=c8d5c4ebd0eed225bc8efefb01000000, repr=) + File "/home/duanjunwen/ColossalAI/applications/ColossalChat/coati/distributed/consumer.py", line 141, in loop + loss, num_excessive_prompts = self.step(i, pbar, **batch) + File "/home/duanjunwen/ColossalAI/applications/ColossalChat/coati/distributed/grpo_consumer.py", line 391, in step + policy_model_outputs = self.booster.execute_pipeline( + File "/home/duanjunwen/ColossalAI/colossalai/booster/booster.py", line 221, in execute_pipeline + return self.plugin.execute_pipeline(data_iter, model, criterion, optimizer, return_loss, return_outputs) + File "/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py", line 1409, in execute_pipeline + outputs = self.scheduler.forward_backward_step( + File "/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py", line 472, in forward_backward_step + result = self.run_forward_backward(model, data_iter, criterion, optimizer, return_loss, return_outputs) + File "/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py", line 416, in run_forward_backward + input_obj_grad = self.backward_step(optimizer, input_obj, output_obj, output_obj_grad) + File "/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py", line 305, in backward_step + optimizer.backward(output_obj) + File "/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py", line 807, in backward + super().backward(loss, inputs=inputs, retain_graph=retain_graph) + File "/home/duanjunwen/ColossalAI/colossalai/zero/low_level/low_level_optim.py", line 461, in backward + loss.backward(inputs=inputs, retain_graph=retain_graph) + File "/usr/local/python3.10/lib/python3.10/site-packages/torch/_tensor.py", line 581, in backward + torch.autograd.backward( + File "/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/__init__.py", line 347, in backward + _engine_run_backward( + File "/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/graph.py", line 825, in _engine_run_backward + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass + File "/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py", line 307, in apply + return user_fn(self, *args) + File "/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py", line 231, in backward + softmax_logits_2d[torch.arange(0, softmax_logits_2d.shape[0]), masked_target_1d] -= update +RuntimeError: NPU out of memory. Tried to allocate 4.67 GiB (NPU 0; 60.96 GiB total capacity; 32.79 GiB already allocated; 32.79 GiB current active; 3.34 GiB free; 52.71 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation. +(GRPOConsumer pid=132988, ip=10.0.0.4) The attention layers in this model are transitioning from computing the RoPE embeddings internally through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed `position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be removed and `position_embeddings` will be mandatory. [repeated 7x across cluster] +[ERROR] 2025-05-06-22:59:02 (PID:258963, Device:0, RankID:-1) ERR99999 UNKNOWN applicaiton exception +(GRPOConsumer pid=132988, ip=10.0.0.4) [T7] Recv data episode 0 step 0 from 7 [repeated 40x across cluster] +(SimpleProducer pid=259440) [P5] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] diff --git a/applications/ColossalChat/coati/distributed/consumer.py b/applications/ColossalChat/coati/distributed/consumer.py index d04ffae2ff8f..453499f03fd5 100644 --- a/applications/ColossalChat/coati/distributed/consumer.py +++ b/applications/ColossalChat/coati/distributed/consumer.py @@ -57,9 +57,11 @@ def __init__( # self.device = get_current_device() self.device = 'npu' + # self.device = torch.device(f"npu:{torch.npu.current_device()}") self.lr_scheduler = None def setup(self) -> None: + print(f"self.rank {self.rank} self.world_size {self.world_size} self.master_addr {self.master_addr} self.master_port {self.master_port}") launch(self.rank, self.world_size, self.master_addr, self.master_port, local_rank=0) plugin_config = dict(tp_size=1, pp_size=1, precision="bf16", zero_stage=2) @@ -82,7 +84,7 @@ def setup(self) -> None: # Init Hybrid ray process group for i in range(self.num_producers): - cc.init_collective_group(self.world_size + 1, self.rank + 1, group_name=f"sync_data_{i}") + cc.init_collective_group(self.world_size + 1, self.rank + 1, backend='hccl',group_name=f"sync_data_{i}") if self.pp_size > 1: # use hybrid tp + pp if self.tp_rank == 0 and self.dp_rank == 0: diff --git a/applications/ColossalChat/coati/distributed/launch.py b/applications/ColossalChat/coati/distributed/launch.py index a346d1d4fae9..14b39ab21431 100644 --- a/applications/ColossalChat/coati/distributed/launch.py +++ b/applications/ColossalChat/coati/distributed/launch.py @@ -58,6 +58,7 @@ def launch_distributed( core_consumer = ALGO_MAP.get(core_algo, SimpleConsumer) train_dp_size = get_dp_size_fast(num_consumer_procs, plugin_config) + print(f"inference_batch_size {inference_batch_size} num_producers {num_producers} train_batch_size {train_batch_size} train_dp_size {train_dp_size}") assert (inference_batch_size * num_producers) % (train_batch_size * train_dp_size) == 0 dataset_path = dataset_config["path"] @@ -66,9 +67,100 @@ def launch_distributed( num_update_per_episode = num_samples // global_inference_batch_size num_recv_per_update = inference_batch_size // inference_microbatch_size - procs = [] + # ########################################### + # # Old version, may lead colossalai init stuck in multinodes + # ############################################ + # procs = [] + # for i in range(num_producers): + # # producer = SimpleProducer.options(num_gpus=num_proc_per_producer).remote( + # producer = SimpleProducer.options(num_cpus=1, resources={"NPU":num_proc_per_producer}).remote( + # producer_idx=i, + # num_producers=num_producers, + # num_consumer_procs=num_consumer_procs, + # num_episodes=num_episodes, + # batch_size=inference_batch_size, + # dataset_config=dataset_config, + # dataloaders_config=dataloaders_config, + # model_config=inference_model_config, + # generate_config=generate_config, + # tokenizer_config=tokenizer_config, + # microbatch_size=inference_microbatch_size, + # backend=inference_backend, + # num_generations=num_generations, + # consumer_plugin_config=plugin_config, + # ) + # procs.append(producer) + # generate_config_consumer = copy.deepcopy(generate_config) + # generate_config_consumer.update( + # dict( + # backend=inference_backend, + # ) + # ) + # for i in range(num_consumer_procs): + # # consumer = core_consumer.options(num_gpus=1).remote( + # consumer = core_consumer.options(num_cpus=1, resources={"NPU":1}).remote( + # num_producers=num_producers, + # num_episodes=num_episodes, + # rank=i, + # world_size=num_consumer_procs, + # master_addr=master_addr, + # master_port=master_port, + # num_update_per_episode=num_update_per_episode, + # num_recv_per_update=num_recv_per_update, + # batch_size=train_batch_size, + # model_config=train_model_config, + # plugin_config=plugin_config, + # minibatch_size=train_minibatch_size, + # generate_config=generate_config_consumer, + # grpo_config=grpo_config, + # num_generations=num_generations, + # project_name=project_name, + # save_interval=save_interval, + # save_dir=save_dir, + # ) + # procs.append(consumer) + # ray.get([p.setup.remote() for p in procs]) + # ray.get([p.loop.remote() for p in procs]) + + ########################################### + # New version, assign master ip for colossalai & vllm respectively + ########################################### + nodes = ray.nodes() + node_info = { + node["NodeID"]: { + # "num_gpus": node["Resources"].get("GPU", 0), + "num_gpus": node["Resources"].get("NPU", 0), + "address": node["NodeManagerAddress"], + } # Default to 0 if no GPUs are available + for node in nodes + } + print(f"node_info {node_info}") + gpu_to_node_id = [] + gpu_to_ip_address = [] + for node_id in node_info: + for idx in range(int(node_info[node_id]["num_gpus"])): # use num_gpus instead of num_npus + gpu_to_node_id.append(node_id) + gpu_to_ip_address.append(node_info[node_id]["address"]) + print(f"node_info {node_info} \n gpu_to_node_id {gpu_to_node_id} \n gpu_to_ip_address {gpu_to_ip_address} \n") + + producer_procs = [] + for i in range(num_producers): - producer = SimpleProducer.options(num_gpus=num_proc_per_producer).remote( + node_id = gpu_to_node_id[0] + producer_ip_address = gpu_to_ip_address[0] + for _ in range(num_proc_per_producer): + gpu_to_node_id.pop(0) + gpu_to_ip_address.pop(0) + print(f"Schedual Producer P[{i}] which requires {num_proc_per_producer} GPUs on node {producer_ip_address}") + + producer = SimpleProducer.options( + num_cpus=1, + resources={"NPU":num_proc_per_producer}, + scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=node_id, + soft=False, + ), + ).remote( producer_idx=i, num_producers=num_producers, num_consumer_procs=num_consumer_procs, @@ -84,20 +176,36 @@ def launch_distributed( num_generations=num_generations, consumer_plugin_config=plugin_config, ) - procs.append(producer) + producer_procs.append(producer) + ray.get([p.setup.remote() for p in producer_procs]) generate_config_consumer = copy.deepcopy(generate_config) generate_config_consumer.update( dict( backend=inference_backend, ) ) + consumer_master_ip_address = gpu_to_ip_address[0] + print(f"Use {consumer_master_ip_address} as master address for torch DDP.") + consumer_procs = [] for i in range(num_consumer_procs): - consumer = core_consumer.options(num_gpus=1).remote( + node_id = gpu_to_node_id[0] + consumer_ip_address = gpu_to_ip_address[0] + gpu_to_node_id.pop(0) + gpu_to_ip_address.pop(0) + print(f"Schedual Consumer T[{i}] which requires 1 GPUs on node {consumer_ip_address}") + consumer = core_consumer.options( + resources={"NPU":1}, + scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=node_id, + soft=False, + ), + ).remote( num_producers=num_producers, num_episodes=num_episodes, rank=i, world_size=num_consumer_procs, - master_addr=master_addr, + # master_addr=master_addr, + master_addr=consumer_master_ip_address, master_port=master_port, num_update_per_episode=num_update_per_episode, num_recv_per_update=num_recv_per_update, @@ -112,6 +220,6 @@ def launch_distributed( save_interval=save_interval, save_dir=save_dir, ) - procs.append(consumer) - ray.get([p.setup.remote() for p in procs]) - ray.get([p.loop.remote() for p in procs]) + consumer_procs.append(consumer) + ray.get([p.setup.remote() for p in consumer_procs]) + ray.get([p.loop.remote() for p in (producer_procs + consumer_procs)]) diff --git a/applications/ColossalChat/coati/distributed/producer.py b/applications/ColossalChat/coati/distributed/producer.py index c45ddd450fb4..f1cc583f6feb 100644 --- a/applications/ColossalChat/coati/distributed/producer.py +++ b/applications/ColossalChat/coati/distributed/producer.py @@ -72,6 +72,7 @@ def __init__( ) # self.device = get_current_device() self.device = 'npu' + # self.device = torch.device(f"npu:{torch.npu.current_device()}") # init backend if backend in BACKEND_MAP: @@ -120,7 +121,7 @@ def loop(self) -> None: ] * outputs["input_ids"].size(0) ).to(outputs["input_ids"].device) - outputs = pre_send(outputs) + # outputs = pre_send(outputs) ray_broadcast_tensor_dict( outputs, src=0, device=self.device, group_name=f"sync_data_{self.producer_idx}" ) diff --git a/applications/ColossalChat/fusion_result.json b/applications/ColossalChat/fusion_result.json new file mode 100644 index 000000000000..ec747fa47ddb --- /dev/null +++ b/applications/ColossalChat/fusion_result.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/applications/ColossalChat/kernel_meta/buildPidInfo.json b/applications/ColossalChat/kernel_meta/buildPidInfo.json index 7194c917d7ed..804df5b51270 100644 --- a/applications/ColossalChat/kernel_meta/buildPidInfo.json +++ b/applications/ColossalChat/kernel_meta/buildPidInfo.json @@ -1,6 +1,14 @@ [ [ - 3383334, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_18208839462778721971" + 1287410, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_5195361436236851103" + ], + [ + 1287412, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_14660501106417545923" + ], + [ + 1287422, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_10947606003133373928" ] ] \ No newline at end of file diff --git a/applications/ColossalChat/rl_example.py b/applications/ColossalChat/rl_example.py index 18948a569642..fe1663500faf 100644 --- a/applications/ColossalChat/rl_example.py +++ b/applications/ColossalChat/rl_example.py @@ -155,7 +155,7 @@ enforce_eager=True, enable_chunked_prefill=True, max_model_len=args.max_new_tokens + args.max_prompt_tokens, - tensor_parallel_size=2, + tensor_parallel_size=1, ) ) generate_config.update( @@ -223,10 +223,10 @@ # "zero_stage": 2, # }, # for zero plugin_config={ - "tp_size": 2, - "pp_size": 2, + "tp_size": 8, + "pp_size": 3, "microbatch_size": max( - 1, args.train_microbatch_size // 2 + 1, args.train_microbatch_size // 3 ), # microbatch size should be set to train_microbatch_size // pp_size "zero_stage": 1, "max_norm": 1.0, diff --git a/applications/ColossalChat/tests/test_hybrid.py b/applications/ColossalChat/tests/test_hybrid.py new file mode 100644 index 000000000000..ed3e22351761 --- /dev/null +++ b/applications/ColossalChat/tests/test_hybrid.py @@ -0,0 +1,143 @@ +import torch +import torch.distributed as dist +from coati.dataset.loader import RawConversationDataset +from torch.utils.data import Dataset +from tqdm import tqdm +from transformers import AutoTokenizer, Qwen2ForCausalLM + +import colossalai +from colossalai.accelerator import get_accelerator +from colossalai.booster import Booster +from colossalai.booster.plugin import HybridParallelPlugin, Plugin +from colossalai.cluster import DistCoordinator +from colossalai.nn.optimizer import HybridAdam + +BATCH_SIZE = 4 +NUM_EPOCHS = 3 +LEARNING_RATE = 2e-5 +GRADIENT_ACCUMULATION_STEPS = 1 +DATA_PATH = "/home/duanjunwen/datasets/math_dataset.jsonl" +MODEL_PATH = "/home/duanjunwen/models/Qwen/Qwen2.5-14B" +Device = torch.device("npu" if torch.npu.is_available() else "cpu") + +class RandomDataset(Dataset): + def __init__(self, num_samples, sequence_length, vocab_size=10000): + self.num_samples = num_samples + self.sequence_length = sequence_length + self.vocab_size = vocab_size + self.input_idx = torch.randint(0, vocab_size, (num_samples, sequence_length)) + self.attention_mask = torch.randint(0, 2, (num_samples, sequence_length), dtype=torch.long) + + def __len__(self): + return self.num_samples + + def __getitem__(self, idx): + return {"input_ids": self.input_idx[idx], "attention_mask": self.attention_mask[idx]} + +def load_model_and_tokenizer(): + attn_impl = "eager" if get_accelerator().name == "npu" else "flash_attention_2" + tokenizer = AutoTokenizer.from_pretrained( + MODEL_PATH, + trust_remote_code=True, + attn_implementation=attn_impl, + ) + model = Qwen2ForCausalLM.from_pretrained(MODEL_PATH, trust_remote_code=True) + return tokenizer, model + +def all_reduce_mean(loss: torch.Tensor, plugin: Plugin) -> torch.Tensor: + loss = loss.data + group = getattr(plugin, "dp_group", None) + dist.all_reduce(loss, group=group) + return loss / dist.get_world_size(group) + +def test_hybrid_qwen(): + colossalai.launch_from_torch() + get_accelerator() + coordinator = DistCoordinator() + tokenizer, model = load_model_and_tokenizer() + # dataset = RandomDataset(num_samples=100, sequence_length=2304) + dataset = RawConversationDataset(tokenizer, DATA_PATH, 1024, system_prompt="Please reason step by step, and put your final answer within \\boxed{}.") + # dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True) + + optimizer = HybridAdam(model.parameters(), lr=LEARNING_RATE) + plugin = HybridParallelPlugin( + tp_size=8, + pp_size=1, + precision="bf16", + zero_stage=2, + cpu_offload=True, + ) + # plugin = HybridParallelPlugin(tp_size=2, pp_size=2, precision="bf16", zero_stage=1, num_microbatches=4, enable_flash_attention=True) + + dataloader = plugin.prepare_dataloader( + dataset=dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + ) + + booster = Booster(plugin=plugin) + + model, optimizer, _, dataloader, _ = booster.boost(model, optimizer, None, dataloader) + + def is_master(): + if isinstance(plugin, HybridParallelPlugin) and plugin.pp_size > 1: + return coordinator.rank == coordinator.world_size - 1 + return coordinator.is_master() + + ##### + # train + ##### + model.train() + + for epoch in range(NUM_EPOCHS): + if booster.plugin.pp_size > 1: + data_iter = iter(dataloader) + step_bar = tqdm( + range(len(dataloader)), + desc="Step", + disable=not is_master(), + ) + for step in step_bar: + print(f"data_iter {data_iter}") + outputs = booster.execute_pipeline( + data_iter, + model, + criterion=lambda outputs, inputs: outputs[0], + optimizer=optimizer, + return_loss=True, + ) + loss = outputs["loss"] + if booster.plugin.stage_manager.is_last_stage(): + global_loss = all_reduce_mean(loss, plugin) + + optimizer.step() + + if booster.plugin.stage_manager.is_last_stage(): + grad_norm = optimizer.get_grad_norm() + step_bar.set_postfix({"loss": global_loss.item(), "grad_norm": grad_norm}) + + optimizer.step() + optimizer.zero_grad() + else: + total_loss = 0 + for step, batch in enumerate(dataloader): + input_ids = batch["input_ids"].to(device=model.module.device) + attention_mask = batch["attention_mask"].to(device=model.module.device) + outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids) + loss = outputs.loss + print(f"loss {loss}") + loss = loss / GRADIENT_ACCUMULATION_STEPS + booster.backward(loss, optimizer) + print(f"finish backward") + if (step + 1) % GRADIENT_ACCUMULATION_STEPS == 0: + optimizer.step() + optimizer.zero_grad() + print(f"finish optimizer step") + + total_loss += loss.item() + + print(f"Epoch {epoch + 1}, Loss: {total_loss / len(dataloader)}") + +if __name__ == "__main__": + test_hybrid_qwen() diff --git a/applications/ColossalChat/tests/test_ray.py b/applications/ColossalChat/tests/test_ray.py new file mode 100644 index 000000000000..ca2f1456adef --- /dev/null +++ b/applications/ColossalChat/tests/test_ray.py @@ -0,0 +1,88 @@ +import ray +import time +import ray.util.collective as cc +import torch +from coati.distributed.comm import ray_broadcast_object, ray_broadcast_tensor_dict + +from colossalai.testing import parameterize + +@ray.remote(num_cpus=1, num_gpus=0, resources={"NPU": 1}) +class Worker: + def __init__(self, rank, world_size): + self.rank = rank + self.world_size = world_size + self.group_name = "default" + cc.init_collective_group(world_size, rank, backend="hccl", group_name=self.group_name) + def run_ray_broadcast_object(self, obj, src, device): + # ray_broadcast_object + received_obj = ray_broadcast_object(obj, src, device, group_name=self.group_name) + return received_obj + + def run_ray_broadcast_tensor_dict(self, tensor_dict, src, device): + # ray_broadcast_tensor_dict + received_dict = ray_broadcast_tensor_dict(tensor_dict, src, device, group_name=self.group_name) + return received_dict + + def destroy_worker(self): + cc.destroy_collective_group(self.group_name) + +@parameterize( + "test_config", + [ + { + "precision": torch.bfloat16, + "device": "npu", + "num_devices": 8, + }, + ], +) +def test_comm(test_config): + #ray.init() + ray.init(address="local", namespace="ray-example") + # ray.init(_node_ip_address='10.0.0.5', namespace="ray-example") + + src = 0 + device = test_config["device"] + # create 4 + workers = [Worker.remote(i, test_config["num_devices"]) for i in range(test_config["num_devices"])] + + ############# + # 1. test ray_broadcast_object + ############# + # init broadcast_object data + test_obj = {"data": torch.tensor([1, 2, 3]), "message": "hello"} + + # run run_ray_broadcast_object + results = [worker.run_ray_broadcast_object.remote(test_obj, src, device) for worker in workers] + + time.sleep(60) + # get result + results = ray.get(results) + + for i, result in enumerate(results): + print(f"ray_broadcast_object Rank {i} received object: {result}") + + ############# + # 2. test ray_broadcast_tensor_dict + ############# + test_tensor_dict = { + "tensor1": torch.tensor([1, 2, 3], device=device), + "tensor2": torch.tensor([[4, 5], [6, 7]], device=device), + } + + # run ray_broadcast_tensor_dict + results = [worker.run_ray_broadcast_tensor_dict.remote(test_tensor_dict, src, device) for worker in workers] + + # get result + results = ray.get(results) + + for i, result in enumerate(results): + print(f"run_ray_broadcast_tensor_dict Rank {i} received object: {result}") + + # destory workers + for worker in workers: + worker.destroy_worker.remote() + ray.shutdown() + +if __name__ == "__main__": + test_comm() \ No newline at end of file diff --git a/applications/ColossalChat/tests/test_vllm.py b/applications/ColossalChat/tests/test_vllm.py new file mode 100644 index 000000000000..325ddc0a9693 --- /dev/null +++ b/applications/ColossalChat/tests/test_vllm.py @@ -0,0 +1,27 @@ +from vllm import LLM, SamplingParams +import torch +import argparse + +parser = argparse.ArgumentParser(description='VLLM args.') +parser.add_argument("-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. ") +parser.add_argument("-l", "--max_length", type=int, default=8192, help="Max sequence length") +parser.add_argument("-tp", "--tp_size", type=int, default=8, help="Gpu nums") +parser.add_argument("-pp", "--pp_size", type=int, default=2, help="Gpu nums") +parser.add_argument("-t", "--temperature", type=float, default=0.8, help="Temperature") +parser.add_argument("--top_p", type=float, default=0.95, help="Top p") +parser.add_argument("-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. ") +args = parser.parse_args() + +# Create a sampling params object. +sampling_params = SamplingParams(temperature=args.temperature, top_p=args.top_p, max_tokens=args.max_length) + +# Create an LLM. +llm = LLM(model=args.model_path, max_model_len=args.max_length, tensor_parallel_size=args.tp_size, pipeline_parallel_size=args.pp_size) +# Generate texts from the prompts. The output is a list of RequestOutput objects +# that contain the prompt, generated text, and other information. +outputs = llm.generate(args.input_texts, sampling_params) +# Print the outputs. +for output in outputs: + prompt = output.prompt + generated_text = output.outputs[0].text + print(f"Prompt: {prompt!r}, Generated text: {generated_text}") \ No newline at end of file diff --git a/applications/ColossalChat/tests/test_vllm_multinode.py b/applications/ColossalChat/tests/test_vllm_multinode.py new file mode 100644 index 000000000000..0434c48e1e92 --- /dev/null +++ b/applications/ColossalChat/tests/test_vllm_multinode.py @@ -0,0 +1,108 @@ +""" +This example shows how to use Ray Data for running offline batch inference +distributively on a multi-nodes cluster. + +Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html +""" + +from typing import Any, Dict, List + +import numpy as np +import ray +from packaging.version import Version +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + +from vllm import LLM, SamplingParams + +assert Version(ray.__version__) >= Version( + "2.22.0"), "Ray version must be at least 2.22.0" + +# Create a sampling params object. +sampling_params = SamplingParams(temperature=0.8, top_p=0.95) + +# Set tensor parallelism per instance. +tensor_parallel_size = 1 + +# Set number of instances. Each instance will use tensor_parallel_size GPUs. +num_instances = 1 + + +# Create a class to do batch inference. +class LLMPredictor: + + def __init__(self): + # Create an LLM. + self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf", + tensor_parallel_size=tensor_parallel_size) + + def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]: + # Generate texts from the prompts. + # The output is a list of RequestOutput objects that contain the prompt, + # generated text, and other information. + outputs = self.llm.generate(batch["text"], sampling_params) + prompt: List[str] = [] + generated_text: List[str] = [] + for output in outputs: + prompt.append(output.prompt) + generated_text.append(' '.join([o.text for o in output.outputs])) + return { + "prompt": prompt, + "generated_text": generated_text, + } + + +# Read one text file from S3. Ray Data supports reading multiple files +# from cloud storage (such as JSONL, Parquet, CSV, binary format). +ds = ray.data.read_text("s3://anonymous@air-example-data/prompts.txt") + + +# For tensor_parallel_size > 1, we need to create placement groups for vLLM +# to use. Every actor has to have its own placement group. +def scheduling_strategy_fn(): + # One bundle per tensor parallel worker + pg = ray.util.placement_group( + [{ + "GPU": 1, + "CPU": 1 + }] * tensor_parallel_size, + strategy="STRICT_PACK", + ) + return dict(scheduling_strategy=PlacementGroupSchedulingStrategy( + pg, placement_group_capture_child_tasks=True)) + + +resources_kwarg: Dict[str, Any] = {} +if tensor_parallel_size == 1: + # For tensor_parallel_size == 1, we simply set num_gpus=1. + resources_kwarg["num_gpus"] = 1 +else: + # Otherwise, we have to set num_gpus=0 and provide + # a function that will create a placement group for + # each instance. + resources_kwarg["num_gpus"] = 0 + resources_kwarg["ray_remote_args_fn"] = scheduling_strategy_fn + +# Apply batch inference for all input data. +ds = ds.map_batches( + LLMPredictor, + # Set the concurrency to the number of LLM instances. + concurrency=num_instances, + # Specify the batch size for inference. + batch_size=32, + **resources_kwarg, +) + +# Peek first 10 results. +# NOTE: This is for local testing and debugging. For production use case, +# one should write full result out as shown below. +outputs = ds.take(limit=10) +for output in outputs: + prompt = output["prompt"] + generated_text = output["generated_text"] + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") + +# Write inference output data out as Parquet files to S3. +# Multiple files would be written to the output destination, +# and each task would write one or more files separately. +# +# ds.write_parquet("s3://") \ No newline at end of file From d67d7b0a856cfe4e24ac2fc3aa4eb31ab55eed66 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Thu, 15 May 2025 17:32:51 +0800 Subject: [PATCH 03/24] [feat] enlarge seqlen --- .../ColossalChat/.nfs00000000078104b100001d70 | 389 ------------------ .../coati/distributed/consumer.py | 19 +- .../coati/distributed/grpo_consumer.py | 5 +- .../coati/distributed/inference_backend.py | 5 +- .../ColossalChat/coati/distributed/launch.py | 4 +- .../coati/distributed/producer.py | 3 +- .../ColossalChat/coati/distributed/utils.py | 20 +- applications/ColossalChat/fusion_result.json | 1 - .../kernel_meta/buildPidInfo.json | 68 ++- applications/ColossalChat/rl_example.py | 11 +- .../ColossalChat/tests/test_ray_vllm.py | 96 +++++ 11 files changed, 208 insertions(+), 413 deletions(-) delete mode 100755 applications/ColossalChat/.nfs00000000078104b100001d70 delete mode 100644 applications/ColossalChat/fusion_result.json create mode 100644 applications/ColossalChat/tests/test_ray_vllm.py diff --git a/applications/ColossalChat/.nfs00000000078104b100001d70 b/applications/ColossalChat/.nfs00000000078104b100001d70 deleted file mode 100755 index 5db53e4f6063..000000000000 --- a/applications/ColossalChat/.nfs00000000078104b100001d70 +++ /dev/null @@ -1,389 +0,0 @@ -2025-05-06 22:50:50,843 WARNING collective.py:22 -- NCCL seems unavailable. Please install Cupy following the guide at: https://docs.cupy.dev/en/stable/install.html. -/home/duanjunwen/ColossalAI/colossalai/utils/safetensors.py:13: UserWarning: Please install the latest tensornvme to use async save. pip install git+https://github.com/hpcaitech/TensorNVMe.git - warnings.warn( -/usr/local/python3.10/lib/python3.10/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. - warn("The installed version of bitsandbytes was compiled without GPU support. " -/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/normalization.py:48: UserWarning: Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel - warnings.warn("Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel") -'NoneType' object has no attribute 'cadam32bit_grad_fp32' -2025-05-06 22:51:04,272 INFO worker.py:1654 -- Connecting to existing Ray cluster at address: 10.0.0.3:6379... -2025-05-06 22:51:04,285 INFO worker.py:1841 -- Connected to Ray cluster. -(pid=259440) NCCL seems unavailable. Please install Cupy following the guide at: https://docs.cupy.dev/en/stable/install.html. -(pid=132985, ip=10.0.0.4) /home/duanjunwen/ColossalAI/colossalai/utils/safetensors.py:13: UserWarning: Please install the latest tensornvme to use async save. pip install git+https://github.com/hpcaitech/TensorNVMe.git -(pid=132985, ip=10.0.0.4) warnings.warn( -(pid=259440) /usr/local/python3.10/lib/python3.10/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. -(pid=259440) warn("The installed version of bitsandbytes was compiled without GPU support. " -(pid=132987, ip=10.0.0.4) /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/normalization.py:48: UserWarning: Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel -(pid=132987, ip=10.0.0.4) warnings.warn("Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel") -(GRPOConsumer pid=132981, ip=10.0.0.4) Loading checkpoint shards: 0%| | 0/4 [00:00 -(SimpleProducer pid=259440) INFO 05-06 22:51:32 config.py:549] This model supports multiple tasks: {'embed', 'classify', 'reward', 'score', 'generate'}. Defaulting to 'generate'. -(SimpleProducer pid=259435) INFO 05-06 22:51:32 config.py:549] This model supports multiple tasks: {'embed', 'score', 'reward', 'classify', 'generate'}. Defaulting to 'generate'. -(SimpleProducer pid=259436) INFO 05-06 22:51:32 config.py:549] This model supports multiple tasks: {'classify', 'generate', 'score', 'reward', 'embed'}. Defaulting to 'generate'. -(GRPOConsumer pid=132981, ip=10.0.0.4) [extension] Loading the JIT-built cpu_adam_arm kernel during runtime now -(GRPOConsumer pid=132985, ip=10.0.0.4) Using GRPO config: {'lr': 1e-06, 'train_microbatch_size': 8, 'beta': 0.01, 'loss_variation': 'sample_level', 'reward_fn_type': 'boxed'} [repeated 7x across cluster] -(SimpleProducer pid=259436) INFO 05-06 22:51:32 config.py:1555] Chunked prefill is enabled with max_num_batched_tokens=2048. [repeated 7x across cluster] -(SimpleProducer pid=259436) INFO 05-06 22:51:32 llm_engine.py:234] Initializing a V0 LLM engine (v0.7.3) with config: model='/home/duanjunwen/models/Qwen/Qwen2.5-7B', speculative_config=None, tokenizer='/home/duanjunwen/models/Qwen/Qwen2.5-7B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, device_config=npu, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=/home/duanjunwen/models/Qwen/Qwen2.5-7B, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=False, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"splitting_ops":[],"compile_sizes":[],"cudagraph_capture_sizes":[],"max_capture_size":0}, use_cached_outputs=False,  [repeated 7x across cluster] -(SimpleProducer pid=259436) WARNING 05-06 22:51:33 utils.py:2262] Methods add_lora,add_prompt_adapter,cache_config,compilation_config,current_platform,list_loras,list_prompt_adapters,load_config,pin_lora,pin_prompt_adapter,remove_lora,remove_prompt_adapter not implemented in  [repeated 7x across cluster] -(GRPOConsumer pid=132981, ip=10.0.0.4) [extension] Time taken to load cpu_adam_arm op: 0.1460132598876953 seconds -(SimpleProducer pid=259437) INFO 05-06 22:52:06 executor_base.py:111] # npu blocks: 3809, # CPU blocks: 585 -(SimpleProducer pid=259437) INFO 05-06 22:52:06 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.03x -(GRPOConsumer pid=132987, ip=10.0.0.4) [extension] Loading the JIT-built cpu_adam_arm kernel during runtime now [repeated 7x across cluster] -(GRPOConsumer pid=132987, ip=10.0.0.4) [extension] Time taken to load cpu_adam_arm op: 0.16289782524108887 seconds [repeated 7x across cluster] -(SimpleProducer pid=259437) INFO 05-06 22:52:08 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 16.79 seconds -(SimpleProducer pid=259449) INFO 05-06 22:52:12 executor_base.py:111] # npu blocks: 3809, # CPU blocks: 585 -(SimpleProducer pid=259449) INFO 05-06 22:52:12 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.03x -(SimpleProducer pid=259449) INFO 05-06 22:52:14 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 15.55 seconds -(SimpleProducer pid=259440) INFO 05-06 22:52:17 executor_base.py:111] # npu blocks: 3809, # CPU blocks: 585 -(SimpleProducer pid=259440) INFO 05-06 22:52:17 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.03x -(SimpleProducer pid=259440) INFO 05-06 22:52:18 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 16.44 seconds -(SimpleProducer pid=259435) INFO 05-06 22:52:20 executor_base.py:111] # npu blocks: 3810, # CPU blocks: 585 [repeated 2x across cluster] -(SimpleProducer pid=259435) INFO 05-06 22:52:20 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.06x [repeated 2x across cluster] -(SimpleProducer pid=259435) INFO 05-06 22:52:22 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 18.77 seconds [repeated 2x across cluster] -(SimpleProducer pid=259445) INFO 05-06 22:52:27 executor_base.py:111] # npu blocks: 3810, # CPU blocks: 585 -(SimpleProducer pid=259445) INFO 05-06 22:52:27 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.06x -(GRPOConsumer pid=132981, ip=10.0.0.4) [05/06/25 22:52:34] INFO colossalai - colossalai - INFO: -(GRPOConsumer pid=132981, ip=10.0.0.4) /home/duanjunwen/ColossalAI/colossalai/initialize.py:75 launch -(GRPOConsumer pid=132981, ip=10.0.0.4) INFO colossalai - colossalai - INFO: Distributed environment is -(GRPOConsumer pid=132981, ip=10.0.0.4) initialized, world size: 8 -(SimpleProducer pid=259445) INFO 05-06 22:52:29 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 16.97 seconds -(SimpleProducer pid=259436) INFO 05-06 22:52:36 executor_base.py:111] # npu blocks: 3810, # CPU blocks: 585 -(SimpleProducer pid=259436) INFO 05-06 22:52:36 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.06x -(SimpleProducer pid=259436) INFO 05-06 22:52:38 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 17.26 seconds -(SimpleProducer pid=259443) INFO 05-06 22:52:49 executor_base.py:111] # npu blocks: 3810, # CPU blocks: 585 -(SimpleProducer pid=259443) INFO 05-06 22:52:49 executor_base.py:116] Maximum concurrency for 4096 tokens per request: 119.06x -(SimpleProducer pid=259443) INFO 05-06 22:52:51 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 17.65 seconds -(SimpleProducer pid=259435) [P0] num_valid_microbatches 468, nmb: 4, dl: 468 -(GRPOConsumer pid=132981, ip=10.0.0.4) Consumer0 num_update: 117, num_recv: 4, nmb: 1 -(GRPOConsumer pid=132981, ip=10.0.0.4) [T0] Recv data episode 0 step 0 from 0 -(SimpleProducer pid=259436) [P6] Send data [('input_ids', torch.Size([2, 8, 2654])), ('attention_mask', torch.Size([2, 8, 2654])), ('action_log_probs', torch.Size([2, 8, 2142])), ('action_mask', torch.Size([2, 8, 2142])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(SimpleProducer pid=259445) [P7] num_valid_microbatches 468, nmb: 4, dl: 468 [repeated 7x across cluster] -(GRPOConsumer pid=132988, ip=10.0.0.4) Consumer7 num_update: 117, num_recv: 4, nmb: 1 [repeated 7x across cluster] -(GRPOConsumer pid=132988, ip=10.0.0.4) [T7] Recv data episode 0 step 0 from 0 [repeated 7x across cluster] -(SimpleProducer pid=259440) [P5] Send data [('input_ids', torch.Size([2, 8, 3944])), ('attention_mask', torch.Size([2, 8, 3944])), ('action_log_probs', torch.Size([2, 8, 3432])), ('action_mask', torch.Size([2, 8, 3432])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(SimpleProducer pid=259449) [P2] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(SimpleProducer pid=259435) [P0] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] [repeated 4x across cluster] -(SimpleProducer pid=259434) Rollout example: -(SimpleProducer pid=259434) system -(SimpleProducer pid=259434) Please reason step by step, and put your final answer within \boxed{}. -(SimpleProducer pid=259434) user -(SimpleProducer pid=259434) Regular hexagon $ABCDEF$ is divided into six smaller equilateral triangles, such as $\triangle ABG$, shown in boldface in the diagram. By connecting every other vertex, we obtain a larger equilateral triangle $\triangle ACE$, also shown in boldface. Compute the ratio $[\triangle ABG]/[\triangle ACE]$. [asy] -(SimpleProducer pid=259434) size(150); defaultpen(linewidth(0.8)); dotfactor=5; -(SimpleProducer pid=259434) pair[] hex = new pair[6]; -(SimpleProducer pid=259434) string[] hexlabels = {"$C$","$B$","$A$","$F$","$E$","$D$"}; -(SimpleProducer pid=259434) hexlabels.cyclic=true; -(SimpleProducer pid=259434) hex[0] = dir(0); -(SimpleProducer pid=259434) for(int i = 1; i <= 6; ++i){ -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) hex[i] = dir(60*i); -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) draw(hex[i] -- hex[i-1]); -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) dot(hexlabels[i],hex[i],hex[i]); -(SimpleProducer pid=259434) } -(SimpleProducer pid=259434) draw(hex[0]--hex[3]); draw(hex[1]--hex[4]); draw(hex[2]--hex[5]); -(SimpleProducer pid=259434) draw(hex[0]--hex[2]--hex[4]--cycle,linewidth(1.3)); -(SimpleProducer pid=259434) draw(hex[1]--hex[2]--(0,0)--cycle,linewidth(1.3)); -(SimpleProducer pid=259434) dot("$G$",(0,0),2*S); -(SimpleProducer pid=259434) [/asy] Let's think step by step and output the final answer within \boxed{}. -(SimpleProducer pid=259434) assistant -(SimpleProducer pid=259434) To compute the ratio $[\triangle ABG]/[\triangle ACE]$, we can use the formula for the area of an equilateral triangle. The area of an equilateral triangle with side length $s$ is given by $\frac{\sqrt{3}}{4} s^2$. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) In triangle $ ACE$, the side length is $s_1$, and in triangle $ ABG$, the side length is $s_2$. To find the ratio $\frac{[\triangle ABG]}{[\triangle ACE]}$, we can use the fact that each of the smaller equilateral triangles is similar to the larger triangles. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) Using this similarity relationship, we can express the side lengths of the triangle in terms of the length of the side of the regular hexagon, $s$. Since $AC$ is a diagonal of the hexagon, it is equal to $2s$. Therefore, we have $s_1 = 2s$. Similarly, $BG$ is half the length of $AC$, so $s_2 = \frac{1}{2}s_1 = s$. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) Now we can calculate the areas of the triangles using the formula for the area of an equilateral triangle. We have: -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) $[\triangle ABG] = \frac{\sqrt{3}}{4} s_2^2 = \frac{\sqrt{3}}{4} s^2$ -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) $[\triangle ACE] = \frac{\sqrt{3}}{4} s_1^2 = \frac{\sqrt{3}}{4} (2s)^2 = 4 \frac{\sqrt{3}}{4} s^2 = \sqrt{3} s^2$ -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) Thus, the ratio is: -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) $\frac{[\triangle ABG]}{[\triangle ACE]} = \frac{\frac{\sqrt{3}}{4} s^2}{\sqrt{3} s^2} = \frac{1}{4}$ -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) So the answer is $\boxed{1/4}$. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) jifss - Factoring Example Number 1 (#17)Sheila uses rewrite congruence notation to write.info@... er Exponential notation for. tables and graphs. These notation, number, and algebra to represent information. ppt Download 7-May-2018 positano.it supportive media and humanist a are. . . . transforms to 1000000 people a year.We've made a. moving up to new social and physical challenges.Now the human. world numerical notation for. But the majority of numerals look. It our socio-cultural environment using Supernormality Theory. in essence. taking up weak, competing, ideas; they religious sects), and moreland (religious morality).taking. the Project Team include Ian Goldin. and Mike Hammond.adoption of more rational. . sum of themFind out what. time, location visiting as the day of ReFESA. Numerical notation for. CFCSC Technology Standards Explained. What are they up to?? . rounded to two decimal places, in vertrite.2 ., 34 Numerical notation for. 25 100 306.14. 8.32. 2.725 158.89. Practice Grid Version. 6.taking these developments a step further.And a recent study in a major. economic journal. . but we need to explain what exactly has changed. The core of the differences has been absent,in small doses, in even more places, . . of the U.S.professor with documentation. . . at least that it. counting without notation. for. . Allies’ Strategy PubUniver. Why? advantages. University. of. London. time. RajeshR., KenHyett and Laura. . million deaths in French- . religion." Choose correctly. [ além. pronominal] .CONQUIS. abor setting up for sub-contract, sub- and. become. Numerical notation for. 2 t Separate 4-m systems?. Mental division n. of decimal points, if necessary.The woman has exchanged a. and Certificates of 1. Our own internal. "This ability for a fast moving physical object to. numerical notation for. Original WOFR information. then doing so on. Words using at least one digital. and signals digital signals neer 1.0. . Very soon, they will learn other forms of numerical notation for. A third problem is particularly.More information . NBP09019 - AS Lighthouse Pilot 10 shipping at home. Another example could strive. The Muslim world possibly solu-. . situated. within. . 10: (12.3482 × 2^. muito bem para fazer algo; is a market. . SYD 01:30 attendance women. Learning the types of inactive buscarGood book. PRP 2010 de purchase. Mathematical -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) jifss - Factoring Example Number 1 (#17)Sheila uses rewrite congruence notation to write.info@... er Exponential notation for. tables and graphs. These notation, number, and algebra to represent information. ppt Download 7-May-2018 positano.it supportive media and humanist a are. . . . transforms to 1000000 people a year.We've made a. moving up to new social and physical challenges.Now the human. world numerical notation for. But the majority of numerals look. It our socio-cultural environment using Supernormality Theory. in essence. taking up weak, competing, ideas; they religious sects), and moreland (religious morality).taking. the Project Team include Ian Goldin. and Mike Hammond.adoption of more rational. . sum of themFind out what. time, location visiting as the day of ReFESA. Numerical notation for. CFCSC Technology Standards Explained. What are they up to?? . rounded to two decimal places, in vertrite.2 ., 34 Numerical notation for. 25 100 306.14. 8.32. 2.725 158.89. Practice Grid Version. 6.taking these developments a step further.And a recent study in a major. economic journal. . but we need to explain what exactly has changed. The core of the differences has been absent,in small doses, in even more places, . . on an international learning and distance course Series.line are initially incomplete.Achre kingdoms. providers. . not just prestige. . Our societyeven. . uses this. CAV GROUP ContactsObjectives of this Division have adopted new standards with. the top-down manner -- using. . essay on the scientific and. so achieving longer deserve 100ams in New York City.is far more important' ' . saying' ' . Video Games: Affect can become. Numerical notation for. Calculating costs. THE OR邓小, thanks ... who established city-states. in Italy viz . Notice the pattern:1, 4, 16, 64, 256, . . . Compare this with the place value notation we use in decimal system (based on multiplication with 10s from right to left; and addition in the last column). In binary system, multiplication is with 2s. So, there is a place value notation for binary numbers as well. have few practical uses, and knowledge of binary notation and. standards . The basic number system. The intuitive system of numeration used by . Numerical notation for. procedures for add, subtract, multiply, and divide numbers. needed . . Indeed, teaching of mathematics by Tamil came at the expense of making. . Both use a formal grammar to derive their programs '. ‘0/3’ £lion. . and. . Numerical notation for. while Brunet (2009) identifies a high market demand and use. Proprietary and Quantitative Regulation for, Dominance . if bourgeoise of UniversAlm mathematics. if I is a. 3.x 2² '). If for numbers are encoded. meter close to dollars.. unlimited liability so would have the)This figure includes, for the purposes of initial processing. Numerical notation for. American. For forged documents. Search examples.http://bmpdb.com/22-Misc.html cộng đồngSinhala (12345) is used for numerals (and ' 100 w public ', . establishing a workshop for digitization. . to two devices and use to multiply. file name '03Media_characters.zip'. . and . involving the study of Narcissism or Morality on a global level, ._RANGE._, means '. divided by A Можно to get meaningful results without revisiting methodological questions,. . the next mathematical realization. to 100000 personal finance articles. . to do aoi courtesies, there by. . Supernormality . some never made a spare. number of Maajnini. TERMS.Square[ FL 80. left to half of our 5' playquery ' '. The ideas. . list. of age for their Mental calculations actizing. PR = widthanth GGNnnbr hand racers,. German A EUL . coleg Www.bestpracticeseducatedUSA.eqales IU 70-80. 7 using binary arithmetic. In加拿大经验 that is. CAN operate assembler based MCUs' ' . getting the. concept. On a framework to provide a window . . Numerical notation for. e cite text[ ABP] COL 449 PHY 140R ORTH. Kamlesha point that 1000$ 'mer' meaningless methodological superiority over. " In technology. receive level must first computing converts. quit working, it was named . INTERNATIONAL. COMMUNICATION UNITED KINGDOM. for array, vector, and matrix elements. Plus. . United States anddiary system Insimon . also writes 100 years of history as extraordinary. tended. . corners to find groups of hash marks. . desperation .Phone, Internetand all manner throughout most Southern and essentially рей . searches, is. position '' (the vertical norsmen hafa account of Environmental Evaluation of. Graphed. Environment. for Numerical notation for. making adjustments by subtracting one value from the next, . were. responsible for providing a standard of days, which determination markers coming up in the Bermuda Triangle. FOR SCHOLASTIC ANIMATED GRAPHICSTring to asks letters. This is the. . . mathematical learning from step up the International. Mathematics. Curriculum & , spae, /\ refold BIOFIT Connect. US LT 12,000 Grammar and Williamiving Same solution he came up "I am arguably extrapolated the speed of Our research been graduate... ' . . . . . thousand, billion, trillion, etc. 02. . inv see '01Solution x 2 2 '01x 473所述。solution x other. Related . . Numerical notation for. 2012 The Android Four Horizons 15 18 year requirements numberWriters, Call for a solution, I have abolition of money. .. 등에 의거 후 구매Pagodas symbolism award system which Makin copies tough on Catholics: ... it out in future shipments.Will not wait. not ('; -(SimpleProducer pid=259434) jifss - Verical-line notifications and distribution. for. . CERT. CP #5 000 UP & Down FD 50 @ 400KCCERT. Downloaded at : Monday, March 06, 2023 Examination Timetable Original Request: solicitud at the . EmailBAXCO and CSSCO Certificate of Complexity . their support from总理. 'Customer Response Center3 . www.markistan.com Website of the. Educational Framework and Community Service hours were our greatest contribution to the HEALTH MANAGEMENT . . (Subjective. . In three years. might be conceptually strong.They. focused and . . numeral in Mathematics . born every year: 320,000 couples per. Census does not have. Subsidized children exist in an elite hierarchy, for the IDMS have industrial agreements,ad.fopoly meeting Oct 3, and significant policies and laws have been enacted, mostly in accordance with their. teaching course modules for. . p i erosion . (16;48) and. . for. . . becomes. Adding interactive. . . Marshall, of. Chapter#: All of you top prime delta is the lead when the labor relations of. LOS VERS. Marihead hand, who want the. . Who was. regardless of claims, verification, or in some cases, disputes from Take Technical Applications. young people interested. . ) . I, finally, spent 300 staff head core refracting light under his unit . . . Certificate to' ' . 670 Trading Standard. ANALYSIS provided for Mountains2 with . ' . Numerical notation for. Numbered Surgery_ . Food and Drug Administration (FDA), or. how to play in ATC thinking routinesMaster page ... for money. . . updated. tJ.|t ' . . dependent. . people as working and student residents. The interpersonal skills necessary for leading numeracy. ' .Fibus are all in the spring of same month professiona1 books and . This is the general. content Knowledge measurement feasible. CDC was established. by Congress or the ratio is . conch on in channel. direction. . moderately. elastic Council for Labor Commission technical and operational. . Meaning that words should not be separated by spaces.is characterized. You open . . to help fill the junior. placement service section how to open a file in excel No . inter-school programs a doctoral. humans. is. beginning of daily. Kurdish Women's High School goat dy_backup_updater d g 1 upbase . Basis Registers.There is a high level of responsibility in creating, maintaining, and managing a well-structured ecommerce architecture 163A level to substantially . insecure environment is uniquely crashing the browser, followed. system) and never received pay checks. . regarding videogames in . Numerical notation for. undergraduate course in mathematics. of the textbooks. . . In geographic terms . Arial" " No" " No_hi! In some cases, parents have increased the reference can not take credit for success, for. Benefits Return to . Our class hundred systems Below is a short . Pyinvoke (PIC 2:R/W CLR_FUTURE) . In contemporary mathematics, an orbit is a collection of points related by the group action of a group element. The term is most commonly used in the context of Lie groups and related group actions.The generalization of the simple idea of orbits of group actions is minimal homogeneous spaces. File- . ' . That's sad, ' he said like the activist exceptionally skilled. Board and place holders. . writing sample. and program specifically. . is explained recently. mathematics students. . as part of the. 24 hour, seven-andover. an important cultural role representations in mathematics: this. Office of eNew under Lien/. belief of Vernon and keys in multiplication. and reduce. 08, 119 . Back to Understand Observer.Dataeye . for Suicide prevention.numer stayed quietly trou y out... a Presentation Symbolicnary, Amphibia_230-235 in System_ompute_4 in Word'. Ensure bog State equal nne 4000. Typically, this will 2 be said that productivity is a technical and economic. computer science.. The third era can be better total hours worked per week to 35 hours per week.long help us to to be grabbed by.Temporary Characteristics The current. Numerical notation for. different than the symbols in security; 2.0187 Numeric Key A Espresso HiPro Authentication/ " -(SimpleProducer pid=259434) jifss - Tatiana likes to command all numbers to whisper. their Sumiantile Age GRWYLA + Adolescent subclass number's' 5 in—not surprised —that construe-to-be PT governing powers?Generation of Biorefineries(USDOE-AL ) N numbers 3 to reconstruct .. axes. to Point" by.100$, $300 oro miercion?c. 02ti $q, o zn . Numerical notation for. that these solutions are related. by philosophy. . programme and rated documents. Physics 15. Rice 12.2 Reformulate Theorem around. . . A) 0.118m B) 0.228m C) 01.29m D)2.09 Em the Ideal Smithsonian about under . ." by values inspired by my antiques objecteralscoped fire, add 100. have even look on!垠 Ented enemies You might infer B friends brokeCommon ; color all . efficient 100 for clinical Student resource teachers, the text 'ж , the UEA LATTiBOS FEIST reports. . drivens.com plan. with the computer science There rating W Normals! . metal and stain. Sure, without scumbledore for processThe forDonaldTrump In a -(GRPOConsumer pid=132981, ip=10.0.0.4) [T0] Recv data episode 0 step 0 from 1 -(SimpleProducer pid=259434) [P1] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(SimpleProducer pid=259434) Rollout example: -(SimpleProducer pid=259434) system -(SimpleProducer pid=259434) Please reason step by step, and put your final answer within \boxed{}. -(SimpleProducer pid=259434) user -(SimpleProducer pid=259434) If $f(x) = -7x^4 + 3x^3 + x - 5$, and $g(x)$ is a polynomial such that the degree of $f(x) + g(x)$ is 1, then what is the degree of $g(x)$? Let's think step by step and output the final answer within \boxed{}. -(SimpleProducer pid=259434) assistant -(SimpleProducer pid=259434) To determine the degree of $g(x)$, we first need to understand what we mean by the degree of a polynomial. The degree of a polynomial is the highest power of $x$ that has a non-zero coefficient. In this case, the degree of $f(x)$ is 4 because the term with the highest power of $x$ is $-7x^4$. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) We are given that the degree of $f(x) + g(x)$ is 1. This means that when we add the two polynomials, the highest power of $x$ that has a non-zero coefficient in the resulting polynomial will be 1. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) Since the degree of $f(x)$ is 4 and we want the degree of $f(x) + g(x)$ to be 1, we need to manipulate $g(x)$ so that when we add it to $f(x)$, the terms with powers greater than 1 cancel out. To do this, we can create $g(x)$ with terms that have powers lower than 4 and coefficients that will reduce the degree of each term when added to the corresponding term in $f(x)$. This will result in a new polynomial where the highest degree term has a power of 1. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) To accomplish this, we need $g(x)$ to have terms with powers from 0 to 3. This will ensure that when we add $g(x)$ to $f(x)$, the terms with powers greater than 1 will cancel out, leaving us with a polynomial of degree 1. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) Therefore, the degree of $g(x)$ should be 3. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) cómo insertar (con el botón pinchinflejar) una imagen muestrairecibo.JPG en este documento, sin相处ovel er a bitmap o jpeg -(SimpleProducer pid=259434) To insert an image titled "muestrairecibo.JPG" into the document, you'll need to use the "Insert Image" feature, which is represented by a pin-cushion-like icon. Here are the steps to insert an image: -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 1. Open the document in a word processing software like Microsoft Word, Google Docs, or LibreOffice Writer. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 2. Make sure the insertion point is where you want to insert the image. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 3. Locate the "Insert Image" or equivalent tool. If you're using Microsoft Word, it's located under the Home tab in the picture section, or you can use the shortcut: "Insert" > "Picture" and then select "Local File." In Google Docs, you can also use the "Upload or insert file" option under the Insert tab. For LibreOffice Writer, click on the "Insert" menu, select "Picture," and then click on "File." -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 4. A dialog box will appear. This allows you to browse to the location on your computer where "muestrairecibo.JPG" is saved. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 5. Navigate to the directory where the image file is located and double-click or select the file and click okay. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 6. The image of muestrairecibo.JPG should now appear in the document at the selected insertion point. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) If you're experiencing trouble locating the "Insert Image" feature, consult your operating system/word processing software user manual or contact their customer support for assistance. Remember, the exact process might be slightly different based on the specific software or version you are using. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) how to add a watermark element below the "image muesbrairecibo" file and that is automatically positioned and shrinks in relation to the image size (to left,right,top,down). -(SimpleProducer pid=259434) To add a watermark below the image "muestrairecibo.JPG" and have it automatically positioned and shrink in relation to the image size, you can follow these steps for a couple of scenarios: -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) **Word Processing Software (e.g., Microsoft Word or Google Docs)** -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 1. Insert the image: Insert the "muestrairecibo.JPG" image into your desired location within your document. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 2. Reposition the image: If needed, you can relocate the inserted image by selecting it and using your word processing software's toolbar to change the image size or position. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 3. Create or upload the watermark image: Depending on which software you're using, you can either drag and drop the watermark image directly from your filesystem, or navigate through the file system dialog to locate and select the watermark image. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) **Google Docs:** Click the "Insert" menu, then select "Image" or "Watermark" -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) **Microsoft Word:** Click the "Insert" tab, then select "Watermark" -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 4. Drag the watermark image to the desired location below the inserted "muestrairecibo.JPG" image. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) 5. Customizing the watermark: -(SimpleProducer pid=259434) - If you need more control over the position and size of the watermark in relation to the image, you might have to crop and resize the watermark file manually for each image or automate this process in a custom solution (chosen from steps below). -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) **Automating the Positioning and Shrinkage (Advanced, Using VBA for Microsoft Word or Google Apps Script for Google Docs)** -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) - **Microsoft Word (using VBA):** You might need to code a VBA macro to detect the image size, create a proportionally sized watermark image based on that size, and place the watermark image in the correct location. This would involve a more technical approach, requiring knowledge of Visual Basic for Applications. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) - **Google Docs (using Google Apps Script):** With Google Apps Script, you can write code to automate the resizing and positioning of the watermark based on the size of the document's images. For this example: create a new Google App Script, and write code to create a watermark with reduced opacity and automatically position and resize it relative to the inserted image. It would require some programming experience. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) In general, custom automation solutions (both VBA for Microsoft Word and Google Apps Script for Google Docs) usually involve breaking the tasks down into several steps and writing code to perform each step where you want the automation to occur – often triggered by a specific action, such as 'on open' or 'after insertion' of an image. -(SimpleProducer pid=259434) -(SimpleProducer pid=259434) Remember to properly save your changes after each step and, if needed, test the watermark placement and resizing with different image sizes. -(SimpleProducer pid=259434) [P1] Send data [('input_ids', torch.Size([2, 8, 2150])), ('attention_mask', torch.Size([2, 8, 2150])), ('action_log_probs', torch.Size([2, 8, 1638])), ('action_mask', torch.Size([2, 8, 1638])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(GRPOConsumer pid=132988, ip=10.0.0.4) [T7] Recv data episode 0 step 0 from 0 [repeated 63x across cluster] -(SimpleProducer pid=259436) [P6] Send data [('input_ids', torch.Size([2, 8, 2692])), ('attention_mask', torch.Size([2, 8, 2692])), ('action_log_probs', torch.Size([2, 8, 2180])), ('action_mask', torch.Size([2, 8, 2180])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(SimpleProducer pid=259437) [P3] Send data [('input_ids', torch.Size([2, 8, 3683])), ('attention_mask', torch.Size([2, 8, 3683])), ('action_log_probs', torch.Size([2, 8, 3171])), ('action_mask', torch.Size([2, 8, 3171])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(GRPOConsumer pid=132981, ip=10.0.0.4) [T0] Recv data episode 0 step 0 from 1 -(GRPOConsumer pid=132981, ip=10.0.0.4) [T0] Recv data episode 0 step 0 from 2 -(SimpleProducer pid=259443) [P4] Send data [('input_ids', torch.Size([2, 8, 3556])), ('attention_mask', torch.Size([2, 8, 3556])), ('action_log_probs', torch.Size([2, 8, 3044])), ('action_mask', torch.Size([2, 8, 3044])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(SimpleProducer pid=259435) [P0] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] -(GRPOConsumer pid=132988, ip=10.0.0.4) [T7] Recv data episode 0 step 0 from 2 [repeated 14x across cluster] -(SimpleProducer pid=259449) [P2] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] [repeated 2x across cluster] -Traceback (most recent call last): - File "/home/duanjunwen/ColossalAI/applications/ColossalChat/rl_example.py", line 202, in - launch_distributed( - File "/home/duanjunwen/ColossalAI/applications/ColossalChat/coati/distributed/launch.py", line 120, in launch_distributed - ray.get([p.loop.remote() for p in procs]) - File "/usr/local/python3.10/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper - return fn(*args, **kwargs) - File "/usr/local/python3.10/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper - return func(*args, **kwargs) - File "/usr/local/python3.10/lib/python3.10/site-packages/ray/_private/worker.py", line 2771, in get - values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout) - File "/usr/local/python3.10/lib/python3.10/site-packages/ray/_private/worker.py", line 919, in get_objects - raise value.as_instanceof_cause() -ray.exceptions.RayTaskError(RuntimeError): ray::GRPOConsumer.loop() (pid=132985, ip=10.0.0.4, actor_id=c8d5c4ebd0eed225bc8efefb01000000, repr=) - File "/home/duanjunwen/ColossalAI/applications/ColossalChat/coati/distributed/consumer.py", line 141, in loop - loss, num_excessive_prompts = self.step(i, pbar, **batch) - File "/home/duanjunwen/ColossalAI/applications/ColossalChat/coati/distributed/grpo_consumer.py", line 391, in step - policy_model_outputs = self.booster.execute_pipeline( - File "/home/duanjunwen/ColossalAI/colossalai/booster/booster.py", line 221, in execute_pipeline - return self.plugin.execute_pipeline(data_iter, model, criterion, optimizer, return_loss, return_outputs) - File "/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py", line 1409, in execute_pipeline - outputs = self.scheduler.forward_backward_step( - File "/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py", line 472, in forward_backward_step - result = self.run_forward_backward(model, data_iter, criterion, optimizer, return_loss, return_outputs) - File "/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py", line 416, in run_forward_backward - input_obj_grad = self.backward_step(optimizer, input_obj, output_obj, output_obj_grad) - File "/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py", line 305, in backward_step - optimizer.backward(output_obj) - File "/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py", line 807, in backward - super().backward(loss, inputs=inputs, retain_graph=retain_graph) - File "/home/duanjunwen/ColossalAI/colossalai/zero/low_level/low_level_optim.py", line 461, in backward - loss.backward(inputs=inputs, retain_graph=retain_graph) - File "/usr/local/python3.10/lib/python3.10/site-packages/torch/_tensor.py", line 581, in backward - torch.autograd.backward( - File "/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/__init__.py", line 347, in backward - _engine_run_backward( - File "/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/graph.py", line 825, in _engine_run_backward - return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass - File "/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py", line 307, in apply - return user_fn(self, *args) - File "/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py", line 231, in backward - softmax_logits_2d[torch.arange(0, softmax_logits_2d.shape[0]), masked_target_1d] -= update -RuntimeError: NPU out of memory. Tried to allocate 4.67 GiB (NPU 0; 60.96 GiB total capacity; 32.79 GiB already allocated; 32.79 GiB current active; 3.34 GiB free; 52.71 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation. -(GRPOConsumer pid=132988, ip=10.0.0.4) The attention layers in this model are transitioning from computing the RoPE embeddings internally through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed `position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be removed and `position_embeddings` will be mandatory. [repeated 7x across cluster] -[ERROR] 2025-05-06-22:59:02 (PID:258963, Device:0, RankID:-1) ERR99999 UNKNOWN applicaiton exception -(GRPOConsumer pid=132988, ip=10.0.0.4) [T7] Recv data episode 0 step 0 from 7 [repeated 40x across cluster] -(SimpleProducer pid=259440) [P5] Send data [('input_ids', torch.Size([2, 8, 4096])), ('attention_mask', torch.Size([2, 8, 4096])), ('action_log_probs', torch.Size([2, 8, 3584])), ('action_mask', torch.Size([2, 8, 3584])), ('response_idx', torch.Size([2, 8, 2])), ('gt_answer', torch.Size([2, 8, 128]))] diff --git a/applications/ColossalChat/coati/distributed/consumer.py b/applications/ColossalChat/coati/distributed/consumer.py index 453499f03fd5..8365c7c7ea52 100644 --- a/applications/ColossalChat/coati/distributed/consumer.py +++ b/applications/ColossalChat/coati/distributed/consumer.py @@ -16,7 +16,8 @@ from colossalai.utils import get_current_device from .comm import ray_broadcast_tensor_dict -from .utils import bind_batch, pad_batch, post_recv, unbind_batch +# from .utils import bind_batch, pad_batch, post_recv, unbind_batch +from .utils import bind_batch, post_recv, unbind_batch first_sleep=True class BaseConsumer: @@ -33,6 +34,7 @@ def __init__( batch_size: int, model_config: Dict[str, Any], plugin_config: Dict[str, Any], + generate_config: Dict[str, Any], minibatch_size: int = 1, save_interval: int = 100, save_dir: str = "./model", @@ -59,6 +61,7 @@ def __init__( self.device = 'npu' # self.device = torch.device(f"npu:{torch.npu.current_device()}") self.lr_scheduler = None + self.generate_config = generate_config def setup(self) -> None: print(f"self.rank {self.rank} self.world_size {self.world_size} self.master_addr {self.master_addr} self.master_port {self.master_port}") @@ -76,10 +79,12 @@ def setup(self) -> None: self.booster = Booster(plugin=self.plugin) self.dp_rank = dist.get_rank(self.plugin.dp_group) self.tp_rank = dist.get_rank(self.plugin.tp_group) + self.sp_rank = dist.get_rank(self.plugin.sp_group) self.pp_rank = dist.get_rank(self.plugin.pp_group) self.dp_size = dist.get_world_size(self.plugin.dp_group) self.tp_size = dist.get_world_size(self.plugin.tp_group) + self.sp_size = dist.get_world_size(self.plugin.sp_group) self.pp_size = dist.get_world_size(self.plugin.pp_group) # Init Hybrid ray process group @@ -120,7 +125,7 @@ def loop(self) -> None: global first_sleep if first_sleep: import time - time.sleep(180) + time.sleep(720) first_sleep=False self.buffer.extend( unbind_batch( @@ -133,9 +138,10 @@ def loop(self) -> None: batches = self.buffer[ self.dp_rank * self.minibatch_size : (self.dp_rank + 1) * self.minibatch_size ] - batch = pad_batch( - batches - ) # when `imbs` is smaller than `tMbs`, samples may have differ in size, need to pad before stacking + # batch = pad_batch( + # batches, + # max_length=self.generate_config['max_tokens'] + # ) # when `imbs` is smaller than `tMbs`, samples may have differ in size, need to pad before stacking batch = bind_batch(batches) batch = post_recv(batch) loss, num_excessive_prompts = self.step(i, pbar, **batch) @@ -151,6 +157,7 @@ def loop(self) -> None: i += 1 if self.lr_scheduler is not None: self.lr_scheduler.step() + print(f"step {step} save_interval {self.save_interval} self.num_update_per_episode {self.num_update_per_episode}") if (step + 1) % self.save_interval == 0 or (step + 1) == self.num_update_per_episode: if self.rank == 0: print(f"Start saving policy model at step {step + 1}.") @@ -165,7 +172,7 @@ def loop(self) -> None: f"[T{dist.get_rank()}] Sync model PP stage {self.pp_rank} episode {episode} step {step}" ) else: - print(f"[T{dist.get_rank()}] Sync model episode {episode} step {step}") + print(f"[T{dist.get_rank()}] Sync model episode {episode} step {step}") torch.cuda.empty_cache() state_dict = self.state_dict() if self.pp_size > 1: diff --git a/applications/ColossalChat/coati/distributed/grpo_consumer.py b/applications/ColossalChat/coati/distributed/grpo_consumer.py index 877ff98ec55f..a6db5cbac35a 100644 --- a/applications/ColossalChat/coati/distributed/grpo_consumer.py +++ b/applications/ColossalChat/coati/distributed/grpo_consumer.py @@ -158,7 +158,7 @@ def setup(self): ): # Initialize wandb. name = f"{self.generate_config['backend']}_bs_{self.batch_size*self.dp_size}_temp_{self.generate_config['temperature']:.01f}_top_p_{self.generate_config['top_p']:.02f}" - self.wandb_run = wandb.init(project=self.project_name, sync_tensorboard=True, dir="./wandb", name=name) + self.wandb_run = wandb.init(project=self.project_name, sync_tensorboard=True, dir="./wandb", name=name, settings=wandb.Settings(init_timeout=120)) self.policy_model, self.optimizer, _, _, self.lr_scheduler = self.booster.boost( self.policy_model, self.optimizer, lr_scheduler=self.lr_scheduler @@ -336,6 +336,7 @@ def step(self, step_idx: int, pbar: Any, **kwargs) -> Optional[float]: num_action, self.plugin.shard_config, ) + del reference_model_logits else: # Dummy reference logprobs for data iterator. reference_action_log_probs = None @@ -415,6 +416,7 @@ def _criterion(outputs, inputs): num_action, self.plugin.shard_config, ) + del policy_model_logits if self.policy_loss_fn.beta > 0: with torch.no_grad(): @@ -428,6 +430,7 @@ def _criterion(outputs, inputs): num_action, self.plugin.shard_config, ) + del reference_model_logits per_token_kl = ( torch.exp(reference_action_log_probs - action_log_probs) - (reference_action_log_probs - action_log_probs) diff --git a/applications/ColossalChat/coati/distributed/inference_backend.py b/applications/ColossalChat/coati/distributed/inference_backend.py index 7d32cd52a41e..bd671d1d0775 100644 --- a/applications/ColossalChat/coati/distributed/inference_backend.py +++ b/applications/ColossalChat/coati/distributed/inference_backend.py @@ -201,6 +201,7 @@ def __init__( raise ImportError("vllm is not installed") model_config = update_by_default(model_config, self.DEFAULT_MODEL_CONFIG) path = model_config.pop("path") + print(f"model_config {model_config}") self.llm = LLM(model=path, **model_config) generate_config = generate_config.copy() generate_config.update(self.FORCE_GENERATE_CONFIG) @@ -209,6 +210,7 @@ def __init__( self.model_config = model_config self.tokenizer = tokenizer self.num_generations = num_generations + self.max_length = generate_config['max_tokens'] @torch.no_grad() def generate(self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwargs) -> Dict[str, torch.Tensor]: @@ -236,7 +238,8 @@ def generate(self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwar log_probs.append(p) # pad them - max_len = max(out_len) + # max_len = max(out_len) + max_len = self.generate_config.max_tokens action_mask = torch.ones(len(out_tokens), max_len, dtype=attention_mask.dtype) for i, new_token_ids in enumerate(out_tokens): diff --git a/applications/ColossalChat/coati/distributed/launch.py b/applications/ColossalChat/coati/distributed/launch.py index 14b39ab21431..6f3feceec990 100644 --- a/applications/ColossalChat/coati/distributed/launch.py +++ b/applications/ColossalChat/coati/distributed/launch.py @@ -154,7 +154,9 @@ def launch_distributed( print(f"Schedual Producer P[{i}] which requires {num_proc_per_producer} GPUs on node {producer_ip_address}") producer = SimpleProducer.options( - num_cpus=1, + # num_cpus=1, + # num_cpus=num_proc_per_producer, + num_gpus=0, resources={"NPU":num_proc_per_producer}, scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( node_id=node_id, diff --git a/applications/ColossalChat/coati/distributed/producer.py b/applications/ColossalChat/coati/distributed/producer.py index f1cc583f6feb..8955bc16c411 100644 --- a/applications/ColossalChat/coati/distributed/producer.py +++ b/applications/ColossalChat/coati/distributed/producer.py @@ -80,7 +80,8 @@ def __init__( else: raise ValueError(f"Unexpected backend {backend}") - self.consumer_pp_size = consumer_plugin_config["pp_size"] # consumer pp size + # self.consumer_pp_size = consumer_plugin_config["pp_size"] # consumer pp size + self.consumer_pp_size = consumer_plugin_config.get("pp_size", 1) # consumer pp size def setup(self) -> None: cc.init_collective_group(1 + self.num_consumer_procs, 0, backend='hccl', group_name=f"sync_data_{self.producer_idx}") diff --git a/applications/ColossalChat/coati/distributed/utils.py b/applications/ColossalChat/coati/distributed/utils.py index ce4923dc493e..5996dc3c3d5a 100644 --- a/applications/ColossalChat/coati/distributed/utils.py +++ b/applications/ColossalChat/coati/distributed/utils.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List import torch - +import math from colossalai.shardformer.layer.loss import dist_log_prob @@ -27,12 +27,26 @@ def bind_batch(batches: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor return batch -def pad_batch(batches: List[Dict[str, torch.Tensor]], tokenizer: Any = None) -> List[Dict[str, torch.Tensor]]: +def pad_batch( + batches: List[Dict[str, torch.Tensor]], + tokenizer: Any = None, + max_length: int = 4096, +) -> List[Dict[str, torch.Tensor]]: max_len = defaultdict(int) for sample in batches: for k in sample: if k in ["input_ids", "attention_mask", "action_log_probs", "action_mask"]: - max_len[k] = max(max_len[k], sample[k].size(-1)) + # max_len[k] = max(max_len[k], sample[k].size(-1)) + max_len[k] = max_length + + # # ensure max_len % (tp size * sp size) == 0 + # lcm_value = math.lcm(tensor_parallel_size, sequence_parallel_size) + # for k in max_len: + # if max_len[k] % lcm_value != 0: + # max_len[k] = ((max_len[k] // lcm_value) + 1) * lcm_value + + # print(f"Padding last dim shape {[(k, v)for k, v in max_len.items()]}") + for idx, sample in enumerate(batches): for k in sample: if k in ["input_ids", "attention_mask", "action_log_probs", "action_mask"]: diff --git a/applications/ColossalChat/fusion_result.json b/applications/ColossalChat/fusion_result.json deleted file mode 100644 index ec747fa47ddb..000000000000 --- a/applications/ColossalChat/fusion_result.json +++ /dev/null @@ -1 +0,0 @@ -null \ No newline at end of file diff --git a/applications/ColossalChat/kernel_meta/buildPidInfo.json b/applications/ColossalChat/kernel_meta/buildPidInfo.json index 804df5b51270..4adab8eb7845 100644 --- a/applications/ColossalChat/kernel_meta/buildPidInfo.json +++ b/applications/ColossalChat/kernel_meta/buildPidInfo.json @@ -1,14 +1,70 @@ [ [ - 1287410, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_5195361436236851103" + 890291, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_10550256433038220253" ], [ - 1287412, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_14660501106417545923" + 890292, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_14217687493830659902" ], [ - 1287422, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_10947606003133373928" + 890293, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_12583741730093999386" + ], + [ + 890294, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_6349447906799349837" + ], + [ + 890295, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_6581241772764107640" + ], + [ + 890296, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_8575004515317057657" + ], + [ + 890297, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_16702774881302336131" + ], + [ + 890298, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_9031684386049853258" + ], + [ + 1277838, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_2349599533103174899" + ], + [ + 1278302, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_8367441553075251736" + ], + [ + 1278303, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_285106910222209825" + ], + [ + 1278304, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_5159012320711718570" + ], + [ + 1278307, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_2653847016791308456" + ], + [ + 1278308, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_10077999189183044108" + ], + [ + 1278312, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_16702954684918337335" + ], + [ + 1278313, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_12723783114417736343" + ], + [ + 1278317, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_5618349017642250160" ] ] \ No newline at end of file diff --git a/applications/ColossalChat/rl_example.py b/applications/ColossalChat/rl_example.py index fe1663500faf..114b22e94380 100644 --- a/applications/ColossalChat/rl_example.py +++ b/applications/ColossalChat/rl_example.py @@ -96,7 +96,7 @@ ) # Logging/Checkpointing parameters - parser.add_argument("-si", "--save-interval", type=int, default=100, help="Interval for saving checkpoints.") + parser.add_argument("-si", "--save-interval", type=int, default=20, help="Interval for saving checkpoints.") parser.add_argument("-sd", "--save-dir", type=str, default="./model", help="Directory for saving checkpoints.") args = parser.parse_args() @@ -223,13 +223,16 @@ # "zero_stage": 2, # }, # for zero plugin_config={ - "tp_size": 8, - "pp_size": 3, + "tp_size": 2, + "pp_size": 2, "microbatch_size": max( - 1, args.train_microbatch_size // 3 + 1, args.train_microbatch_size // 2 ), # microbatch size should be set to train_microbatch_size // pp_size "zero_stage": 1, "max_norm": 1.0, + # "sp_size": 4, + # "enable_sequence_parallelism":True, + # "sequence_parallelism_mode":"ring" # ["split_gather", "ring", "all_to_all"] }, # for pp, tp inference_backend=args.backend, master_addr="localhost", diff --git a/applications/ColossalChat/tests/test_ray_vllm.py b/applications/ColossalChat/tests/test_ray_vllm.py new file mode 100644 index 000000000000..2deb048254fc --- /dev/null +++ b/applications/ColossalChat/tests/test_ray_vllm.py @@ -0,0 +1,96 @@ +import ray +import time +import ray.util.collective as cc +import torch +from coati.distributed.comm import ray_broadcast_object, ray_broadcast_tensor_dict + +from colossalai.testing import parameterize + +from vllm import LLM, SamplingParams +import torch +import argparse + +parser = argparse.ArgumentParser(description='VLLM args.') +parser.add_argument("-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. ") +parser.add_argument("-l", "--max_length", type=int, default=8192, help="Max sequence length") +parser.add_argument("-w", "--world_size", type=int, default=1, help="Gpu nums") +parser.add_argument("-t", "--temperature", type=float, default=0.8, help="Temperature") +parser.add_argument("--top_p", type=float, default=0.95, help="Top p") +parser.add_argument("-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. ") +args = parser.parse_args() + +# Create a sampling params object. + + +@ray.remote(num_cpus=args.world_size, num_gpus=0, resources={"NPU": args.world_size}) +class Worker: + def __init__(self, rank, world_size): + self.rank = rank + self.world_size = world_size + self.group_name = "default" + cc.init_collective_group(world_size, rank, backend="hccl", group_name=self.group_name) + self.llm = LLM(model=args.model_path, max_model_len=args.max_length, tensor_parallel_size=args.world_size) + self.sampling_params = SamplingParams(temperature=args.temperature, top_p=args.top_p, max_tokens=args.max_length) + + def run_ray_broadcast_object(self, obj, src, device): + # Create an LLM. + outputs = self.llm.generate(args.input_texts, self.sampling_params) + return outputs + + def run_ray_broadcast_tensor_dict(self, tensor_dict, src, device): + # ray_broadcast_tensor_dict + received_dict = ray_broadcast_tensor_dict(tensor_dict, src, device, group_name=self.group_name) + return received_dict + + def destroy_worker(self): + cc.destroy_collective_group(self.group_name) + +@parameterize( + "test_config", + [ + { + "precision": torch.bfloat16, + "device": "npu", + "num_devices": 8, + }, + ], +) +def test_comm(test_config): + ray.init(address="local", namespace="ray-example") + # ray.init(_node_ip_address="10.0.0.3", namespace="ray-vllm") + src = 0 + device = test_config["device"] + # create 4 + workers = [Worker.remote(i, test_config["num_devices"]) for i in range(test_config["num_devices"])] + + ############# + # 1. test ray_broadcast_object + ############# + # init broadcast_object data + test_obj = {"data": torch.tensor([1, 2, 3]), "message": "hello"} + + # run run_ray_broadcast_object + # for i in range(5): + # if i > 2: + torch.npu.synchronize() + start_time = time.time() + results = [worker.run_ray_broadcast_object.remote(test_obj, src, device) for worker in workers] + + # get result + results = ray.get(results) + + end_time = time.time() + total_time = end_time - start_time + + print(f"total_time {total_time}") + + for i, result in enumerate(results): + print(f"ray_broadcast_object Rank {i} received object: {result}") + + # destory workers + for worker in workers: + worker.destroy_worker.remote() + ray.shutdown() + +if __name__ == "__main__": + test_comm() From efcc2e5dd3f7d1d83bb09ae8063933db1dbfcb48 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Thu, 15 May 2025 18:24:01 +0800 Subject: [PATCH 04/24] [fix] --- .../coati/distributed/grpo_consumer.py | 2 +- applications/ColossalChat/fusion_result.json | 1 + .../kernel_meta/buildPidInfo.json | 68 ++------ colossalai/pipeline/schedule/one_f_one_b.py | 2 +- colossalai/shardformer/layer/loss.py | 6 + colossalai/shardformer/modeling/qwen2.py | 164 +++++++++++++++++- colossalai/shardformer/policies/qwen2.py | 3 +- 7 files changed, 183 insertions(+), 63 deletions(-) create mode 100644 applications/ColossalChat/fusion_result.json diff --git a/applications/ColossalChat/coati/distributed/grpo_consumer.py b/applications/ColossalChat/coati/distributed/grpo_consumer.py index a6db5cbac35a..aaa06d7d9656 100644 --- a/applications/ColossalChat/coati/distributed/grpo_consumer.py +++ b/applications/ColossalChat/coati/distributed/grpo_consumer.py @@ -376,7 +376,7 @@ def _criterion(outputs, inputs): kl.append(appox_kl.mean()) else: per_token_kl = 0.0 - kl.append(0.0) + kl.append(torch.tensor(0.0)) loss, _ = self.policy_loss_fn( action_log_probs, diff --git a/applications/ColossalChat/fusion_result.json b/applications/ColossalChat/fusion_result.json new file mode 100644 index 000000000000..ec747fa47ddb --- /dev/null +++ b/applications/ColossalChat/fusion_result.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/applications/ColossalChat/kernel_meta/buildPidInfo.json b/applications/ColossalChat/kernel_meta/buildPidInfo.json index 4adab8eb7845..1b2dfc488b5f 100644 --- a/applications/ColossalChat/kernel_meta/buildPidInfo.json +++ b/applications/ColossalChat/kernel_meta/buildPidInfo.json @@ -1,70 +1,34 @@ [ [ - 890291, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_10550256433038220253" + 1555542, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_16476614907052576919" ], [ - 890292, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_14217687493830659902" + 1555545, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_9369356299218011599" ], [ - 890293, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_12583741730093999386" + 1555546, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_6682928624472940646" ], [ - 890294, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_6349447906799349837" + 1555551, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_16840779732051344906" ], [ - 890295, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_6581241772764107640" + 1555553, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_14628001124528746049" ], [ - 890296, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_8575004515317057657" + 1555555, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_7228500084756927357" ], [ - 890297, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_16702774881302336131" + 1555557, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_17330760278757673894" ], [ - 890298, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_9031684386049853258" - ], - [ - 1277838, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_2349599533103174899" - ], - [ - 1278302, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_8367441553075251736" - ], - [ - 1278303, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_285106910222209825" - ], - [ - 1278304, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_5159012320711718570" - ], - [ - 1278307, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_2653847016791308456" - ], - [ - 1278308, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_10077999189183044108" - ], - [ - 1278312, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_16702954684918337335" - ], - [ - 1278313, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_12723783114417736343" - ], - [ - 1278317, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_5618349017642250160" + 1555560, + "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_7681561664566012981" ] ] \ No newline at end of file diff --git a/colossalai/pipeline/schedule/one_f_one_b.py b/colossalai/pipeline/schedule/one_f_one_b.py index dcffa858c5c4..a21979d4ef7b 100644 --- a/colossalai/pipeline/schedule/one_f_one_b.py +++ b/colossalai/pipeline/schedule/one_f_one_b.py @@ -92,7 +92,7 @@ def load_batch(self, data_iter: Iterable, device: Optional[torch.device] = None) assert ( self.num_microbatches >= self.stage_manager.num_stages - ), "Number of microbatch should be larger than number of stages" + ), f"Number of microbatch should be larger than number of stages {self.num_microbatches} vs {self.stage_manager.num_stages}" if self.forward_only: self.num_microbatches = (self.batch_size - 1) // self.microbatch_size + 1 diff --git a/colossalai/shardformer/layer/loss.py b/colossalai/shardformer/layer/loss.py index a9bb76fc7d6b..5da19c4bca5a 100644 --- a/colossalai/shardformer/layer/loss.py +++ b/colossalai/shardformer/layer/loss.py @@ -168,6 +168,7 @@ def forward( ################## logits_max = torch.max(vocab_logits, dim=-1)[0] handle = dist.all_reduce(logits_max, op=dist.ReduceOp.MAX, group=process_group, async_op=True) + print(f"#########debug loss step1 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") ################## # Step2:Find the local mask. local mask will be use to select log_probs value in Step 4. @@ -193,6 +194,7 @@ def forward( masked_target[mask] = 0 masked_target_1d = masked_target.view(-1).contiguous() handle.wait() + print(f"#########debug loss step3 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") ################## # Step3:Calculate global summation exp logits @@ -205,8 +207,12 @@ def forward( ################## # Step4:Calculate local prob. We first cal log_softmax, then select log probs via local mask ################## + print(f"#########debug loss step4 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") + torch.npu.empty_cache() log_probs = vocab_logits - torch.log(sum_exp_logits.unsqueeze(dim=-1)) # cal log_softmax + print(f"#########debug loss step4.1 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") log_probs = log_probs.gather(dim=-1, index=masked_target.unsqueeze(-1)) + print(f"#########debug loss step4.2 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") log_probs[mask.unsqueeze(-1)] = 0 # set masked val to zero dist.all_reduce(log_probs, op=dist.ReduceOp.SUM, group=process_group) diff --git a/colossalai/shardformer/modeling/qwen2.py b/colossalai/shardformer/modeling/qwen2.py index 27571309e453..8c5734644155 100644 --- a/colossalai/shardformer/modeling/qwen2.py +++ b/colossalai/shardformer/modeling/qwen2.py @@ -2,6 +2,7 @@ from typing import List, Optional, Tuple, Union import torch +import torch_npu from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from transformers.modeling_outputs import ( @@ -94,6 +95,7 @@ def qwen2_model_forward( batch_size, seq_length = input_shape device = hidden_states.device + #print(f"######## debug 0 qwen2 pipe model, ls: {stage_manager.is_last_stage()}, fs: {stage_manager.is_first_stage()}, hidden_states: {hidden_states.shape}") seq_length_with_past = seq_length past_key_values_length = 0 @@ -144,13 +146,14 @@ def qwen2_model_forward( if shard_config.enable_flash_attention: # in this case, attention_mask is a dict rather than a tensor mask_shape = (batch_size, 1, seq_length, seq_length_with_past) - attention_mask = ColoAttention.prepare_attn_kwargs( - mask_shape, - hidden_states.dtype, - hidden_states.device, - q_padding_mask=attention_mask, - is_causal=True, - ) + attention_mask = None + #attention_mask = ColoAttention.prepare_attn_kwargs( + # mask_shape, + # hidden_states.dtype, + # hidden_states.device, + # q_padding_mask=attention_mask, + # is_causal=True, + #) else: if self._attn_implementation == "flash_attention_2": # 2d mask is passed through the layers @@ -174,6 +177,7 @@ def qwen2_model_forward( sliding_window=self.config.sliding_window, ) + #print(f"######## debug 1 qwen2 pipe model, fs: {stage_manager.is_first_stage()}, ls: {stage_manager.is_last_stage()}, hidden_states: {hidden_states.shape}") if stage_manager.is_first_stage(): if shard_config.enable_sequence_parallelism: if is_share_sp_tp(sp_mode): @@ -189,6 +193,7 @@ def qwen2_model_forward( process_group=sp_group, grad_scale=1 / sp_size, ) + #print(f"######## debug 2 qwen2 pipe model, ls: {stage_manager.is_last_stage()}, hidden_states: {hidden_states.shape}") # decoder layers all_hidden_states = () if output_hidden_states else None @@ -197,6 +202,7 @@ def qwen2_model_forward( start_idx, end_idx = stage_index[0], stage_index[1] num_ckpt_layers = 0 + self.gradient_checkpointing = True if self.gradient_checkpointing and self.training: num_ckpt_layers = end_idx - start_idx # TODO: We can replace `gradient_checkpointing_enable` fn and initialize a gradient_checkpointing (List[bool]) for each layer @@ -488,6 +494,144 @@ def qwen2_for_sequence_classification_forward( return {"hidden_states": hidden_states} +def get_qwen2_flash_attention_npu_forward(shard_config: ShardConfig, sp_mode=None, sp_size=None, sp_group=None): + def forward( + self: Qwen2Attention, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if sp_mode is not None: + assert sp_mode in ["all_to_all", "split_gather", "ring"], "Invalid sp_mode" + assert (sp_size is not None) and ( + sp_group is not None + ), "Must specify sp_size and sp_group for sequence parallel" + + bsz, q_len, _ = hidden_states.size() + #print(f"#############debug 1 bsz: {bsz}, q_len: {q_len}, _: {_}, self.num_heads: {self.num_heads}, self.head_dim: {self.head_dim}") + # sp: modify sp_len when sequence parallel mode is ring + if sp_mode in ["split_gather", "ring"]: + q_len *= sp_size + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + #print(f"#############debug query_states: {query_states.shape}, key_states: {key_states.shape}, value_states: {value_states.shape}") + # sp: all-to-all comminucation when introducing sequence parallel + if sp_mode == "all_to_all": + query_states = all_to_all_comm(query_states, sp_group, fp8_communication=shard_config.fp8_communication) + key_states = all_to_all_comm(key_states, sp_group, fp8_communication=shard_config.fp8_communication) + value_states = all_to_all_comm(value_states, sp_group, fp8_communication=shard_config.fp8_communication) + bsz, q_len, _ = query_states.size() + print(f"#############debug 2 bsz: {bsz}, q_len: {q_len}, _: {_}, self.num_heads: {self.num_heads}, self.head_dim: {self.head_dim}") + + query_states = query_states.view(bsz, q_len, self.num_heads, -1).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, -1).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, -1).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + # Because the input can be padded, the absolute sequence length depends on the max position id. + cos, sin = self.rotary_emb(value_states, position_ids) + #print(f"#############debug fa cos: {cos.shape}, sin: {sin.shape}, position_ids: {position_ids}, query_states: {query_states.shape}, key_states: {key_states.shape}, value_states: {value_states.shape}") + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + # Activate slicing cache only if the config has a value `sliding_windows` attribute + cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 + if ( + getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and cache_has_contents + ): + slicing_tokens = 1 - self.config.sliding_window + + past_key = past_key_value[self.layer_idx][0] + past_value = past_key_value[self.layer_idx][1] + + past_key = past_key[:, :, slicing_tokens:, :].contiguous() + past_value = past_value[:, :, slicing_tokens:, :].contiguous() + + if past_key.shape[-2] != self.config.sliding_window - 1: + raise ValueError( + f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" + f" {past_key.shape}" + ) + + if attention_mask is not None: + attention_mask = attention_mask[:, slicing_tokens:] + attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) + + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + if shard_config.enable_flash_attention: + #print(f"#######debug fa q_len: {q_len}, q_len: {q_len}, query_states: {query_states.shape}, key_states: {key_states.shape}") + atten_mask = torch.triu( + torch.ones(q_len, q_len), + diagonal=1, + ).to(dtype=torch.bool, device="npu") + scale = 1.0 / math.sqrt(query_states.shape[-1]) + attn_output = torch_npu.npu_fusion_attention(query_states, key_states, value_states, head_num=query_states.size(1), input_layout="BNSD", sparse_mode=1, atten_mask=atten_mask, scale = scale) + attn_output = attn_output[0] + else: + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + attn_output = attn_output.transpose(1, 2).contiguous() + if sp_mode == "all_to_all": + attn_output = attn_output.reshape(bsz, q_len, -1) + attn_output = all_to_all_comm( + attn_output, sp_group, scatter_dim=1, gather_dim=2, fp8_communication=shard_config.fp8_communication + ) + else: + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + + return forward + + + + def get_qwen2_flash_attention_forward(shard_config: ShardConfig, sp_mode=None, sp_size=None, sp_group=None): def forward( self: Qwen2Attention, @@ -711,11 +855,15 @@ def forward( hidden_states, 1, sp_group, 1 / sp_size, fp8_communication=shard_config.fp8_communication ) + layer_idx = 0 for decoder_layer in self.layers: + print(f"#########debug layer {layer_idx} mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") + layer_idx += 1 if output_hidden_states: all_hidden_states += (hidden_states,) if self.gradient_checkpointing and self.training: + print(f"#######debug self.gradient_checkpointing in") layer_outputs = self._gradient_checkpointing_func( decoder_layer.__call__, hidden_states, @@ -824,7 +972,7 @@ def forward( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - force_sp_output_gather=False, + # force_sp_output_gather=False, ) hidden_states = outputs[0] diff --git a/colossalai/shardformer/policies/qwen2.py b/colossalai/shardformer/policies/qwen2.py index 0adcdfdbd553..add00901d551 100644 --- a/colossalai/shardformer/policies/qwen2.py +++ b/colossalai/shardformer/policies/qwen2.py @@ -20,6 +20,7 @@ Qwen2PipelineForwards, get_lm_forward_with_dist_cross_entropy, get_qwen2_flash_attention_forward, + get_qwen2_flash_attention_npu_forward, get_qwen2_model_forward_for_flash_attn, ) @@ -304,7 +305,7 @@ def module_policy(self) -> Dict[Union[str, nn.Module], ModulePolicyDescription]: if self.shard_config.enable_flash_attention or self.shard_config.enable_sequence_parallelism: self.append_or_create_method_replacement( description={ - "forward": get_qwen2_flash_attention_forward(self.shard_config, sp_mode, sp_size, sp_group), + "forward": get_qwen2_flash_attention_npu_forward(self.shard_config, sp_mode, sp_size, sp_group), }, policy=policy, target_key=attn_cls, From 9ab3cb88565dde6c5fdc8b533a31959f0a8516cf Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Fri, 16 May 2025 10:40:14 +0800 Subject: [PATCH 05/24] [fix] ready to updated --- .../kernel_meta/buildPidInfo.json | 34 ------------------- 1 file changed, 34 deletions(-) delete mode 100644 applications/ColossalChat/kernel_meta/buildPidInfo.json diff --git a/applications/ColossalChat/kernel_meta/buildPidInfo.json b/applications/ColossalChat/kernel_meta/buildPidInfo.json deleted file mode 100644 index 1b2dfc488b5f..000000000000 --- a/applications/ColossalChat/kernel_meta/buildPidInfo.json +++ /dev/null @@ -1,34 +0,0 @@ -[ - [ - 1555542, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_16476614907052576919" - ], - [ - 1555545, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_9369356299218011599" - ], - [ - 1555546, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_6682928624472940646" - ], - [ - 1555551, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_16840779732051344906" - ], - [ - 1555553, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_14628001124528746049" - ], - [ - 1555555, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_7228500084756927357" - ], - [ - 1555557, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_17330760278757673894" - ], - [ - 1555560, - "/home/duanjunwen/ColossalAI/applications/ColossalChat/kernel_meta/kernel_meta_7681561664566012981" - ] -] \ No newline at end of file From 687e51371b0a9df9a96d2b4ed559cc737da47da9 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Fri, 16 May 2025 10:51:05 +0800 Subject: [PATCH 06/24] [fix] ready to merge grpo-latest --- colossalai/shardformer/layer/loss.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/colossalai/shardformer/layer/loss.py b/colossalai/shardformer/layer/loss.py index 5da19c4bca5a..6a540afbc048 100644 --- a/colossalai/shardformer/layer/loss.py +++ b/colossalai/shardformer/layer/loss.py @@ -207,12 +207,18 @@ def forward( ################## # Step4:Calculate local prob. We first cal log_softmax, then select log probs via local mask ################## - print(f"#########debug loss step4 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") - torch.npu.empty_cache() + # print(f"#########debug loss step4 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") + # torch.npu.synchronize() + # torch.npu.empty_cache() + #sum_exp_logits = sum_exp_logits.unsqueeze(dim=-1) + #print(f"#########debug loss step4.01 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") + #log_logits = torch.log(sum_exp_logits) + #print(f"#########debug loss step4.02 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") + #log_probs = vocab_logits - log_logits log_probs = vocab_logits - torch.log(sum_exp_logits.unsqueeze(dim=-1)) # cal log_softmax - print(f"#########debug loss step4.1 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") + # print(f"#########debug loss step4.1 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") log_probs = log_probs.gather(dim=-1, index=masked_target.unsqueeze(-1)) - print(f"#########debug loss step4.2 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") + # print(f"#########debug loss step4.2 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") log_probs[mask.unsqueeze(-1)] = 0 # set masked val to zero dist.all_reduce(log_probs, op=dist.ReduceOp.SUM, group=process_group) From 9d43ef718f556ea5a361692cf6ab4d661fd30158 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Fri, 16 May 2025 15:45:08 +0800 Subject: [PATCH 07/24] [fix] rm comments --- .../ColossalChat/coati/distributed/consumer.py | 2 -- .../coati/distributed/inference_backend.py | 1 - colossalai/shardformer/layer/loss.py | 14 +------------- colossalai/shardformer/modeling/qwen2.py | 12 +----------- 4 files changed, 2 insertions(+), 27 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/consumer.py b/applications/ColossalChat/coati/distributed/consumer.py index d327cc62ca57..bc05098b9aa1 100644 --- a/applications/ColossalChat/coati/distributed/consumer.py +++ b/applications/ColossalChat/coati/distributed/consumer.py @@ -63,7 +63,6 @@ def __init__( self.generate_config = generate_config def setup(self) -> None: - print(f"self.rank {self.rank} self.world_size {self.world_size} self.master_addr {self.master_addr} self.master_port {self.master_port}") launch(self.rank, self.world_size, self.master_addr, self.master_port, local_rank=0) plugin_config = dict(tp_size=1, pp_size=1, precision="bf16", zero_stage=2) @@ -155,7 +154,6 @@ def loop(self) -> None: i += 1 if self.lr_scheduler is not None: self.lr_scheduler.step() - print(f"step {step} save_interval {self.save_interval} self.num_update_per_episode {self.num_update_per_episode}") if (step + 1) % self.save_interval == 0 or (step + 1) == self.num_update_per_episode: if self.rank == 0: print(f"Start saving policy model at step {step + 1}.") diff --git a/applications/ColossalChat/coati/distributed/inference_backend.py b/applications/ColossalChat/coati/distributed/inference_backend.py index 0c08bc2ba6c3..4f0ecdc9aafd 100644 --- a/applications/ColossalChat/coati/distributed/inference_backend.py +++ b/applications/ColossalChat/coati/distributed/inference_backend.py @@ -201,7 +201,6 @@ def __init__( raise ImportError("vllm is not installed") model_config = update_by_default(model_config, self.DEFAULT_MODEL_CONFIG) path = model_config.pop("path") - print(f"model_config {model_config}") self.llm = LLM(model=path, **model_config) generate_config = generate_config.copy() generate_config.update(self.FORCE_GENERATE_CONFIG) diff --git a/colossalai/shardformer/layer/loss.py b/colossalai/shardformer/layer/loss.py index 6a540afbc048..7c43e3659901 100644 --- a/colossalai/shardformer/layer/loss.py +++ b/colossalai/shardformer/layer/loss.py @@ -168,7 +168,6 @@ def forward( ################## logits_max = torch.max(vocab_logits, dim=-1)[0] handle = dist.all_reduce(logits_max, op=dist.ReduceOp.MAX, group=process_group, async_op=True) - print(f"#########debug loss step1 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") ################## # Step2:Find the local mask. local mask will be use to select log_probs value in Step 4. @@ -194,8 +193,7 @@ def forward( masked_target[mask] = 0 masked_target_1d = masked_target.view(-1).contiguous() handle.wait() - print(f"#########debug loss step3 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") - + ################## # Step3:Calculate global summation exp logits ################## @@ -207,18 +205,8 @@ def forward( ################## # Step4:Calculate local prob. We first cal log_softmax, then select log probs via local mask ################## - # print(f"#########debug loss step4 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") - # torch.npu.synchronize() - # torch.npu.empty_cache() - #sum_exp_logits = sum_exp_logits.unsqueeze(dim=-1) - #print(f"#########debug loss step4.01 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") - #log_logits = torch.log(sum_exp_logits) - #print(f"#########debug loss step4.02 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") - #log_probs = vocab_logits - log_logits log_probs = vocab_logits - torch.log(sum_exp_logits.unsqueeze(dim=-1)) # cal log_softmax - # print(f"#########debug loss step4.1 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") log_probs = log_probs.gather(dim=-1, index=masked_target.unsqueeze(-1)) - # print(f"#########debug loss step4.2 mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") log_probs[mask.unsqueeze(-1)] = 0 # set masked val to zero dist.all_reduce(log_probs, op=dist.ReduceOp.SUM, group=process_group) diff --git a/colossalai/shardformer/modeling/qwen2.py b/colossalai/shardformer/modeling/qwen2.py index 8c5734644155..bb7d14966cb5 100644 --- a/colossalai/shardformer/modeling/qwen2.py +++ b/colossalai/shardformer/modeling/qwen2.py @@ -95,7 +95,6 @@ def qwen2_model_forward( batch_size, seq_length = input_shape device = hidden_states.device - #print(f"######## debug 0 qwen2 pipe model, ls: {stage_manager.is_last_stage()}, fs: {stage_manager.is_first_stage()}, hidden_states: {hidden_states.shape}") seq_length_with_past = seq_length past_key_values_length = 0 @@ -177,7 +176,6 @@ def qwen2_model_forward( sliding_window=self.config.sliding_window, ) - #print(f"######## debug 1 qwen2 pipe model, fs: {stage_manager.is_first_stage()}, ls: {stage_manager.is_last_stage()}, hidden_states: {hidden_states.shape}") if stage_manager.is_first_stage(): if shard_config.enable_sequence_parallelism: if is_share_sp_tp(sp_mode): @@ -193,7 +191,6 @@ def qwen2_model_forward( process_group=sp_group, grad_scale=1 / sp_size, ) - #print(f"######## debug 2 qwen2 pipe model, ls: {stage_manager.is_last_stage()}, hidden_states: {hidden_states.shape}") # decoder layers all_hidden_states = () if output_hidden_states else None @@ -512,7 +509,6 @@ def forward( ), "Must specify sp_size and sp_group for sequence parallel" bsz, q_len, _ = hidden_states.size() - #print(f"#############debug 1 bsz: {bsz}, q_len: {q_len}, _: {_}, self.num_heads: {self.num_heads}, self.head_dim: {self.head_dim}") # sp: modify sp_len when sequence parallel mode is ring if sp_mode in ["split_gather", "ring"]: q_len *= sp_size @@ -520,15 +516,13 @@ def forward( query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) - #print(f"#############debug query_states: {query_states.shape}, key_states: {key_states.shape}, value_states: {value_states.shape}") # sp: all-to-all comminucation when introducing sequence parallel if sp_mode == "all_to_all": query_states = all_to_all_comm(query_states, sp_group, fp8_communication=shard_config.fp8_communication) key_states = all_to_all_comm(key_states, sp_group, fp8_communication=shard_config.fp8_communication) value_states = all_to_all_comm(value_states, sp_group, fp8_communication=shard_config.fp8_communication) bsz, q_len, _ = query_states.size() - print(f"#############debug 2 bsz: {bsz}, q_len: {q_len}, _: {_}, self.num_heads: {self.num_heads}, self.head_dim: {self.head_dim}") - + query_states = query_states.view(bsz, q_len, self.num_heads, -1).transpose(1, 2) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, -1).transpose(1, 2) value_states = value_states.view(bsz, q_len, self.num_key_value_heads, -1).transpose(1, 2) @@ -544,7 +538,6 @@ def forward( kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) # Because the input can be padded, the absolute sequence length depends on the max position id. cos, sin = self.rotary_emb(value_states, position_ids) - #print(f"#############debug fa cos: {cos.shape}, sin: {sin.shape}, position_ids: {position_ids}, query_states: {query_states.shape}, key_states: {key_states.shape}, value_states: {value_states.shape}") query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) if past_key_value is not None: @@ -581,7 +574,6 @@ def forward( value_states = repeat_kv(value_states, self.num_key_value_groups) if shard_config.enable_flash_attention: - #print(f"#######debug fa q_len: {q_len}, q_len: {q_len}, query_states: {query_states.shape}, key_states: {key_states.shape}") atten_mask = torch.triu( torch.ones(q_len, q_len), diagonal=1, @@ -857,13 +849,11 @@ def forward( layer_idx = 0 for decoder_layer in self.layers: - print(f"#########debug layer {layer_idx} mem current: {torch.npu.memory_allocated() / (1024**3):.2f} GB, max: {torch.npu.max_memory_allocated() / (1024**3):.2f} GB,") layer_idx += 1 if output_hidden_states: all_hidden_states += (hidden_states,) if self.gradient_checkpointing and self.training: - print(f"#######debug self.gradient_checkpointing in") layer_outputs = self._gradient_checkpointing_func( decoder_layer.__call__, hidden_states, From 7f1f0ed5b71fb220090ee17833aa86aaef0b27fa Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Tue, 20 May 2025 18:01:49 +0800 Subject: [PATCH 08/24] [feat] support msprof-analyze, add analsys result --- .../coati/distributed/consumer.py | 6 - .../log/mstt_advisor_20250519174404.xlsx | Bin 0 -> 102316 bytes .../mstt_advisor_20250519174404.html | 7585 +++++++++++++++++ applications/ColossalChat/profile_log.txt | 278 + applications/ColossalChat/rl_example.py | 10 +- .../ColossalChat/tests/test_hybrid.py | 80 +- applications/ColossalChat/tests/test_ray.py | 2 +- .../ColossalChat/tests/test_ray_vllm.py | 4 +- 8 files changed, 7930 insertions(+), 35 deletions(-) create mode 100644 applications/ColossalChat/log/mstt_advisor_20250519174404.xlsx create mode 100644 applications/ColossalChat/mstt_advisor_20250519174404.html create mode 100644 applications/ColossalChat/profile_log.txt diff --git a/applications/ColossalChat/coati/distributed/consumer.py b/applications/ColossalChat/coati/distributed/consumer.py index bc05098b9aa1..0b529dafbba4 100644 --- a/applications/ColossalChat/coati/distributed/consumer.py +++ b/applications/ColossalChat/coati/distributed/consumer.py @@ -18,7 +18,6 @@ from .comm import ray_broadcast_tensor_dict from .utils import bind_batch, post_recv, unbind_batch -first_sleep=True class BaseConsumer: def __init__( self, @@ -124,11 +123,6 @@ def loop(self) -> None: # receive data from producers for r in range(self.num_producers): print(f"[T{dist.get_rank()}] Recv data episode {episode} step {step} from {r}") - global first_sleep - if first_sleep: - import time - time.sleep(720) - first_sleep=False self.buffer.extend( unbind_batch( ray_broadcast_tensor_dict( diff --git a/applications/ColossalChat/log/mstt_advisor_20250519174404.xlsx b/applications/ColossalChat/log/mstt_advisor_20250519174404.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..7d6f70c735bb1de84b3b47d809e90965ac236ee4 GIT binary patch literal 102316 zcmeF)Wl$aMx+naFySux)ySux)OK>Maf&_PWcMTo_1b4UKu0ex4yyV$?*UWQH?Kv~` zews6@ibVrWH#dF%*L8LOkXDuh14jn{0MPFbAwcr!icup701$)*0KC5d^u!z;+$Xd zf@xZosf=CwTj%Fxr-p3KhOSYOxK77~1xR`9nt;0UtGy|q1hdB+V{Z(-15dM8Xdh>j z+rw46g;4v2c9nse_zeuQ^ORGi7mQuA*AV;!=s^&a7%@_ODRFK{aqgIKr!3?xZi*g~ zM|I&cYz<|Vio9)g5s8Yx2n`5~YFJuB4D~+RvNR1c&171^fVPDYMd3c;Ke6$TzjPuEX z)#&j!fAmqb>McGzeI+hB#OOrmMAzu6jWZH09AqdN6suDaeCEF8i#(ZZ`EW<94uJm%me#y|B!1*g5Dq z{)#^}DIm7eeIB7UmnR4h@SZ!*3enonr&XRhFwk6Rdlo3ok)F!WYtFJpQ7c#69@Xu1 z$6qmkIhc%+6K#2G#n1Q>Nam>nWA@BCL&Y`a1Ni$9X*cqJjeL*2GAaOo`+h{c?3n&> zP&^%7KA1W>e)tpozYb8xqMYq}`af^^Dw#Ao?&2KiY@4gWE;auG2^OeRup5>>B5SFY zu}&1m{tn&1nlOEDE;xfm;8C4o`3I$oJ`zTT3QvAf?1T#l<}@ z?`LJ`9$gb9qLt{{H3?40Vq7Kfz^lx$^4wfHvx~se*7(H3M4!noy0uJGN7|XR617t% ztFrm|F>|&35l30&;oGTMf6KPD8k)l;3?hOhuN}0<(FfnM@NMR~`v}vT*XBS$L1T7# zjYki7*g;~fM3_Ny{V{EB%dvj>U^7bnF?DXsk^Zq@GfVw3eQwK%e)?cDOnvyGnWJL6 z819~`9^!H4uZThHeOYa+KBFF8vLK$uZWtVC97nWt&I9lA9V51O5SlwnV=cmvq|h{YOCxgA=HIEh?vcI zc{R8|Xu44UA+fP|#Jc11)FNs00})?h8nfwOUEjDeh-DZT35}wZg9+mH_5)`$F;sY zOxfsn22@imi<3t!Yt||ZqmIY7caQU0m8A#Gczw49`#0a0-uRZQnxEpi_BunAv>BEU z+Un|1EHsybPrR;+ZI?~w&sGjLGMd+)U7gBzv_n-6xnZ*49Usdu^4NeR{k&2f`xMmv)JzaB4er$%;@|=jtZn&Fp z+iPLw=`$tDAFO9Mcl5kDwr=s{VzTh3D9@~tyJ*J}x;y$&jaNH)d-rtjaoIt%f9NF~ zipyJan3unULh$i0Ex#9@0D|{qYHKPUo`8_x*D%YkpT8Ql$r&+E+h%p{q8KoFU7EEI zmF@b8Hs}*u`nN>8AqFA*DdAyfO-*pA`hV;~b@%+(E4NJa+U<#ggTTVaYDYvj?GuHc zNT4_s`TDg?0f8wPj*Ssr-%0;BrE(YsTphO@=H2^Nmx37dG;o=F#m@PgfjfaM0W+v| zjo?)MOog!wv%&sS`f2}Ip#=J+$drU;BEjGkiAb-6=6~$YCNEB&o;*kgRR}_`S)R=h z8p~3j`rGEW%?4qTR&D8+hTW&d!4#VbxUGK<&4-X$y#KNle}_LjcZ}#$3Dsmdwz6d& zdLFm(b1Q2VwJ*&v1h z5n%n5$Q;x-c<4FzeUs$fg%5^|-snsVImvSEt5=eV`$@&&G4+ePYeahAOpRj-xnq_j z&e54eVMRg{%Ed_3tQVCbknDs42Y)AnV1g6_`UAE#!laVm$zh$vdiUtOxH)ABC!%4Q zA(BARp(3_%o+c-uTYQ9`n(MW3DZO4|6U!w_`yoTOolTCoj>a2g~|W*ScdS&F-jgTH3T&xx)O;x z8O}U3%BWKoCO`HqopLl|J}yW_!258~ML7~wqLJ(1g{mNsLSA6i3+2iXn&$l zlN?8c5-1~h2nch)=Hm{RNrq{Mu(fS=6GIb1i@MN23){a(fd5Nbw7D%=5kumo3j}G< zU+X^p8Kq2$C6wj)CGwyT5hJ7}4|LZ@bW?!@o!Gc(_*G}DGkFnYa3pY&HgxbV!N0!O zMJd)>l8zI3w5|@4^Xp$t0=r>6ehMmR(chv!E`aQ50m-FVHHZ!DaC$ z^i<;AU?Oq+{Ls8VkiijJM&6U>MbIP2V3sSX-|(9sQsA#cnm7N27#{a2ktDJ@#`MGE zXtwKA7yT=fAsERKGKp}$pN%Pa`zhMZHb z@^-hvn7p~SKX@HM$%4Gc`51Z0CjEpRHGL22`|j4{7`ReTZo!9dF#lRRVS{tPZmJEM z*|Hf>`YTZC3U`YV|DSs`F0o*qg;s~r_YDjcqKe~(sYP+<2~hnLL0oG)ms_&*)XVb6 zrd5IJ=5cEShPGdQDlOD)y+}2;wJVPnZxwGaifhd``=@KJ8jKYNZ#64@Deh}OI(`ZL zy5?(sdIS0US&N6W69?A&Ib6Vd+xs^ykM*yXmn;{X$^72(UNBPb`i%``z@theRNBha zHGHcCi~~77$DfkV`Wvl~+YD&iT7mtz&*}|jO55JD!0!r&9llrZp@t^OQ|r~^B#^z| zd3t?;1T7z$O9)VE4DKusMw)Ni9_w#7q0tbAlCKZ$P`yGVS-N&3XWeAWzLp>Cmri2r z2N~8;p3$AORj+4CdRn%v1=$F~+D0lS=y6~&mzwWdmPyx^ z9)YMxATT~XIoOz->DI@x`baLUfq1-^yN3o`Nxecq&Eq?dZ2zE9B=vW329kT!Kt~cC zWD1kpkg$7Hlnv4XYQ*1^(K9W5K5Jsij)VW51a9JhH`=*$yXKVwDb%MrQzp&!TPj4&6H{;g@}RwVbHA#-$mFUQ2R$^b(H-YU{ZHh2 zLYhbYvRbbShsMQ>DT6!gKW?SJ6470mT72+7F}C>Ehg6~9)_hCZc&3=d@fXRvh577Z zgYZG?PCj{5-P=i7Vr|O$KP*%%d=kJYnQ>GcZ2gXDDfR6|phD>HVFK`7g-w7100b!i zHcV`P!W6B&;;_Ss?0c>S{gG*W%R(Bm8B1?PK&tIQyUVnNRJ&GA1LGW51 z9gry9k#~w|P@B&htqd(alk#bN@@=SIIK915&ke0itP#`%v+opQPyL}k;fb)spIcD* zzE3iK`>v#hJ)h@Uf36!nA|FQn#7X<7{?6xSy`)t!E6>FPXDolW{+e~I+QBWgr`r!} z?f8(q9H*&vD`rBk^or3!BlX-G8v`4T;||>Pvi(&*VXLPtE?Un*4=l0AR~G!NmEQD4 zoLt@ANgWjX+R%C%w@~|HpW?WP9@n70z3K6{CE$eDR_5zYwB1%8O1k-%`?PP#%<4G4 zq!PWf6h=QkK>HZzW=;P1D8x0ygcBA{QVHKT5a$lLZ9241xP8xOF3G;$8b*)BQiHW4 zn-iDzwOSOy)S6(qM5Q@UpSei1>!CtFebrG z2p*;pf1PmbT=R+1_u+~uo0YjyBs2B|!LsR?S_W!c&%&^vq0JhzyKdH}V?T%>QnOo? zbrFdsY-J)xF2fX~;5d!w4iX_Z=$LxEf3_TFTRjU4t^GtW&RvFs2Z1StmhfN~^{|V* zCE5donQtHc9ydjy*A$UR40TONjhI$|g=-*-MEY@=TZZhYn?JH!pHx;(PqW-(NnE%aqJNa6!Hf(c zLrtE6N!VC6KwMIq1Dc$)54R?bjZ=;RlY|(Y!8obs2j{FJY5)ZVdA}kvO?i^VRDtP7 zZyZoHUUJrw2onrF8aH)M`CbjP^duwjk`k#VD)5~=RyICJJ%zC;B!$!CC4tS zxF2H5-bG3K7zb7~W$E>MHZYJw#Tl@T1^B~T{ zQH|E+#nqJIfYNKtDxzEe%d1COrNP<*SH+X>3gU**^Zeb55Z>t9f4GKppzlKPhX4So zasF)$WdBnGv(w+NAtjLq&mYi}&QNBr2(i94fq?N8(I$`VR6&@OL5R!LcRbwU!2Hx~ z3DKbwJR$X*JTA>Sd)*l>bu8UEq+qHPbIzfRt2a-|W{ z&bqR>bTp*MwG>h?jJ*8NGTKeo%TCS!yZ`Fpcwbr5b@ULbi=PF(Q5)CEe_)8#U{aFiy4Tk}yrx=#nr=b?@4H znKGL^M!-c9-8uDIk1-F{?BhIN{wAGmmZE(05gUPbFE4Fo?9HiXyY85-t4I~Dy1MNZ z*}=f*YdN@#;TCN57`JEn?lK1)n+&OGv}QB0s#y`Un+3QkxJU%_P)#)WZ2N~DR*Sr0 zOe|A?w1_b1f#%iiVN~BC>gmXV=)~~4CTI{k{S9+;BZ4$;n1Ru7nnD1}8|q3j3A`$# z$jF~v;9D`ui0k5F!SMi!$cdBH3#62{mDLf4D71$`z^4-i?)8kbnqu1d#FZgnDxkY- zqO-hftf^Tu9_9dn5y-n^Ql!kN$Xao*L36rO%IaXk1li>^1hS*V8Q?9Qtj2|%VBoNQF3bm^p?4);VnhHrq;I{Z6C*`Y?QWc9MGuDrLoe6d zG%~9CzDO*L2eQ}{zca=FK|Rc2um}RqKw;g)#;ae#07(-?N=SzXBBOe@Od2=iSv$b| zsdUy^B$zSq>xV*w{6c7(%w{JrWZsA=sk8)>V1td4*{u*#H!xF_YC90sI8^<{SrANz zMjdbp)`VkTsR2&5nV%Uq+UvE|Zz<2R8ftr?|KK8N7@JlT%c3 zJcHr-ygZ^~F2P3~caT|c z-9b)aL!MxgvR+^a_ZTn1;0r{~&QdbVIj{=#eP~toRjHm~-+P3k$_Y0jdIl<*!xeY1Pmv?`)+fNLi*hz7SPH*&{ECo$TK=z{{S@c`*;YqRhy3bvU8t-1QYXzY3D~aJ*q-l-Y;k(PpyoMr=|PH#|1T!vv;MwIrdle-j_C? zw9s$vLigeJS~?rYUcvr;;;M;M`?cr&Cm1mv|F&pz{3+VS1_rLH+zCFNM)|V>MB+Xu zo582i9OGx2Ty5UY&MkLUO)Jf&3{7G)QFpH|0rm834S*;NE~K`da=SceayX4)(w8Hq zxAq5@+rf*o*GIRfNBzqE=M%4|*Xymv+u7r*x7XVF&cUmOr_jn3BrPMwmcfOCw@>Xt zZ=;$1-Z!Uicd<{CgFC%CxSO}HHxCD)p|za``m3F9KJBgFwMuITSIi&0zPGi0l=g3b za&vyWIrF;Nx{%^BVw9h4#NyIgXgS!Bx<5VB8sPSQ_Ii1}Kf8Xt%iKBby%RZ$F{&X< zc)H82?Of08^uF2RUD`k9Y2)+r@qN{LS=n8C=+D%OReL*qyg&83IJde=etvO#9DEKP zMXH_5#Q8|_wNi@db?5A)_hM)M+=%e?>e}A5{q_2Rq&g$l|MK;E@Able(xK*{m|v)^ z_3>cm&A;vS_wR=rZ*|I#r+Pb+ldpd5{@MmY{23c>es50SA9Bx$U)bmFM8;-`_q~zL zl0C-d2EHnpEgrDARr*+0`E?VY*&yG!#{C`@x>XXo74kpPxL$8!dHxP}I!k?e$!pkN zuw|3<6?-q$A3xU*KiB`n=sVupEb`hU0Rrt|NduEM?AkN2)|G2os$+%Uewm@y)v?f9 zx1xQDVf($QAF}$&8B!TCP5Goy91#;#dAJ$IrP$(KcN?<}Vi|aJ7uDX;2>n+sB1TMIYgd#D@4_rWJzH zNJ6K>boQ7bS&s1sNG%h4x%>3@nF(1|@Jl4d7(??l2wkyt_k9t?j(vuV*{iRTdY|t?!603PTx`9%Bt&)cW~iY%~_}z9;Ri%PJ?h^RP)r1Ixi*6GyJth?t{1jV!-ji`=mR z)6{Dy#$UwBQkq>l8cWeb+gCRy!>Q%Wtea+1CQ-20XsxUt391SZMw)lKzlj$uitPK2 zcY_&bv7|VR%K+?hzyv2jI(M0!yR2?gV+I97O%Ul1q)EQ-teR+p7Ra{jToRdrnR>S@ zSU1lWnJ{r91w||F*GA!1|7<|_Lu=vs)7lJpX0s^FSDksB67cbL=kG8c zi81)N#zi3ah8tS#e2w4QmS?khH2`1VNIKt4aICD~#4H`deSHyopNKXe+t6dHi(5k| z21>Rwn!TDD2N#P`tVw?6w@2V%uQ}_2yy7g_%zL;Y=wXvU@J${-a4&k#JI`|l21Qin z3XB>58ie}I^0F8oga-0l-~vk6$B(pq*+)(&-Z~ia^XsHBI8ushQI#OYY6?=0Nq8NL zJ@_Cfwm4tv1VN1yNCB_zp+Z#xNVeND4GOaHXP)1)5+_4a)dgfkQkU^^liCzuuivJt zNaVqpi}R-ThNl=><;@RS&GR7M=TG;uC&8*44RKl5F*iL+wZm-phy zsX_%Vcj5{?iVwdVI=-qMDOby8i8BcSh+ERxkb+~fX%^eByshtr+Xwk%ioZ#C zGyq-UI=W-BTH~x=NHqEJQuP!+%^wJpN_Vm6+p8TZ#eon{RHi7+VLTpbY(6<6geAd5 zoVW);Gb%AoPsW8U{9bCekTNqi%qy(6mB=%e!c%m(b(cc2)>P;{=9x#9j!vj3Cjl{Y zIE3fYju;Bmj{NLx_kCWUPVF9~gT;WCjQ=>E4t!}qS8-&4T~U5VTO|ZE?7UUZW$`gC zrDbn5kRXOJnt{^0!TT&-5LY=d!fQ&Wk73gvqt7u? zPe%5@yCLE;q3$b?S8Ma8lbLM`Z7wacIEU#q20wh55`qy$FAa{KM)l=DJ(HWz7NEZ3 zLBp`;X89$}xc3=9QYUbys2GW!+Yvx!)APz%?Q)3&^N<;XA(-KD;#f*5*Gcw-r@s08 z!RY1=@RM?0oK&rvB#suMNiAq%;A81TSo(=YF4(X!Xkgw|wF-j}0xrN@73J_-421ac z?9Z>Xoda~L@^1(Rb$e0C0>1E4+hZU~D`Puo!ofQEd$P9XO07i{O0A0ApI@A>B*>l7 zWkf~RXCO$_gZKwj%O(!9@R^_RhdAYt(x?z-iXe8}907CsS5BwjLB*xZKOFnYsOy7q zTa0ASTIeJlSpJ-XSY%#cAua3|$#P+>-^3`=56NYYI`mX72wYvr7WnK<88c=>2TiUh zf;~@MRmWbz_MM7K))$d{XT&AvXYmbcMwn!{YGn+ORTcD1x!VSt4j>^g(QF8kWc#;> z((J|jMMph_5N`5_$v%{*sH~ZUzKGU=zI1R(MkUnOiMC0rM|Ck`A=sXB$nNN3wd`pL z9_2fpxLv3*vnGucEHi+-F5_TJFD%MccTqawXKVBgS3H0dft!p2XTumnAXKZ3_Tn-L zDXrfRzWP){&0Y>JB;)Ls>`cKDT+nb8WmS$a$AUI0vLqKH(OfVYQ~Cpza{6i7ugI)` zp+~oM=yLEqCzNP1lsRi9gz$F?_|G2tgUi36-C)B<#& zpI#yf*m>=+#o3h5k`le`0H$cvo8A~mmP=Ks%LFXHSO?fTf-yEGZK5>n=_ODWc@bTX zvI`Uzc}dPa@VvPFWk5 z_*KIv$Wt;h_PZp(N#AO}djpQ!Q`*(l5CX=isVgBHc{Mc^B-t$<-;b_2B<;8F@v-xe zL8c|B2w8NJPB(3m!m?uA=4lmKusF5z>^Tg1dksnkB~vP4O-Lu;B05WgcDySi^oCgMjrl%+NLMyk&eNQc&_hw&h|3J6qF zx|}W|51mKM5rKpQ{ff-O+7+JAw7&xqWz!3mdXOldgcu)aX)7FVHa=&*t1Ni*h%c=Q zx-au3xZbA-@$*GAEWK$pvWfX?9c16?2n`Ynwiq~^IwhK?-UMNDkCGp9r&#cW1l* zLXEu^xJ+qdnG!PVUEtu5y0b<_UM;0XMgY;4V{z@JwMdv&iZ8*JS_UlAPuCh#U)X0g zrH!HvBjStl(s8Y&%>Nqm#Oc%fn8C$=7zeTv2<3o{Wu^R{oy8!|QYePka`=ER9#(_r zCXP;PeOsORB$`gJTYBKPN$|jPC*SbhM7nWT7J>jJ8TM`UN>?FDn_pPc5?x#ctO75x zfU?ljMf=>9co;qoS7Mm+^BV0>d`)vJ#M%p9^T@Pvj;O$Cx)_cXAsnB<2{_Ena|jsQ z(}EClg~`FY_vaN9Y)>f&Ow1Rh#A)^i8u$0A!*9kDJNwHVBb|!VftCyo)$TcJBtq&A zIzDiFU@lW?dUbpcbR$jAO}#QD@sL0PHGyb*S6C{WUiOHfC?r^6Q0lY~`W*H>QMr)d z+?s+4l5@N6hpxmp1YbK@OFlE&YRrM+U33KlR#CqV@SxYuL!Pka3nLx0hZHmP66CGo zN)oyx^VElqK{_d$qZC>61c{})q4J`qt0)U19C}sxWJHzy(ta2VAZPo!;Oh5F>1XAp zT&ei34oJ`VCo)^KIP?RkVtFJ$*j?h(WGlW!3M$)DA>VMX;O{PQF+uTQSckU z_Mt~yloz)!_)6IqV90oA*R@TRAcUkh*nNH|f~tJ9G=+mLGdQwX{djS^^nDkKF7Ho0 zY?R|9VZg%j2QxM;5dZ^(3yOpwOkowbe-jbvyYkeJiBmVAk<4cDi$4TlPBx73(8x_pZSJnbh_IcAkB`;uFNDB&a882jo8U`-$+3#|3HXoL1 z#p%10qhg%)tdf76H%7+JKCGUL_TR`%4Rp|YMDB8{2|`xy~(M5HeTcuEKAX=g#t8PwQ$}q z!6Kp=l!AJ;)}YLWOXeW}`aVUa-aomyk>6g-er5Q)pPG1Y!QGz1oXWh1)R-rw($^*c zx(gG8YvWYlpM+u(Uq`G0OIrca5v!q|i6Z@w^FDkTbRRu|BB%pykx254LH`ZOmr2{^LqJ~m38h0_vP4+{lvL9iELWMtlcY~e1=7VJ@Z zFQrsL@F;ZQv!>~Lc{=y)&OFJU6Mz!7CNmP&VJWbXFo}lF{)HzY6o!UaY+l@t*BDcz zHW@x0gR;Gb&H|pwa0sE&lTUxd=jSG{b$`Wyt_x-$!tI{7R_AoYnn7K9_uiRl-aE6n z)t;0yWY<_mlJbC4QXC@M#d6h-rObZc%(|qXaOCmU&2zqP2}BgJ*)U=)SxHoe5H)js z>ir(GUoWoJuI;{X&_fp;A57o&xPFn0h>&~UM4jKGoxx1-%`Cq34ET38y>q<1```G0 zBv=rp2{gBsi*(Pg3*5x*c8gP`!WeR|NXwXk374aVnyg zS^`fURxaTsq{?v3B~Xi9gxG*b6qFY7vI)9Kh- zvM%Lxz9w@ZgXuiH3_?>m2X`j$f~g^wK1_$5*s)Cp78@)Z{>gTpmYP-O&%dU9Mr z$oyc$Wj7Md8MHaT=JQc0TJCU;_5Fh@3j{9uwF(qLbXDy4sOzPA!3?nQkKp@tG`pO# zK61u@I3=6J=O}XF?+3_ZcP4(CR9Y8ev;fYrXTobU zqPVSN#`X^t^?vn=pA6m_*`Ad+id7Iiq_h6 zsmjI58@AR*!a=^%VbDUb%!ve1MdmY7g+%<02MLdT=1ziE^T;j8--L>Yzlxj+@${z< zzFMH_j~6AzGbfsc6y83b&!c7)qa839=l&V>lyRZVrU*f%nn&!*oAf&k6h|X*$M}7a z&>&Z12^!(IW0Fs`98va$K)z~|##9N361K1Y##_Iq{JAbW} z*9d1Vwiu3`AC~-cQEJ^`8)Zykse}>|%YM1j6gttz@1-XHV;U81z9tf3@v!7AUlnvh z3{<`lvACxy*T#K@AB^PsL&o3t!Dlo@Ezqb|FZqk@(rmQe9g*CVFjLy1kr05Qjtoix z*fq=NKp+;U`Wi{F&_qy+?a8sD&35)-Kx~{FBkJ1@MC-H}6#2QAq|9#4*Y}L^#V`#- z!g~XR&zSIpKQEpYYfx_DW#zW4%r&=}uFgl9jeo1fLE_j^XWsU}LKly%^}O=1>Jwoc zd_NI%;0iOjN|bdD3ZjZw(bqJ`YF*9%WDN)D*m>@))D|xzOnEU?4@qT1cMcg$*e&y@$$fB#iip)2+{N)DbQMXdlyn?Y)V~PJ^ zz<1oM45u{k;>p2?{#CNKRx3B=sj^LJq2|f z33N}Odjj1P=$=6L1iB~CJ%R2CbWfmr0^Jkno2m_XN5p&^>|f33N}Odjj1P=$=6L1iB~CJ%R2CbWfmr{@?1J@Asqob5HvJbVGAy zAbbMh69}I`_yocy5I%wM34~7|d;;MU2%kXs1i~i}K7sHFgij!R0^t(~pFsHh?*^YN zK==g0ClEe?@Ck%ZAbbMh69}I`_yocy5I%wM34~7|d;;MU2%kXs1i~i}K7sK0-wi%l zf$#~0Pau2(;S&g-K==g0ClEe?@Ck%ZAbbMh69}I`_yocy5I%wM34~7|d;;O~|0ejH z_S8DH0tEn&U;u!B_owb`f8aAYWz?aN8C&x13VrAa=1z_@?}%L?QI;%P(p9Ie8(h6> z$RlB5B^DtV{LUy?A@JhjVvjB4eygx13eBXwNFng#%NO{W6B|v=Q1F-6rUzV-Ah=cY zZv_Z4Q$jt{VviJpYPS<`qQN8(2ujK%;%%%7KbWMW9|D_;;yHLMI56vn5GfsTx^))H zYUBs&K^q1vf}tnK7^8=ehkoW*k3h%=cK^@t+oVp)u;;93<|sa>&P52IaM*BTm_ zmpCt6^gh7D|hc$0wq@NX%&|;zt){%&5&cvFUF! zDrB6OB`q8vT6=396OSieFbZyNauXxMaSV=Bp!t6E5)Y^Qc1rR|`3BW0j=Rv@+f&ce z`Z&hk-qYlh=H}`~Cl_~nWv`J}43Xq5e58=}^Yk`%Ln-@lF2r?@;D>H;PQkMj#71f*du)WPK%j0WjQc# z^!I@a|+_$r<47!bS5dpW-=p(oWEeCoI}84OM_Q6XHuT#t7_cr zoIs#eK}6Y6ExkT1hxiEdkYoL3W>;b`~(=xePo4XPV z1FjlhLymB64f&=DlY5EuD?E}*N3G&Bq#J8clj_T|yM@YZigf5*<7|3FWC}GwLmehARG;A}T%N(n z@C5)clG08Ac|$VHh=t@oa#3TFelH`tIGsSmUI$4*|42dQW4YAq21^zvBmXWc3iV~3 zET@l*><4RLn}Bm@N3&}kNyKjXCKYj&?rq;Qf;GXh4E?QJxgd#FEqCx0Dv;& zzt0lKpDaZu8#opbqlcW|X(6A{g%!r2HQCI{f5*zS<9u>fz(`i_2e5g(y?`oOwUG^f zEX4PkeClGszj|uPhq9u?z%m2r1&DPG$jz)2NM*kcGtSBjxI|$@glg>< zV@If6Oe|9b!@Dsh)lq6%`f+q$WSqXRTD5cq7|s9{3=SPc7E|9xd`WE3@x=!Flz>HI zX#cyFF)y(o`S#G`LL)GP<9hDc(bUhf{j{-M*G`Rgn(hu9tfe^EIM4A+;7ZriGC0U7 z6b)7EcTbX_%c*8%;*9CN`?P)*CX&kqvA=*cdJ)M)!F4C2^-|o*GunLPSaDi_n=OaR z(|8XU%T2|}Zcw)Shk;4W3;mEKK`E;bZqxiV95Nos#Af~$$V@@sd@hm5+DxiA_Q{?| z;;~S@;d}7Hl*FFnva#w{oD!NsHiwYHJL0^+%&SSf4noNdrRzcJeU`GPoVR@xQa znocsGccgnOumAWJkKfJvrKEs=lm=whxKiF|+DPfC7Ed-SwIrDgx9pI}>ALwuic#&r3ZkJcOh$hf;(~*6VLN! zZ8W6wZJP>+4(pw|{S}4b7?s1|*3N;dW-Dm8%LBsgSwhl_eh8!A&tkEY%+F$I9(8l% zpYBQQYoo}>+yauVKFFT9@X*|H7gOto2rgddAZ&>QMlT^y5ES?;75E-d8NNaMeJVR! z@N7T)7iTzJ|2V_Zi!ETr{`0(#Td$pRnF~u?a>nOaGH+Pwb(e~W1Cd>>Xn6C}>jS%K zo5Cx;{WW|0DWBE%v5L-WGML^wk(B>3t~Uc=a>O#TV51Y$2yItdA~Q;}-Gt_Gko;qY zZ#v<(`#1=*k?BCmk}~Cr>~A%z=v5^yeR@9+bFv~cXxGs7w3-8_tMJsd7%$p~s%@6? z3Qb7QoPYB(aoJ69wPkP|QKM89$TAsWTF_0~Wh8%}JQ+lxSDI$asq)>OC|Mixm-UF# zT8LR1;yWNJsT)z<#hL3c!CE8rgQha$S<^V85~9~>Wk_PwwGOfKqG(epWmKSuSDT$Y zds4Qnkd7ycF+GWX;TE6ZOSDy~sQJ{87_s0Vi+rXb1@ntymmX(B;OF_^<3Zgb(NB_%@EMz~oGYv@L02>? zc@EaJs^}*&S})DL6YMJi4PZeA~ zu3aPLP3D^rxzp+7S?#rAklqkW=ewS*`^E3dxTo>Dis#^Hj`?8o!NBsC= z6<(n6u}ob<;S=TUq9lv)zxv^df!uh1j>&pE7g5|s=O`gTw|2fpfK7rac*RVUvVDM& z-`P1Qw$7l&_^0Z;F8{9Wr%O3O+bc^xKq!AeJ;YfSo=5Ov1=z5C0hF@fj5S&7RgFzJ z0sL=X%Hf=^>;@>qUwY7bCwxN0xT4E8WwfARMij^)Kc4wKG@}LeTNuI-!MzZJDFhH5 zo;iohq)Y7?fl7WdDV!Rr{OQ>Jpo_(gpzIQ~Hc0vdPhG4RspBbs&>;wEp*gJJaD+-D+0f)if?frlH;l;HT z0%v8nso;~8*{5RXNE0^>rusAtoiXGhEmGICoAx0WP4kCvMF*BBhZ^yr{*RFBJzNgD z`SeHm7dkfe14Hr)^vjA!)3Bop6HL#&$Ir8%X#5+xg-uURJrdkPKQEd7KFOFFr#1gY zr)2%>-`@Sb4hi$v1`nTb$7I;mFP&_C2a^~AP}t8e`FApG2Rs@aMyozv|Jo}4@-S=;X^Sh#i98;I$7 zCpqb&m4`5mRG?93A0$`IUe$3kpkAL$a1z$fdT(6iMiBPRHx*M|PArBpR$YPf;5Iof zS_p*^t&Z4;cr-l^Bhu;G%avm?OewHxM!qiC(sm>a*$!E~`IouqP!7RS#vjXfPz(R$ z!*J0V{=vHfGn~;WHAJ8;1w%-FPoKWN_7iXU<3V!`m$NTb-`(9g65`u7G^HS7!J=;$ zDtyYOUHAP~F*KBnmq0eLOS_aLDnh~`=>?HNqL))2melocyOJRDj{`Z%>8h24H)~eDZMEv69h0}^c!~$NhMAE2 zcwOQXY2E?Yo@4aHkg@4nVHX2qsHV>>SCA~-X*weE+?ManE;=2dug_{g6C_Km%L(=y z*JgaTJc#kvx*nz$cn^e0(BFL_2N(;FCi9KT)mh=Z!w(k>t44A^ zZ?qmy%v7^IOI6E%@KY~P%zx-k@p?^>c?0`#94pAV5SY6*8i8OsAPXv+xYx^C=a_X3ET z=o*q(^eY^V$x<+-9Pzdf1y#HOGv0>ff*nMWv!o)EXLdUzeA#zwZ{>#~`UZL*$JIwSBM2=ZIumNHxR1O6(;rB;AkzKFYDso)%X^Cwi-G3$2;ZH(4{zpPH|4Qhn zDUsvS>#ZTO%nXkeRODUbVVGappM>Jv)J{Gg{7{3racx#wbK-5eTV`k808qLNjKTGN zGqVI?@fvFvjpDLQLEec?=L&-v7=tzKVQC0SvmUb}rW9mrhTY5p4N>dr0EvaSgarge z!*;H+E}CP0rs7P~`jbh~_e|pTVgKoo((gS|QWY;Od=s6Y8{#|$9k5UNMGw-f@Z zKnwp7P@`|1DOJa{*Gq#%yyuG5svX|SHF~x`16jOn4_$vhGbX-5{XL+oVT-l@RSEs; z_eg;O{eK~#*m(1LfSkqO@6nE*Lp6b9|Eq-l6a4?@i;e%SCeHwD^8Y`Yyu!Xd z|HRsR32m|K?7$LFaGO}_-WFisEg-V>ESam2^1aj9IHOBe7P5MpLnw9qRI31XgSEPP-L(sY~6tP(ImW6TY)iuZfS zxCFqwsJ)f?UKGuK!gZnH<^0DQihYIq-u5|>(ca_f1iSvP%i)0mYplO6hgse)huI*) z{CE_)j&oWZ1>q z;lF6||3Lr0T@EMx6Kt~06UHr(76@oWA*pRDCtjcIQIO^w(K=LQa|H$3iv~rk}O`2e8q{`X|_A`~e#SbiBX7=EFPKbaT6W z(B(WdP7Bk+=bs$q$h^8~wv6VL2Cpoe=}nbbl4Z}dEUsa@at0q&yFOGf`~x-ylbv$S zsaOj!>3@OE*AdlVgd@YcNC)JNFNt+o4sUBCE=PrMI9fETb zcQ_Y1B$bh65Yg@?R%`fZZ` zkB+whifiq*MR9j`*FbQmacf+I1$PSx7F>h7ySqCC3+_(v;ND2^;BuSnbMO2Aed<-g zf};3r>X>7Gy{?ehN5ULjAWCo$GNBGp%#3Q)AqY&W&S$wLLb-qOZ^|-Hae^!0_uvZn z2o_A#fB`9OQvjI0(Xu*ymF)>pKmQDbG^$o>u|TK;ICVQELG>?zacI_m{D5$}txF2< zbP#BA?prTiN{n)Tmwc@0R;cXg;Pzhyah(Q;6zi`??IvV-uhj4I#P2@(1U28dHvrGq>8$PCMs~SBfm?+)t!*D{Ms(Fi z%591*Uwqt;5+&ONF@~M4r>m%4;iw2H5c#0|KfmsY_&)Dtyt=(!%ubQ}Jf13g$gX=` zy?pQTzS}z3czIdNczyJ(er@o%nkbn%^LgA_V+F=CGYt#QWvx(Qv)<`TBd#`s>5#>|Wp5neR#*xro=(pS6SMsTg11$HUv0 zvyJETy?jGo_v83NugBx|9AD2XyDe67^16$yA#&fBmB*L9jhE-e+b(Y(p3948)-&O= zQ{RWfgAI`%_b)5tKI?TQ8&?xUw<12b`-Zb!tNI2%+#jF68~%Fvp0n|KHG8~L@q~rBst?Tvj-q-i;;O^z1>-E~b>(jt2t?%Rc zSPARv<3x>!$l4h1#?!;8NY~S=i0{kRmGZ`qGS+IApZU%M8J}MsUSpG-x}9;s=_CUDMQ0Mb4`C`c20!}Z)FWpKDP7k!CdO0a6iS1BW@$wQ? zDD22ueFMZsI!=tBZA`@bp!PwW1~%deR!h5AYbhl$y|iv2uLzx%$Bd9^iGmVxfmEi)#5@Lc(V1Ndu)d;BzM@yNO>@4z#!wD$!Eaz6 zm1(hUOxs?-frJtAbF+<8^!Q5WxCnE4+!UkVpSQD=!I;yEk8T7ni9zt|L7?$-_==>cb`tK8;JZc zh}z_CTVt8pwfddtjM&#}hO3L2&L+(uxMBV>;C(w?9`SdqynXh?;6Q~zJy(i5=wINk zOc7a}=|OCj%O8S1o$D&x=;n;@$L}kC)WP~RtPy*7D3ik>+$41G&Wpq+K(@L6&e`N& zza|!_Rl&jUeMAt3ICbZT?(8Bj<6VQ;fS)0J4riB6r6L>mOTp!@o!&Dabc*C2T{@Sk z&-G6^W4#l=*o_@dBH|poQNAYG3oh{DiQ98b@USar`(jA$w;6Fo$m;IsWeR^sJ^uTO zz$Cm;#PpHKyw+D1i_EWju3K1jvhS=4^|wR+h((L*9THxhnuhV+ z$}gF7e+NxL2v?F%Z7p(@G2M3@5B=Af?EzcA0>0nGC)4`=D2R+A$Ubv@@as>4iK@*?||5u*SAJBID{Z9MU+!Dh~ME(q659kd`RXD=RT#8~Xc?1QRx<~bp0DNL09KyY>&gyRP57RdqiO^$k z6HTtS7`Or@5fZ+RE83mMMrPWjQ(d>BF&Hue*(<$^$Sx|3E|3*=>pQ2Rvwy2&M01Dp zGsEda*6>eLNXg>J^vY+m?wqAW^X>d0F}dz2WLFyom|CnK33V zuY0O$ogBwk2lL>au$#e1wHynz&T<+G+hWMyOtAMs^EGqSzPQZ3{)j1pQ@4DB1Zrw| zW*_b&@v|>@8=ZyX~z=x)S+D=62#vRZB+DJbm zXt3S5>H@gg*;Fm?4IIz>H|63;LU8sNb%fzMFsxBXOIeD>%X<9f0$amKV_o_9x_`Lz zWVGB`%rpe`^mRlLHlfwej1*5kbMi!bit@^fjT`SV{{dRU%3B&yF#%%uhjoH1C z3g+fKm>IuQay80VsVjsJaKa~3Ku6=Il*JOKXUaB#?C3JqE+C!p7ncsj7VIYfyy6eL zu_0Qj@l?EjPa0iyHG2YN8`B@eqzm4mJQ|D)@6vE}oN-H+Xf??F18h!_ zVPv9RLvAGcj{c5j(o%Z;#NIJiHEw%q5BEH{Xo%ROwyW)4htOYj_tGYN3EIb{0KUVb zf$wJr$;N0Bfoxw+O4KKu%okUB`iyckg(`hfA}5OHTCRP-PXc9c!H^@})|e*uuc(O} z2SqQC=hD?(CfJzb4A`yb_v2Zvw9@I92lm*+ ztDdK)kk8|=*PQTeSjU!=1vzyzGyY-Eo+Gm*WC)MWdU)KV<>FY5R(|-)_6YKqQ=t)h=8-*OZcWUR5D2^uPbcQ(PQ>{c} zwr}kza(Vr7Ta>9^IUTDuO&md7>F*T}TV=!;?x5_K@f#aWqRc|xYM$=R6udL=JiuGzyH2M%c) z{!xTQ;`C1Ebpes)H>tE=Ilcv>Ur*2-cd0claoT??kB5jIbG*26!KE zivSVS53yPtz3EHvWd3#NfSOU@#yj`RA4uPZRurZW_;<|e>2S{cB$Kl~;IBXYvLFkK zEa3q=)D;yg_dIb%97+=8In^hY%s*dXgFD(rLMjz@9|hylpFDDg;q|}fD3T{f!c?XQ zU2s!lD%6L6FO3rXg#aViD0a!*_yy(HctPxkdKoOK=6aRGR$o6(h!Q0tLM8o~a2S&5 z%3oc$fJ1WfO$E@<3z<=t56`nUhuU{XoyBmDW`>^ zsAQg&KA5QANMeUMhVMv@gRdk6VB{CIeRX;(eVA@>ch3 zNvA93Z|M`PgveT%@r@QVK1Hj@yv?e0Cr&cN{up2tpg7iR6OI8!3A&sGxh@*hlx|wo zxA8ajAQ9SL50LnfKl5^JXyjK$?dF^DDv6eIDb6_?jR0?K?!MJpRAz=Pz2|XrxQUJw zO@&U^*6cyuWb@==C)4_Bv@JW9jDDRK>lwU}bkCp{NR?j3CKJ5on%^U5AnE9>|ux;cT>B7Jvr7wC>Ii!Ghh$ z)XfnDw?d)|oyq*AxqARg3;CaH8xGmT1kD~gUp-#%)gfCMo-Q-e<^kmo%}XvC;iyo@ zA<~2s4lz?W9G}jnoqq&uHC~I(rVxaQf1F=&Qs0Z%LZf3$@qpBB`0_7_;54Y?V@wPh~B@ zZ_7dA(r-@9v6;E-S+LOhu6MYqp^(7Z8&29goLw1kg_g|~jLJnm=&=k0WtdNA7XY}}ZjfUA*fJcy!*?3|7#I}LfBi}Iz9)~g( zed)gTut>3)M(}5Rbc|0YIbf^ZqYg$7WqNEcHB>P6*Xwk4fuC-%E7xEgdijI(!`x z<%_|q22q8~{LEthU&?kaA$ez4cbe$wOz(5%%zq>oe>T~o;PUAy{bR4XJ)$z+zbp}Q z&hK19G#?XLIY`EB8&LtC?x*+-<_#>Pd?gfgwG>QsP!Rw(`O?fk0d^9vMSIi6KLd6h znqrVW7l2!<9Gz6sTL74^1mq#=>#_1FhV~IKb_Oku`F!z{mlY;!b)WRu-EW1VEoXW9^aGqA zIywBtBuiRAxhyqKkD06CzZ)u>a0(f@JIne#_{=DLZLPyFSikmGJmJZkqU4z}n-vq5Bc^srb8LC>Kh0$mpAR7eixPm$mN0(&1bj zZK&W_#Mb!9FHaFS8`_p5H?e*QrT#s&B+M$ty7ZzRq^pUaN@?j}7evd1%RDs?X`F{J zEC&mE&&-5NrV|!4TWy3GWU3I3ii{kHU!)U5u!ME?)5G!KzMKm;mk)Iea8MDl!DMav zYDA?hb$V78g&b2F0<}k_g9f1|$F&B*V!O;OLM7(vq5638s_=U(mS8q!G(IAxvL8q! zt7b~f{TeFBf3NLdNJYVdt8y7OP}ED!;_%=NPiLb%E$bC8lUClmvVDR(;eg)?$!u!w zC{-*O2X#5u+@BNiQMQepQ*viHbZhyCyi#yQ1r8)_c`cGsQ;lvnr)W{w zS)XXj2{wm*2c+(pt;y~}YclP37ILcwD!g3fD^j7RAmr{b2_^3Rrc$H~1h`fdkOr+? zs%cjFHAtgFVbLFvo^y!oT%$o%KECcwcJR395{i4u7M_}W%KbpS@Sd}?^HuaIW_w<&AwzFUIr}cp6zB|0N@KNv79NA>*bQ_IShcmyCvI1 zY5{`M0f_}fayT|__Yg)=JWZ46W5ZQrItp`bfT_tywlWREu1QUc!qwhE6hsbsZP;%l z81-_cssrMhW#XEHpPSGW3U%mQ*gedKlglB$(AWH7+T?(fIaUr}3`s*$mQAg5Ul6ox zxDTqb)I)aM%h@O>s1A zWaAAqq_uTvwP8Y)p@YY9oa2R1l?Vh@Z*L)m}TR`T;L9AxmWwBcNt#LFoRGqtGzjOv)0B91kP`SN(` z?<^NhP6HMVe}a5G(=RF9X{F=O3KZw;iFMNMD|9y8l7foR|eOWb6 zRDB_eQU#?y`L~~+d<$mMkPFRmha(5(B)MtBMG+NcMO+_N*p|(5Yd4RY&&CE*7eVk0 z8c-h>N6NCSLamHx=W*jh>CEs90QVhN2d4>z9@)VtR;s<90AWPTsz8l`!|@YeMiuMz z%I$Y{^pdc+-O8`_{pWzmBtxn`2>9KqBj&6xXdgbh@4gr2Xn@9U&7M-r6QFKA3LI0l zNh)GpD|KP%<%o9)t696V#=*(Kzl1-1F#_P(FqK5u$p*iF995&+>7ZS*@hVfvw#l?Zpp(|W zDm9`W6oyZBy@AHk!i{YyZ}959N@>0X>XA%T*V9%C2cQRzLE89T37kb$$$NeB3U3q! zi^17N{n9=F%;&sgnqq=5v4g(Ugq=^0Np>IkZ>bTgqvBd?kmxUY z40}#|28oste~@wRR@aRujYaqdpQ&7*3;#G48PxlMpt8@dciI8}GY(4DCtcXWYNXM` zhPe;!g)FDzq^VXvIodOM1)NsnGVxQ25q{*sQ>5Jp#$r#$DHtiEG-hApEPxNwxhZUiuQ}%hkad3O9&-h*|mKpqLKUS z@5Gxr`t)ITgM=e-0*XRvNPKD@!p9&4*NUS2)hdN|FIqI&w3XiWHsA2+(`GV-4KN-B za&zZvBVutIe--39j7q}OWh+gy1^@Dn!ookOMd3!Y%4{}VIW&RbjSm)Zyvh{CvGy6< z*KyFpSP$V~-l`m*#vS`5hfn>QPKORSb`h1Rpj->*22YmfFm($4Ad!uAI~f7XjKm)s zoO(AkC)f_7xPxFafcLl#qZE4PuCT%hq{uo5(ib~9J+VGG1iON5w+Kv_-u7}&|Ac&I zojc2K^#qgTs}NW7-~^}dsd-Yu!z0CC^F-n2nuv~a8`?BI=eU(@k8S1x-2f{x=XrEu z5km)l@&-mx_XhF%#69%(`n`5doztJ4)K1K!ICxCl5-%Pg6|2y!Z2suvZM7QDk*tcp=kmB?T(NR%_OT?MIMfvF$ncX}@arFV zr?aCva;M~CMCICc)&Vb_4)O_uBw(5UB&21g(lHZM9_u`-s<$c3Z49kbG*qd>!i-8E zdSSErtG1W+cv+SBcHf_-fsP`U+j1fZS*$5ahGM725uQ$uPH^0lVy-QBE%&s~`!Ia5 z!7N~6G5!kGt^6YrosGf1j>Q#n`i7+{b3IsJwz8b2(gRhi8Sp>H)BTtiGWYOUpyqkV zbRu0@oKZ)lb{pM0L?j{G+$t@taf#b0-@)ipg)n$_rf(Q&9C(+M5d zt(h+?ZrAKSMD1{Y*rS;iu1zlNtbiHuT0C~QiQVEWt(gX>Zgft+!#*C=@iencv6SiZ zKYr(s(5&P&p=s@`3Ue@+?InP}u7hKmaEcUqAaFge7Tc44HzalLf4-zZ1j2H%iCAQ$ zzU%KiM|)p!QA8OQvGb{yxbIyC1vSQjpTL4>Gj4@xlnm)kl@w{}m;2c|n94vNh7(Di zvs$r{KOImK$wiP8M;#;Agq9(E>$xRD=EdhEpWwm)d(IxFr-eqStrT* zS&@NcmwtK+qV6ooV?;4}gSgi1aJa>E4OA=G8C`PFI~-Nbax1d~E;c_{r0v7;TL}(8 z#M#UzvW&$0#0oS!pWrjJH)XEY0TdbBkjs_)$843?jZrc@JI{ioscyp62GKpZ(VAN1 zehI&o%Yi$?rF@_zY0WuD&@*W3++-3a6Ogj}5NSWmls(U`^z~ywB*Ap(fwgJj@^Psi?CPcFykYemaUV;L=KyUz-A{)qb zs!}%{S(;{YgppVF#F6uF!~~rly$p|zn1XkMqiVCHn#!Q^1g*Bp`K3Q+pap!SoO!;+ zcz$|WdGAaPih%^3=)iY-k3I#`NJ18-k?O^%#9+ zjs&731jxNpLOuZ~TUU6WQv~Mqyd=q4W0OFCMD|bc_kasiTeT@l>C}Gm>X={{vgfP4 z$KkA$shz|qMwcG-Ktc5Y)&RfqTNIKDOZ~I%sS8Wvn93I2=9p{XGQ5G~Iu-|jR<|^1 zobzc&s46U5I6Sdeb}ct-Z8MSJ27J?BH}t%*W$?IC&bZ3lG`z$F*|W_To{WJzhYoV< zjF7JlO5PW|DWuBhI$RyP!-7ai!u33gObrUl&#KsmLA0anqp2oJJ#;%G&amlbUl{6BCUd=G ztP!H=sJiq>U@RKIySaY|vkFnd7scL7LpIDA{r2tp)Ac7Xvz~q6Dls8vL7hzPUTvHY zJBjsAsTZL3_jEW@lZiqiJ4N5EIefqgTB|B{Q}LE0VFF(dQ|oRZ{dFcOy^QPtBe6O_ z5PNyH*e27JJLy3Tcq1>SyRGxtbMwH5qMlr5hC14_Fj2S518nJ=A80f(KR8pfW;e&% zDnqbd$|n;sx^2fP+>cPFT^GoBe&3dg{=iVAL8uabB4?p|ySzh9q(X!Rt&Wii=EH@~ zooX=HF)HV(MNG9uRM_{B40N zSB>mv5_FL3njZ*7j`gaxdd0YHc!c@d()ZPB;6sw88ok`Rk$ULXUd~>ot?U*FWW_B{ zVhAba@$@A`@ZFJm$P5(2OzGRe)e5HLx7Ivqai*u~!K2lKguSa5v_ zqKkd#ueGg^F+0<_9fzmh+e%6kJ55rzM+E& zjWuq1S!=A-0T2}5?I|gk?fTVBYOjinP`AxNG(dV!4$tjTur+q{(cKK+&HRfAzM10s zwGk!yP3M?+|GPwSrDC&iR&Hpg7U2vc{6KT|LP|1z$ctqNd*ykC^C1}?6n!s(@k*^jO**oJM04{D%N~V zU}x~FZPcRm2gL(qZ7r>7p+g|PbB7XtKD*R`gL!L zbVJLQ4oGk$SPhu_5`$Tsip<&CjU0@;CZEPw$=)js6UJpZS(I0&i}!PlL&g0LdIoHX ztNHDDxK%78Fp7nvKsgTHRl_)zJQ-6GSL7@OXP%h>NiFO(g=;c##$n31_s}E36;Vy_ zK4|r!c18o%!dzPbDWYN2>>Gm-KN#+yp$|jmbHefs*xy+r3N#-h+zhSgxYW&9JBZ>$ zMe?i7DP7(}X9X`Ju0V^xrt{`HTS?(ld(aJg2~vy64w_ntQbp}ciP7uHX*fK3rC2sR z2eB$6k?GcmGZta!td3?exqq_ojPl3y&YQQUgED z1lmB4Pv{E;?{14^zqto$DQguf!eqi9x!+edX9{>UOAMd)vL_}KX6!64dt^EKv!c3> z2w?#<1?JicN@k7Y)CCSKCLx-7M>EQGq~I&m6gb(j8SUDT^F68lN^}AlAd~!xFnc-= z^gNt}Sl#5ANqNKynfE*I^htH%dPRh=cTr4fK(eG-G$7k{u0EUGgs@4vwp5T!en!~3 z1Va#aVUW3-S&5Z#nhf$zU^-eA-+TBm7)5a3E^HgW7Z>rNQPV;j)tO^vSrQ*xU(uO! z#uK5aHdwL-MUr)W=@(eZl~f*ixXS*BR9^(4u}nT}Rs$C;7w0;d|22jpgsTQc0}n6Y z7qkex4i{@}e#y^k;I!f+^gSi=5O{kHE2+lZ(<73W*!l|2+%prBXOuX7=^>3(B$D*i z_w#Qn<$u~MRTv>&9)urs@qmFLWIrfQHqZO-6Txy9n(t{pDl}HE7MMHWHb+-lWi<+yo*ZzQm<|r$x)n&&56Zfz^-2!Ca=q8{ufO3Tc25e2)cy1imgxjrqMf) z0N!g*RtyekdQ_M&UP3MZEuqwtfo#g#JL;k$>9@e~aKI~d8Ge%7F!W2D(AD3o9s!eo z)|5UABY6O~M#b2Se>8y;<+Rk+sH#Z3<0E@-H_YUCM|^?WazypPhh*?Aq5f~fVOqnB zmSk0XuuKEsoHP@2LkbwvtiR-9%@@Xq|KSK|RyV6zrS`mFcOcbi{i38F1EbSTPH@oE zkBi8`)i*}_1SgONvdfPMtEOv90|^DHX-NFij80Fm`q&EQ!>-HGszA22U@xW|mPA6} zh#OV>G$#Jvevf1eTp6u%PWxv>f|X#snx5NCzKk>#t_nPj2L8IIhg87eJkIxj>$N4g zzlx;)JsCFGb)zdnKN``d!2Tjb?2NaOoK|cGm1@pGb;K6sv+wuPunhmAYNcSp2sMlk z*6PsA2j-eu8|<2>w{H$J(Rl7@uPFR;N(pf81>BlBL-mamr$QpOJ-A5)kbMbqi*s1$ z8D<&&%6ucLi9Z9a>4V>vqD;SVsGTvX$Z%iO5|$MCc|S|&)6pzn=r);JuOmQRG_iKI zAn+nM9xMT^rWnj%f~{~EZ{G~L)S&-%9)bm`!a&^Izze%tu)4IR5G9!Kl|G)vZSp6? zBI-fxgu@`(8rvA-{Y%c;61v*{@K~XEnr@A3lM~0oa{HLo{;KXA?MP~QyX`12Wj)?I zhE&J8)Q{a+pRvey`yz!wr|AJ^EJGtT6)L-^t?W`=le-5m$!O+qIF#&_5L-b~JvZIT z4hIbug7uDD`4~c2SR>HHsog;zs)5tFf>#ba|Lr7+y+rZtNPtN7-RfRySBKvN(Upl< zNo0hR>g&~^hx%4xx4?XK4{5e`iUYhbWcHp2K)kA}n8}Ah5gT*4{11`Kp)wa#UF4PK z9|PxVpq!bMGVIZxR%s=l-?5cN_-e<%Z1hJUw1oymJEpYNAp^7sT47%h6xs1DzKXR& ze2y|39Zl3gLtF}_+A3K~*m{>D1tek;CE6+Z4wmDv6cIF2=YW-+bWE9K02#{{seoi2 z*p^EiNeXl~`8zQ5{e|a2fz4_|QMNn+Mw-=*p}1Aw#G-Xo(HSjaZ4B}=amn@~Q?|f~ z-0Evd6ndb1caMxvi(l0WJ#J^Da9ZcYZpMP7m{*R&6PA`w{dlysC<=6Cs?F^J*$2tE?v$y7XxdEiTfmwp$l60=?q18$xCuab#A@AS z_ysb}Y%Y;T`Fm3|+RP2l~K(SR?sa`~pW+RF2`C60-)TC$u5xEhrmE0&2JKC{oETB;i(| zv3Db4P=}Gh_dX1^3iv3_+`fvAuBU;KC;4p(E0hnI#{X{IJz zFC$Jx{}V+aROC2W;j08+3T%oh30B;TIod?C1oGNJiG-Om(v<9H-S1>oKxMbxzyOZI zg(QlHbKVlLPb*_IfoI8fDv=)#F_m&P3Se1IY1V+RwXlg^gJUq?ac1*P6`p+o+ z&6D4lh(+T|^bV8@kkix#E?S;j52tgtt#Ohu`ek7mcsko+U#HkEm>oHiPn!&^Z z9M!fzZi+~hS|PP4k)9Ldc$%RUzXQi&A0DzM@Uvv8`f?;YRnf=|Wmr?BJj;beYnuMH z^{n`6~LfSGAL7gi#S z-IhOq3`pQgCFb7NP$k<*WZWNO_8GQ>BEDd_`^yapKppxWIAJi}QIgg|>4&wOg6(lL z0u4cvqpV`_jd<`%fOginYAJ=)t@qc8mP4t; zQVP%mSnp$ZwfqD7Z{KS4X{pN!pfWZlg*ZWKF5*3?boeJUQA zI*}4KCHE@;9OPYn;99|EqXN(i22S3!J@bL+i*1yY@=X|0f&W0Bsg=wr_RX}RH@+h- zGzDE`iPEJ8>PM(dCRl8JEJtM8%V#CkyT*cPCH96=N9oard6-j=Bo51RoblMLr(rbY z&M4@LZtMGn=d^8_tT_7J#^DD>*p74IUcloDQF0Sjumj2%Gz!;l%T;AZO(fR)2~hO8 zj>2Mi`NQk&wZ~;znSJF6!=~CdrOuw7jmv{QKCe*Y6LG)}w{M`5sTvMx{DG86S&gy) zMvj;Hzx|V#uk11^88DSAUOzCi7jRo)d_lNTDh1xZXiJ7M9Q-Dmrhz-7+~s5*wl zegL?F`?fGY+_^w5!0Cz$FJ;VYo0#08jeWGLlNdo(_?tN;#+(2!9z6J=bW%!Pz2YJ5 zXp$o|d^M>wd=(vKwPgEq14+m|nK4_DKw;(}i=;`M9P7enI2&S!s%q=pk|q`abwa|kc43*(mqn+lWT|4B1{*bs^Law zXKc%8x8h}1yDV;i8;A_0=Jke7=5hv?DrVSBK^FT9m8*M{d(~)+uH5$KxO>$%46cB5 zFzYmyv8wJ6&tIlc0=K2-MtLBIb}R#V-74UVbz)93T-dh7J`|YQp`?C4JTW|D@AsI+ z+B*9yp_dUr3duL|B7D_Bs4|UQ=I#QVs1W1dHV1*@(3h7g*=Yu3m4 z!=!7gDsd1EK{VK?uS_ghTlfaV1HH|9c)-;xb+}{}&W|=XGv2SHYPU8j(c0>75Cu_k z4VuY?lJSH08!PtR?_2fgMHep(m%t%GL8NR)>`x#jD<0K}sU8_z%^TsGhBp_(;Nt?k zs1W;T^1BS|8go(?&`j?8BRep`w!7^D6cUR{HNe;?e_qG@0p-$4k@VjZ%v2YDHZpl$LoC(s@0C{=9p|@-f??T0s`uCz+>idpUk8kp|i4Qli&T z-xO2`?G`BVqVmBx3I`VZWn3?$5v7-3ynCaJc9v!${oJTZfj5a~!+6=%?NP zX9n!bL%O@2a85UX){G^k!{z#!ei#vo;fg=fN}(Pim!EOj-Lc3`;&Xog6n}3(gKvT> zT{dtXn2-Iqk2z9iYAgQNNC3LP%zBy4!s(X=TO-2b$V04$zhV9>Uw5R@6&45{q zK3f$EEYz9{E4NA!SE~Wyy(QT)$xxYP0AEtZtZRp2ssU)q?xA5?e?4Odf9-D+^En@W zS*4@d=5A@wl}i5d3=(>-$o?H_^y!0E%#{WI+?^y6^vsYZ;|1tM2LG zjyocX(HP>e4bj9hkOooTb+@pLLWq5{$sV=&Ullb?FL&%kWtdtBwkc z?Ji^%0qIaxvWXJ~jLiSJ2<0dXB7^-@H>SDfDjJwa(p;Sm*Tg@5!-M%f6#nD}ZJE6` z`Tt$G2XRwW3qkt|!|3%@f)~=((UV}xR@I7RW23w|kKe%Bn~SXfH{-GA=LK%#s9M~g zbStY`l(k4O>HaLl4}e*&c6Fr{p)F6=v-sC4Qrr|NLeMfAIxK?cPT(gPZCWiOi}lG} z;?#J>6<{rlqg4ERl?A_tKPV)|=6{a6r@&9-D!?b&%|CW#IGru~7MnG|>He=alyN&! zOR!1`nfU72CBjEJ!^oa zG5(753{%l_8%k>GK%{45on_{sTUMKtSMFOX7U2E z#$rLVG5|~0X`urwUGQ^wH0Y|ulMa}rLj&rh<>=JopLybns(I}t;ETsd zBNVFoWhV9jPaV~TkLaeOtmvi(T(n9&sYB5K#-?kcbJc+$8HekfxgJ z7Vgwv&V})Pcwqdk##(8Z9@Bsfq561OT4pS>nT?5W%teODKcox!TFg;WUZLn%3T;Tt}QTb)52{wLQ%x=0A7~5SLQ! zz+oJr25wPJx0>kjfheWk!xn|Rx;4W4APJmR9ZiqEAmOiZy>*IE+ah4rjDg@+<^E4E zQF*%T`r{zh)!(?0+aB_b3~N6F2&NXec99U$a2sC)j zcO%p%e^r)l42%_$gfZ0nq(Mq%*q^<`J1N+p{}ICniPQ4+?*^0^+4BQE#DT@~Vs6_9 zZ!lFE16-UG*3W`VKSruMzraBH1+7bUE`hIp@X^#9@8TlqE<{Bxnx#WnPG%CfDHUXG z85w9a@6z5P=UJ>|%673*97rujawr2#`{$D%f?I^Cxz_JNrTr_y@L$1KOdKfUEo>ed zD2`Sz8V4CaX1*0z50phLutm}MeSgDeVjIpM>au5IYH;}U{WIl7x<0F^p281}rC zwbpRh9bh&1t%*C}bL2rZJ%;N8qJK+!tloiW_0IRV{#r|f-hqnX_U%XVw+RMG?=FAR z+XBSMOal*tUdD%0>_r#u=6}3>WO7%Yyfd>+4A|sIh_vbfCn7e|G&7prj~TE0W23x?SF<3o>Mt)v>T9teyv0!n#6rGfL~J`Mki8^YQl zvC`1%t9T2kQjtzHW`}*Y*l?7gqC4X!yp|SSIv}(BhA8 z)!bjkwY?Q^)A%b;4B??sH$&*q5%GUV$Jow&QMzqNbFc_%!#m+LvyXB@v%`Puwa6iF zXa5iZo>!*iA~z*&lP<%qkMV%{(h+!l7Kg%+4;^UV!KfC7_aJ5b1%da>9hD`eCANHD z^urEJ8(1dn_#YoRCh+E`0sL5`YUn8a9lrm-rBq1I?2nMn&Yk30nTWPY4W8>BS@&j4!l>*FM&VF~eX3SFP~S`5PcHM~ zFZ(BhD27T2o_sm)krB-Z5zIPP-zt=Is#yP|z;E_>-@*(OuY;}9J+abayC_b1GiIsA&hgkP4{TC}Ynyyt$v*@sabWnFIn0J1O+Ui9Msdip z%+xj(QL}8|F!5}@sN#xf9ZCpv0*KmxVZFYj#5f9nCGSj+WDT7MwT4=?G$rss2)nUe zJ80|t2JlPY!0Z2>w@n_HF)muWlWqb&Z+nDQf{|2vgg+Xc2lotdzE0M`yNF0L@lI`Q zte=CelJW5tZK^*%g=;xy;2Spl*iL^OtIo5>j_2i^K5?6M1Dmfs^(CG5jJ;{@|0sd7 zeMq`ZkM1}2v(HScCHT;chBN=YI^~4ujp^DOW$cO>gi~qd%0B%++qTsnM8L)_45ocM z?)<-wJ5ot#74jtg`R{QpVE-QPuv z@cwhrPHu2V0#U3!=>j~?+TcGn3)e1B_S4k~gZ2K<_V2126N=u8;_%o%(m|**2b8Q# z4Ag0{ec1Fg0Cabk&+7K&9rsKc>nZ;sC^dH{fTV=R`f9=DC1g6-RqRd6JMjLBI_M;q z?C?^XY+v%b#L?F&$=LNPq?y~jMZ21o{g-;%(%V4E4H7OzrA%w|r#a6SEZGpv#@p2V znjo;POP9(d8wZTnQ3s~k$2zBIg?6%b{eNaf^RVw0TpilkDs#Qpmn+zqzczs5{r<06 z-G0_s&~wu|2GTXM;cY(Bzx%|(4481vDO~IjfAqfRUSXt~3vZp^S_Q6^ zPr37-_ls|A|F4@lY+*hCQ#_NA&Tq%!8aXiw+y8qk%iCDl1l!}cu`K^J7IY;a%vmhM zanW5`9Gen7+3BQ?G-*f)+x3+G8+UQl@GI({{8*gKZ$C$9alq;2mMSg){2$H#*oQ=8 zEbI6%jM8-)=+^KKWg!jWr@p~|j&(`-XKY`1>e^WMetMkh@_yJ)xfMC(i667S*qQQu zJ&f^vecHNZeQoXfAx$x`7(8~Xz>zl(YX_`IT zwl!_rwr$(Sw9RREPusTL)3$AU+P1yteZSp(?%ikaf0b3gjEsy7Nk@8ulob(vg`}X1e z3ssMGB4Jo*s-oytwUOQmWQi4AE>3kJg;kKPg$!O9@R_N2mla0* z1}-mc|9}bzDKzUZjd?7H+9D$kBfa^$%-JN?3-Ui?@Ey=CZNrjosNE^7ilL>EH;ZI_ zr;1NLD}M}5Pg50BC!Iv2_`Gn4?l*x`Wf>>A{7%RzQdL2cE|DB7 zBGkMx!*QvK)Cg5(rf-c?VGd51#|LjPpGW%5Q6_ZbJ)eO?#v%ovo@5)JAVSpzjzjJL|CUm`Rj-{}OGkmN&J z{X>j(8RrIic8m^b!svDiA79PEsahAXT8WD(w23 zj#F@pQLwAp8RzpmA@^UV^ChW@vNN-N(;?_ulmnH7(QOC|lPdN;Fl>=htTI$lk2z|i ze_q(yl_|j_CRxJ>Lz}Mok0+tNgUFyXT|m@IvdTM9KcT9}+7!&y)4>kRvx)}wY4yQt z%aCQw>pWnBU8TSzJacyKFAyCjaXqk(PZ)|7kWk!gA2i~{TZJd!sZk3Z-?!VAlZt>Z zlsS!gmyOrpF}8usd~*K2OOFvXF>&NS5Qz;30@l?vby^t|ofnVaJ4y(D(dz=w!t+yu z4pDR4jeFC{{VkvtBG$$9#_8guIT$XX5g;zPJ)J%ZUfE#3jG}~KvmT{EhNiyuTg2L# zsFK#Ygp1b6_9wZa5!5_dy>0Ng=A5YX#AHK@;^RJ}-Do9BWQFY*(PMkuQgV3#@d$)! z^I~=QXkl|uEirC|qdWxi%Q$#N30{ewr<$-*vM*Or3}(Z!eP!8by}airyPl;RD0$1F zEGRzM&s-oi1B8X)PU`%u@nt;ajmj71SW(yEF(r9!G>idNFy;gKJfaLFjY;B~F#oTg zxd>{72n+IE)cKR+`ahIWCw0aWL|>^*#0m_5?NxrUDS_4Mn)l-pJ1!d=fvNxImgRY@ z2v(-{c9oQ?+mkWDnIac59<<9k0WB3pgBpfH#6z8ve2~7mT)?PM|54KAr6%0MGhzNi z1-I&DK30^?;60ua%SH32y66G=VEOOtVj>h+h=w7fD7BUtHX1(&wPZy_vzC~0=%Z>E zsRAvT(h;2UKlA~_#CFk>76K)6miJQ_Rqd`-TDPzjh#jsz7KquL_S^ZEN!N}El$I|0 zsEqQFri$T&vg#_LBuy}{Cj%8~BUY9A$%@Kml`(_%7KOFVN?pWCibB#Fl+^7OHPxu0 z9Mlz#d0l^FiL|S5T(j8E)S$)G*w5154EQPNq*Q5QB42FG8b@et3VzbNQSqP8vAmqf zHEJPNV*Dtj1I&*onU1?;TElTis({?C94x~L8zeG1)ZvPr!2QaHRxhhw?oM+wjDh2 zcHn8McoSqif7(`a7=jJX9dMg8@;v%Kw-pQ{Rt5l3$eS6&4BnbAsooFJp!{%`3NRr~ z!L1So_NfiAHVxK0QOM^$DX{}+8bML2T^q4qDu4QRa)ZXV~)A4bH^@v zF^iGIi6jLfE4Oi95e!1J;Vj0!5Lm00!kg0-5I4MEbqIryZCS+;qNhOZI1S@hqOOEE zqb2ta05?#-JB#r)PpSLzPG^HejO;lr6*uNudrQC&M9NW zIObs_t(pB+tp7v+PgD!GND@-_zOg>GFMe^*Hst#kc)&IMvhn_W8Bz5KC}%m81U$ z*2mMw_kQ>E^jv|#fWQ0m=kx26W81gabFtX=j(|_MueHDL%jMzzN?6B+dn&`*$7wMO zdDn-}`|ZS33(dzv-7(Jd^^U)fw|hGv-Peq}K~A^c!S2<x=aCOB&8b3X+LuKoNw-|nL>C0-xDz8p3TzAm@UaU1g)z&rH2 ze}8*;++JOMUhW9+eI9>a_iXxpTn?xt(%Bo_J$)_gANm{ke6?D6-tXOC3NUp z5a{oawhY>Ky51fQ)w;(joR5^syj<=Okcv0h~nnncNJZ0ND0&!=T_T( z`NjMA##=-xcN-My>X|$ZG4#MEPQ-C<@o#l|KK}fEj|zkRe0g;1NWM@ghTm`55cqO^ zUy-TObHLsF_CS^Pdb}gp2{`$5ukS{#|3zW*B0;#77RtfggBy<=|P6m3honS8b_ojSk2@l^`_CZAm5;$0msDkwv#&N z+b?h;bzZ@#6@ffq&*R7Bw5o-$rsTo`n`xyTORuqJt<7-pyx%*lO+AQ+@z+JI=*#1V zM7%5sh$u{j4v$R`e}yK@Zkw&_vcJ+Qe2f*14VMbWHF|5yz;)u>&~S2>R6!e+nDCar>dD|@ulJ7YL z`O!Cm|7M;=Yc#-H%RqMD1EDy5VqcyY6PA!@j@0R(MITc}HZt)Gck-2Fu^3ZB4wqiY zX|77<+zL1Iq0aA-oSTJtt+nZ+oSqhYyPPXBS=wWdFvke>(U#p4{?VNiZ!-(HY9pqu{V{@` zkeBo+{+4(r;!tnD$77Xxunz4TJK=$^WSous)}0qbo*RL^{L*dEFhW;*cBU)%DRjw( zYo=fvZ38nFLzZelYizyK^ii-z+0yV6C6aX@!A#$uV{i(5^-g8OrlT4Fee2WiEpt_w z%}7tqj~FBeYxJCM1&Fr*#--F9i^;=L5c8|Fk?z-X$S^o6&R@+ ze;}td@?*>a4D{N;ptiPRjkp}c_F{L_Q!b7HNBJ4S&9IB=JIlc+Ra7T^4-8(=gLyZD zD4J>yGNRSZ1pwD|^4p*5q!ZCRoL&Rs{rLm*e)ng>bgkkl{fk5QrV5Hyu*D%gnM?5S zU75H5UdoGcz1%MoLazJEu(VI=ROr`dgLFF@n@|6G)KHAt8hbXE1J$Fyu1&#zA_0y} zi(J?IX7;WC6%Wz0Y|tB%LO#c7nDTCK#d>-GJIv@z+folG(sWi35LKkzTIjGzoQ(Md zQkoFhpUWTEcB4$P;@|Vt{H660=tN%jx=O73eHI$JJq6zt^l&AzH4DJgvTHF>hhnt< z#Pz`T0O?;>l+%w^gvqDNuD`jq1DA}cY)c0SM7cfPeFY+&g&8tjdjOgqss$M}+o=36 z2t9{P3z#D;v8pqIj0^1&BkJ{$yuOf^@~Ql`-iFDxCcmJnCtA?AJt9}FE%F|lE~^}! zK7B6qcprE@r}h>j$@P&s0B(J%zO9#OvTZCNsHz0J+0w#O)vBqvzuMXB`M`B?62t-;! zEAV^+Gmk3seDk%Ab}#%4q!bSoT-LbenVujq%qo_+*)LcYjPq&iY4~a+y*+c;5%ln9 z{;~t|ogQL&wOgbWdbh&3;ReTU^3QRW*}J^rjh!*-(-z`oNA89@U>TRn6Z^8-8E?=q zBUECO8C^G4P`r>$tdl)e0@l%T<#0=kUHXze06%}xCNJd2ytn$^1H5io;`csq1%!PT zh^f4YRIJC0nVccYPM502`xQ>mYe^q6GCjZn&LPu-K#76L-AEU$R>)`Kfoy1mWpPB<~otG4Oe^bz`lox7iRM9n4z? z`^N!g9086y5wPO<-<>m@&4aqJj&7zBv0TsQG-g}KEc~k>&XE?eo}QR?=xeo0i1&AX zS4ZeY9DiImu9>cl z#Uizkkroz#U`K98P<_C~MX@!RmF|wwz%ASv*`J~VT%Y|Z=t0|SJ=VpE`C#o-cilVh z+*0B12HgZKoeN{nb2hk3pR|ovq9E>5H-5oeSj?Y7tWeo_XJ{4bMR0L;vCzAP)4jS+ z=80_nA0zn0aB{wBnQufv8K))ux*Mi`tptSQm7OG_b`?35^$LQl5-CHGuW*v`V`E(W zaFsUwo8yScs^u!5es(c@4pBx)EJ#RAW7sv;{6upY5lZP50ZcU>yL8KpRn^eWumBsR z*y@L*^xoy=73G@ogwu3i=!|EtHV_$Vksw={%LbNmqZv4`wZba-JFwY~h>RxPas{l* z(lN*Q=jx?t?uGDKTmdSiRc$d`>qq+E1lk}(k#qc`W#8~T)pl3JxL+3T=!}pvT##@Q z`xVN4aI-0;wRTdg=?ybL$Cn8chJFMkhXb3VG?FtsC*u6N?|2z&W+_Y-T@L5;Y4L#*nLK1 zg5w$h$I9(Cu`7O>q>Y5sp{SzOwM!V~Xryl4Oq{72XIygQ!;bR+!2Ps-M~3|CVLF{T9vv#K!} zv~%LyF1GGnmpT7jI@)glG`)nowmQhpA`xhg&;?}UD$xV`157k=jJ6YEP=S(YjLi^F z`-!vVLIg}~p3Xa%*^MdJKmyc3PH^$9t~m5?qU+`C{0Sqm^|~%;8?4^FemLJK)_H(m zRFQJ4PGYiHrks&}2xR7JLpV+*8mu0=uOu-atFtUicb{kM59S`BBE~gl0rY>r4R8a{hsEQTL zTl9HPtcUKFE1a?|BW)%CFKH;{egR#%DHapEXB=R(xsm6g)R)%WzrO@8qSM9Fv)oc* z)XX>K3y}W9ysviyXD{czpd(W+ir4cIEm|TOp~h-@E;#Bh-Z5aJ{FTjafS+O5>ZWM8 zwv}2K>^9lVR7}1dl67n-Ny@j3PV0$7cAEMG&^nm^^uD8n6!^h4jgP;-tyyWi2|J3G zo@`WRyAhJrCq1y~ozVgCaL^?kFTrNaSq1`%y4$g7dbMoiT-|BV()Fama*IVpDGPPc zgeUs`ssOJ;hYjwP~@&);8LVuUT1f$I&Rx6Bm-f!7#k zb5I(K)KqhDSp`9aL4IKZ16v9?Gy0+$w7RAlh)i?F+Vb>|W>ow3yR`eTAC0Y!lctv6 z>7&WUT^A!IRXXSkZ$=GVcxPk+Xu{KNZBj5lXQ|4gI>+8#J^&xCQDR-Hla>wA>KkY+ z%x;#pwE-fdetVn33q<9r)gfo9mC@#qZ88TtwzOKex0899oIb2hCL8?1t(E;1dJ&1{ z@{}*R^Gu~gOrs%|d;U%B=iuPeUN$G`ue_5#yc+m+2p?I;%m2p1W&Zv5N@J=)$9#;z zEw_MJJbjRLeGn4{A;ndMX5@}A&$NsgXNOuszCIU+-28%23x;qAdV|n1*XbK*6AD7* z2Bj4GeGbDG>!0jM0g}l_DSqL3((T1FmTN;W;rqs#b2gPrQeo6s`NhVTZc3KzR$S|NbsKeOR3l}U&{7GoPaJV0>N$ud3&bjZ6%GAcz72*in>7->Q;g7R!P}xI>UtludAk;r5s;wa5l8p7FMX zBozDeBVeB*`}_!QV2O?>6&a<oo(Mo+Brcr&P*R_tQ5^BEL_`G|F6Q;GZoF z8OCX=;(Y(}cRGO5x6Uxj z(l8-xHJM`PEPwG@gT+Pk0Kyy z@6w-PzyLC|vkskTkR8R!EZhtS7q4nO-<(54PLgteNS?p8RivUPrwPZb#J37qfm*}y zjGRf&J6W$oxr7NFd*yz- zOuu`&>tC(w@*%68-cH)TkR!?w*)<_k0MblcU`MZ49eu&drcBg1pB-8GXP=s*-X2?f zu)y_8#L<(W-|Bwj^jLZYsj6F{`xTh$&q=WcW@S}Cr3j?KF#^Eiy-pFj6GSk*jIt1~ zd4zd4sMjC=g8S%@tPj=uT1fIzEptyUthBlRyAyY>g}!Fqt~uhgP$tTeI~-Y*0=hIx zJ16+AH-;LiW-vz7KcAggX-frwJ=#-i6ZbN?Z|{(1>_U~NX$EXw4k9n=-N;XzVp=^M zTM?oh&Ne6i_3o1zvJ=i$`+@ty3?4SF8}G*TSV*stEJXE#A@BXwt|$BFoBwP#%jKge zc{xK}abAx}L`7nh@RcDX*S38fn^NPrK(qAbxy6JcdVW^$c!}{9YN^)hps`!+aeWd3 zIbsO_?amX4Hb?aj9JZ*;hG>b6Vq5&;oyn*nmEY$8RNY|e`T{RmU>^U=#cceA8E_H< zEEt}0e2S`9pxaPWzyl#TZ7VSr2_b2oR{xERFV);*l{6>!)#6XU%M9K-jUHI176Ual zdt{Mn22LczkEx{>8%?~nd1520(ICG=`Ac74_5d8(N!5Lkycavb|#_>b)4_lIRZhz8om`Sxu=aTZ}RSGU~fj2*n!ULYi zXfCzEOdEWV69_RmBxqG}bHeUPc$Mw^2dEZKInB;4zC;P7m-0?kacz|9+cc*aaz+aR zkFoQuTlOea-WBE@pQ%_3NTadO;?}ju6c;qt;9@$aVP5M@sx}bP*oE=0jG;`n26{*H#t!kCnc0{5art^uB;f|qrV!u zAR1?z)z0}P0}60d7gi=A@3PWX<f-!TLK>p&>~bY9t5zApzl)4!o_C{#%bAO4 zUWvr$=pgq~Iwu68!Uig~D#1c9UjL(hPzXdsaKL3tBOGbXvkdzFKDRC}qJ3_uBFL!& z5p2Cm`s5^;?-K+=IWBGm#bwe$Zd|qAN>;r!m#A*iUg5e7y6C1(ztzT2ZMhaBC{++1 z8ByK8ez%M!FOBeiIvshYl)<5WyXDFa;2JdC|+^AcdTj`Ar%FvFs2`LTJn8=9zLK}e({>oH=1{Vr+5oNTu2nUyN*!%$^oPLbErwif`p?h3iQXLKm(QIARH-~71$Kl&el z>L^F~sgp%eoTQzg@|A5>n@cu51`H)Qbbg2eZ@etp%ypdKBW=t_H%k2=`-wJusV?fbR5Nr>qFKDA!f!t_|Djmt4<)(QxC z9>+BuZB-Omx{?pbl94~~)CelF=1Bp6f8Se|H_<&GdyFDQ2KIsJj|ajmbP6XFw%0MF z)}e)3o3Ki~1LoGpxUh`>SE>^r3LC(8z}gnS0HXm{{z}Ki*-3N& z*afHwrecqF8_!Tc8KH15TH#jF>fLgPdC1`t)8nxnYES|d4#!LrPy4a8tVZ+n9?<%n z^@a-AXurcnT!&p5?3AHEo*dmJFxA};sHX4EhPj|zGWDzfz#FH>12X=O|Hgn26-NO! z@q`f|+N~;|SIvTU%DhMVeR$efWX_`w0rQW^GUuhVI`w3GA4yOBjV73h9YzJIp~6TC z!um+dX!e^{Zg51Ni)n8EcQwd+uUe)4M4JU{eIN&(Wi_W557V}Nm*CG29nhk5SN>>O zZ{qQ~1n}E6eFxOOYEAVMBq;)39-dh@(W_yxamBV$kqC>KEHeZ%y|TEbQ@H*|6llmP z^|BTo*d}TmE*;$HEkrE@SS71CQnV6GYW!Ne@csKO4RfS9PYT34&m};>da!i1nWRoh z$?4`d?R6^hKhY$Uwy-0lmgsngYEm-Z0VnZjVV~SLo3c|nFy413__(J69mp50f^A zlL0d>y*01nbS5H?EQ#`D+yQ3SIlT^qvu*881sv&dgW%KwnOqF~^jyt9t}+_Y*4oaf z=%B0dr%wROZXNUje>2_j2O2PelV}(`V=RKgZlLNum z%l*RV%0%uu$=k|%Y3cinKj6*Q)@4UW*rmf1eRoff|JyzT!-mC&-}g<+#Bt}V|9>wm z2mh=Av;Zg(Ob7NqziRxqiuG?5z1S>5G~er5!cGmakBo@+wQ2KtepagtKi z7W;2sV2e_>sCb&etIySDq(*=`?f)nu2)2NH_HfO;{Tse&23Bxk7pHxHiiZ z@t8N5D>pShsWZxar!mf;RlQ}ftJo};$n$C=kR_m8w{zw=?laz~_Cm;4`03-D4Hb=v9jSh> zYhsiwK|~u$DPf$eAj*CY@mXs{RGU|e5^W~y+52HPW;kW2MJI{R_0h4~P4J}Qm@l%k zi;jbFF+eS6>3ft`8b{yLLx=ya2}JAVUq2uLTEO`~Yr=nuMQ_MBY>1Gh?X z_8zYLQ3M_2rj$qdrpba~2OBRMXu9{XqmlF@we7i%KXWi4=7dD%jgtjOuj{par@H&k zXWb>jN^)CSrJ8r^CHuy}r|aV}0fFD-k~LQ5K*^eG*i-vk>5mEy4gp<0x6jhue7ljQ zQ=qWjb}y`v{X;LlP5wXc=B+0&m%A<#dMnqNw=VTP28biwcI?MIm+m$?D~??b&0%M! zP^o;*(N|aY-W%Q5_5#-%11~Od-CJz?0~g0j4~qP5TcxM{44LN@IR!S!pFeAEb$ZtSxKAB#y?8&rICpgUV=4GI zGj1I=Z@?e>YrJ1Rv{b3TADzE`k6fDjwAN&G+^`qf+LoykY#4pi{g`Rl@xvds|9(~( z=Os8lrF?p&d^)5YUViA>y6@WZf0oB2T8b&F^Yh|89{K9V&djXr_SStoe)wrya=fYA zzOeJ|)_!;Ca{1c4a9d)rJ0Vv#u=Fz-HBFT7tTk}>*pRo z8<{7IEY@vSqGLAlqdaf54p4awo1mHT_@~;Iv1`L=cAriRYu%;5>B#ehH zVLwPbB@pv)1jy|p2sDPIC~zE;KAL1B2mlxIY$11xMabMwm9>AF!FguaX?ElLo%sX7H3!rtpP=Dtq0!1U z4x*C(;P3tmzVpA}8HNCOgs4+2CZSQ&M+~w-7fo46#s~crK@-`>jI^jz403#g#M6M# zc5oP}maVMGAQ=zY5=fj16h_jmCue3%b!a~F1yFcbXjVFnC{t(VePQT-o}ZyuYSZr& z`pKJXQU;DaNQ$U|gdA)P@BpL}Nj9&TEJ)-DaS!O8a=}Z!KmxA@jh0}+s6eahC0il^ zAU?TXoSWEI=XS!BjvrSp!+4&WquKkACOZ=sAuNe(gQYYYJp z35>N+w&p0D=43BG%0k_Yp!IRsb!mYB0WDroXcmBBurL}I#A)jy4hoz9r-$;nSRkMW zZ0dsLj4DG$@8D< z{(of5l_TlQl-hhlDr+u`Q-Lw6_KGhzDP9mfB^w~J6|^Rk)XmbUZy7izJ1Vjiw5FJJ zWJyn}L>g@lf`5iGImpY@5sA@^4vgpu{ovV#w^GEF0c)}qJSW>Lq6d;+u8%f&RRCy> z0@fCuWvL@+)hLN61KXoaIxj9G*v!~C=iio0o>2*J*ngVoLI7?fm6}DNZOzRmMTyvo zMh6FZO_*6k0GzWu$;wW$V6d3vY34opT5O-83N7U`o7GO{q( z4g;h~)D%zhe;!a4vHO@Ar)kxth5XY)JVqFf1s0?6p9NpPQ42RR?=!{#h*?=CeFz0$ zs{l~wL0cgb>V41@L8YZo|(O@tUl{wKMt>$DuK#Q!owV{~)4;?Vg|9hPg zgCRWGP#2Q@?_vqdsxa7@SWE_&7rDu=1ZR}!ua4`p*!Qa{ca+))%d41h=hpalK}nJ| z{c0zF$m$jhjbils=V{sbH>wNYJMnUTn}fgI+xw-F=H+_F0@Hca0#CL78~gg}aXn9t ze@6Gw(r=TNZ@1HYJ9aw(J?_uujvCw^y|nJT$Ght8On>129#dVw9J&cWfq){Rfq)Q! zfPg$~8QdM5Y>XWoZ0J4gY=5UXC@cu#jpV*l)BDn&|F9c@`X!5|$107sdpozSS=?o_ zKHbNHs_K!KcL%fw`kQ4~BIq`d^nNEZKe`LHmTrJx(>8_G>U#a!;`yB4eWg5|ZATGY>*_0z)p^yogk^Y)h^khMkeIQvGEjXH6DY1U_Sz2_ZIzd6N<}i5dCt9c_t8S(q zrrU%C?=%t7y(O�oxYgd`T%z%?QSQ5E6V)$71S${ypo#9A)=zE0c3Y22p0g6unDu z^hp~rpvDBtQ|NK7@%PG27;cjXq zp+1f0zcAR@7N=`cbt&w}5oBt!p0|vzq*-dL(I@6rJfLSApE)>5a?n?Hcuw4(yaoJn zzFw)ns~_a`)C0|=5^Fl4&JpCf7#BA9Ww~+3rv3b#Db~FF?yk@!QQ}Bb6Del6$0iV^ z&GAf!!%W{gdK6+2lGA{VhYwfnFr1EvbY6hru~Fb%a_`hY`NFmGNMIPrd(#SyCW)|M z4XfrDmyu+#4aWB!2A~u1?Odl&e8dB`sn&2e=SC|SqDS=Y?2uTG$ z1c{PD>7WYy%3p)|s#i-quUNF6)pLjM%AP|E?b)s8nMbFqMd7}lRIjc6qTCyHF@Q!^$M1k{Ax{91fcIt6QER^9^Sl93MK_Z}9(aOJE79krZS=Kn8Un zKxqH5C1*<`Co@wO7bh!w3+I0f>Qe9A9d#t(d)gp>Ag3d)F`{Zt2WgdU4>=b$^q17r zt=Y6D&IU{&wi!Oe6$R)ncJDL<2yj0Q!`0MKT=2vI`%Ta0(&%032dvH(UxJxy>_^-| z^x^mw+mnleyQ`Y(Qf<3;g}}#iq8J@(O@?~kth-%ue|$BO+6?w<5P!SP`N+tVf7~`s zgRR&MO-6oYM{|yHb@g|*_e5BVS=e8-t@S3?$z~azx3{XzG;P(YIwfUSN^GXd0^@<) zBgeIAIwuD`6qfX)yYBH+I%)gD~=o)G=VeJOgS=Z>% zmM&u+@iky(L`p!a*;CO1%=X+hBVMXsZ4|a7EVjmUU@^(JqK;{vOEPR!$tfzYbJ}VR zCTINi_0!5Xvngk`GI~Be_cLLSYo>eB-7G9B78a(iWp))GPbbbEbjpZ)($8-;NuJL{ zh?qn-d8!f;KI2oAuawoF{(t6%ABw{mseoGf2EYY6LX z@*s}ans+8-GGN&AheZlTzJDrQO`M)hIBcc%OwY)x?{vE|@Wa~06W2>JqsR~Mi=BUy zPX{-%HVb^Oq=#hoM6N$er#J{CMu_e0igKT_F<^c7YdBOg% zm~0?koHlO1`(C+O%0PWN8+;%6dG}?fRc?CWL=N|-%SO)2jgLD_dnb7f-ToAnj(-*b zUMQ!1p~C@mj&!b#Oe?8A+sC#&bFir1MRCpOvOvwCN|+>77) zMj$E6O>&Nv1kw)iq}_U2VaFv`7I^FGascG0&%tSIE*W=N`IEACTl@i|wfyUp?P9GOQxH4a2MBVbiXI;IVJo zrIfK*SVs6ras90?tNbs+6+f}Y#phmmI$sKl6OxcQ#&lnbgCKiQ(7rI17vy#H0tdT& z__z_O;y4;EWBg?*D1EVu8*gvxA5LHr`T-$l?X~N;SUwa}J1_B=LdbOUnF$IYX z37^MT7P|%9bA5Guy?!AxeIHEs50Zg2bbt0aAY5$k^zc4Lpv(6SHo&Cm8p({A>AZ{WH;_hU0ZPg3QqS>)wydeSzk&lJ;fUbMOskd`$oj^v zyJ30J%5H@=XGoWoEGGqWDVLPhALV-n>E&8AGDpeBrAmn=ABdNXRL5eW&6Lx4>h^vF zPRoD-(xRkYKbtQ_4(JWCnzF?NwYW;uFoGmOZYz)@WT1u$phCm6@aOoisASxk-a5wny|u#Q z*v&TFd&JVM(+&N12T59W-#1UsE9}Mc54VBrZpb1~PuD0>%lBK^!y{0PAtdgdE#O>` z0DHeDDZMn)jCr?a(Lms5i{54q*bIc^c(I=sr|0)QQz!)#J+w2Etp?lL!Xpe87~Q_A zey*Nl9e-a-{i--7>iKO>a)SuN+eLNEc)Z_Xbk?|m$hqQbhtM*i! zF2BDH!-~T6>sD(MdI)G*)I5CV`bnOXNjA>znU3VK>vZ_0qyGCv_$(6Bz7x~|dL4>w zm+lD(&*m{T>~O`P?(e+sJ4ev4(9H@#VjkpR;emiFZAV^g=mis-3nA<`kT4F*pO1H< zWjjk&i9We5!*_3@pR9Sh0EupSTWobWZ1Fr(uNcs7wnHT%bAqmpW&0}my5}Y*6!0)6 z&>-alpNEV8^7nacxvM;8OgmH9X*>3>^!aSRyCKe}!az8L#fTse$PmZrEUj{f7(*}} z)BV15JiH&%9@taQ1ZM&Zp!uUwk3OY%SD^sa!edYilvek41i!En67dd%RpBrr+$#t1 zXkBpk9Jb`HV2D$1C>L2wBn)doSr^0!;{sA3p9Y2!v59BH^;6h8y@%zpyG>~Eu4Cce z-5&noN9fAvL5Dl$tjyt8At)^$OJDk$SS>Y-)}Aj$ELekY253`CnFLpl24A>$dSB-=_;`3p zjh}0=Ph*gf2=I9Os~pehOn5B^wE zLoPLG4F|*Iaz43=97i?VBSmkXeaAkN2Z2k9M{mmp~R zDtT_g>ThiY4|P@fadj!0)dsbfC~C|wA}BnyNh34WWarQl;3D9Ox|1lOq@Y8N^#-&f zoL?_~ybm)G=QrO(NW85@js+m9)te3uC`C-}9d9QVc}ranGUVDK7 zvoRm#mg}B zy8ogSb74H`BwQ5AP_sqi(m~Ocb!d7BRwOd%ew5rDcstu?Z(LQ;pjPlJo58n+ zJWtWM%bM7POUHv-Sa_SfRf*B9l-DJaP50Aldqg*cZ&<$V(Wqd!)&SmJ+9PUk5CKJw z_l;%TWtv6c7E9xIFOf$tT+59$7QaYn-QtF_$;?ehx*ZSkxi>)};c5O^2_G35qdyv7 z&90|vG^iTF5Xt^C*fkt{RUZ!nb6#(N_`AfCL0ogB_Ay_*nt+|^;ppX{nuLHyZEumq zCat3GFV@;8+@-6m0m4&kY%{GuQI@Tq{;W=l1-jfmaX4Q^4lWJ#(XFDTCk!q%gM_Wf z(&34HRr@3oP6doaV+$km-tYY~OO><~)cKHNNVIUqDTRWMx4Mt9Zdy&cblOy>(RR+N zqp_4PeYKWF(2|5>9ZuF81E=p^iXh*1qU3>nNLjUdHRm0Rm;$!cAF6crv!Aa%H77sS z#`b-TI$!H|&bZ*1y$?*}ZTgfo$p-coJ$p(tO+K69-Ed6VL@{Q^W`{s$oEmHLxC+jK z)WHY#C_;Tl=YpM3p>0n@`*8Npv_+b=Mk{sqwL~BbTT(0i8wP)*kK$z6hQj!~fDLCh zI$Cd!m4~T@RQe8b_Oaby@AL|ppxHLMtscNg{|ePQ@N`~8`Ew^F#23%@5(sY_JUr!%D*TdVyf)G*#bbH$U6`?HGCW6AI?7a(R~ac~ z5g0%d$v+Ms#_I5c7Vj7)rXsRAHP=WhFHz+pX!v&zJL-3%VQfT_@hJAupJ_aB8)XnaC%CUCkgVCUo=3bKZNLSry@8#~cx;#*n zcq`GVQxqB!8lFFTxlY&7XcEyT?=2VTZyc9uH&z=B$#{RMeS=jb+P9TZUj*xm#+n37 zcP=L_NfTfw3O7>*)v2&$M%=EF+~y6Morno|Q=+dA{G9H#I~t$9_wWbd@26M=PH|Zx ztqr!79u$6j23I_r!B5rEik%ajOnCNChw#S*q*FwzPVvieiTMdDwYpe)bAN5eASGo5 z!0;R<#!H&kb{5anOTsfc`PxeGVx$oJY8$oQDpVR)5+n&iAt7k{x**pmINF+^`N-{r z7cTsbbBUKE%R?+yotLM3nx)^G^MJsa3T&z)G;EQpD{4r()_t$XGLr2!T)RtaBg`M@G0|_tbB{+0{3Z^p&LLo z-{fa3H9rH5I#_!o_E{bM`+;*XJV6B;mFk+9cuG&pc_0)xKVwZ}K5H&$o5Rf->n2=R zSvF5$$@D3$LQD1R;Nt||C<%tFV1$~FN=ExS@kdAAoF|Ex6oIN_CqiwLr-;TOa{32e ziLew~xIQ_P`$$K)#3-YzL@m59J_Lfh8SAa;A0yT#`(@TD^vWi*Sfu;%998F->Tx!l zmG#``>%Cwtm4QJKk0^ZgPfs`5`ghx<2D&#DbCUv-JdF0;eAx?24KL|Lrq4?AYtzJ2 z1BuJ*sTfX-so-MM1f8X_AF${~O;slc>``U+)ywn{!la8Zzthev5Wr4Lg+19I9jl6u zhDI9SNB&o3XB`$**ZqBJ2odQH>5vWy>24{hLFu8pr9oooMv#F$=65@`gaL5U#+ z1b+vgxSyM6-uL~zGuO3e|8e$b&R*xc*4gJ=d+i+r)o_P_CTzk3OeD`aPbzg3ktbE0 z&T&Wxdy@w%u0hpebW!78v=oZJBFGyN>Z{pt4hryJkkj{*7QInZiHJMVn5P~J@_JDY z64uYx<2$*`DILK9y%Ux~iRfL9WB3fqa;{7{X;9BLo}_%+*1s-#Rk;52V0rx4j8W>= z=D^z?IlLW{((}1Fj&@^tX8fn-H?^U7s8#M&+ql)qAKr0l@2FJYL}8l_$8j+15L6`k@VL+|7=O`1VI-K8!+HD|SbHlSB7yHr@nEiM#`+ z?|{lw(4Mj4Yv$JRq}>0Ya~52tf}+j329vObNTl#65n=GRYBCn)K7!DWE2T$*v{-$5 zo-rAF(&E~ZnkBvJol>9Mba6jTzZVeT>^ z^7b`ePN+oE59v4My}Z(2cgt?|IvQOV1N&Ac%lenRx6KrNuOPTO^8Imw7Vk{fTgG$@ z%lWrqJmH=cw#AHD4&f9#ewP7{3->6rC+gqtil~+#qS_V+Q^6+qyX;;^)sr!Zo5rN> z;y6ElYClHezos`?*)w$*o-@4ry#KUnk083HOR1t&Ic~7P(#3b``B(po zcJL)P3x%2sT3mj*xnNHY6H+{WRJa*lxW6e&=Yj?BzY*<6t-wf*`ZMEOC@ zCqXpMgmrq=7uavC5=L;XJ_sQ*=0D+3*!7fRnO#kca}lzSGu`S7dnOw2Y{dBjQ_s`5 zv1t+GrMl>(AcMo}wp3kvhJMJ5F{H?bbY5_9vM^x2Ic&5z>`;$+NTw^YGafxrppr+v zLCp2pDW_QP8YpGBx=TsP_YzA-onU?UaC-$m+3OX4;o>29t;TGCO#?J|+IjT(ho#)C zvK=BS5`jaUWxO9>ALq-cVq3noRBTDNWy9yFFRlmpTeDUC03MB3u!*J(J8ns?^U0v{ zM`4-jZx@fN6(l;`mBt|Ct}YkM`8w*pUWHdqGpnveKSCAgl#ycm*hUZ2FMS^oRP^e& zVi!flHPM@$<|;L^QvLzQ-OtGdsXeQSmN_M$X+QhR=0W zS5)USiQ?GgW*YRut|G!ePh;=#gi_3cDNYQhMB%Rcm8%fi50Wo)F|Bcs|sl-1m;gE?d6)w6IEj--nKV-SJP z#U`68FS{}>(_uQ4mWW|_zI*NKB7@1>y^<791{D)T5INo`TE0@#jcw2Cvq>v=8!D!( zAk0a?vpvA{@CBdb@j#QGg-UB)DXc#$S<_OG5^$b8N;91x&j6-M$~2k%FlBrqrV&~9 zM$b)_Jf(j}^l`OA8ijsLRmNE50Qb^nVh=-$I`JwXcjAc#q=SJ$AJt&j67%P$EG)T?o{#&A zS9l)=U;N4_T|s$Q;x&(vm@=ImbvhtC%cx0ilE~oyty3J`h@;)mdV7)QkptyneVD&3 zhL$dAP&rMb9ic!uGrb+4kgALaG!LUbk;_Gnl=hEYour@25V{@ z2^fT2<|H;pyIq0kRFyD6I7v_%Bajtluf1>U_CT|7$U&QeAW&O8ZQiZj`4^wb+U5ZKvp>HU}U~B1EPiXgj;{jNtm&P9&EtDyXqIA~VyGFc`efAt1L9FoI z=O`U8-VX6e0^9g1Pe3Yx&^HZc8qQY?=%UI+b=aLxni@J=UVf1BzzlKf#1>=2KtbtD z55}{cMa#$V^%vc#sg#K?%Mkrt3W5cfF4TQx5FbYb`R}TqiKTxfAYFRXS3CZ@oz7_k$?0AFK(0LTYqO9h6 z4zsE6jb=?2vr46oX38iMoWn4)HxSk$N<7rz581@TvnL_AZG(#OIcZrJNZPP6jauzx z9^_!b$!G3C;{&!QQ9YYTE|htdGQU_%p)bgu18?8Xi#c0a`}f7fMmgTlLSIvue9~WA zh)J{)>diCFrVVJ9=OE1&V5nJ02*_#C^LL+Wa~aLLY`Q$Gy1I6C878+)*3LhVeF4T& zO=%Ht8CxpPDoQ!6Ti;!e(D#_y^5Dng6zCb*!ffc!XMlwedGNM)1{BV?SlP#@T(@01 z*x1K-5!>8cn{4#e5Vj9ERMEZWn19p~=~Ktav4k@!n;Ypb^3e!Uz6e33gBDHIz~XoL zSLKBO^S~&{m{zj9sI^}k3hUO;%oFY7w-kyZquOsblR_{}f+^Y9)svp zHU*oiWkR1xIDHnF&@hsGxs$} zI#Mp8$hBu%<$Hv=JiGh(u@@gV6A?qHrM##p(Ud*5FrAigf1{2vy~3%aPd=FFz$&#m z_o``sb&r{lfu}~gOWB*38~@eor6eL$PGHo4%1P2{d>Gt?d0RQD&asX8il^zqTUENy zugb688cw5hpol4*jDUeGO>kd>OQs74bX)PB8)GNVv-=Pla9;zdD1amnMHvhf3M3?X z#?WAzdtuDCmCwWe4ox@^EkC^=8l}z?Z0TgB9;-L^C ziyhxHX>*3L$^mRU|hw{%#<%b-eE4;nJ3@3zc(}9rlE&f~uS@ zIE*jDy&mg*mTD!VHla3@ZH@FI#D2aSq%7ie(eIN#jm{b|$iNlfDS=cN8YFWV5)_1j zibtZb;vUs?wi1EveTbllDe46b<)|n`1-+e~-+SHz2GdIGSp01IGEvv5N3U&w`6DfD z)*`^Lig0W;LBlq(jOUe1QRTTrz4limBeA>&@z&HfrHGHF6<_zGV&heUd}%DVqD>=I zg3CQ0l`SBcJ>x#!g|1rFFZPPbKNA*^D@IZskiFTp5?BwnV1CK}Ivp18ep}fB$*hM=!*+Q1`DI(HtCA zgGDB+n=DP{devZx{SzjdjG-l(eW}IJMoj0KJ{;6lj*l;(EpME3!^P5}?k598{yag` zWa8-(xs=bZrE@9StZBv{#B+@&(Yt_?3Ryw#!VRu)JUcS4PVzvnd2M!fgs?U> z0=mJiILofZsX$dAf$HK`&>FsINS0O1dnlS%4IH(WC`i@YD`5)Etw4^rbsW4~#{sUZ zG4(`B4U=Xv(!Q?;-W9RAs0!X4f=ejbL^Z1N8dlYQ7AGK4)!w-N!%? zD8TLeinSK5Ag7E?~8fCAT*5*cq^wpS)g8STL-g_QLNEGhtl%7=+{dN*R;pR?#>V zG=frx`v8O8X6H$|CH~kzhuFczpr3WBz$*D2rD0bf6Vr`V@vGr_^04$qyQqYZsIQ5D z!~i^0IXn_YWBQ4=3P{)jC)JX&$*m@75zM3-N5BDj>p zS*aka)|;6I7;lHnr=&gw_)`BRzUiR+aJ6MX4?X1*| z+%(!Kc9*ZxwR7I7QWh#rtjZ>lqfOe5Fe&gcBOx$#`~_6ZNz%nVNWMNjmE62vblEAI z9>|S%N(znO5$&oxQd64ZcSyHB*tmAN%48Jc9%H671-5Ir7^@x zy8(*S|Mab1d?K%IBsD!0q%sD2l}P9Wk{l*8do&Re)BR{7E^w!niJxvm5~cgM#^#uk z1wew`Dj!P#NB>v?T=Z4#TRF}M=t~#0r$sf@;(CG-)1fUu|Di2_nQ!9=j%OL~z7j1E zUkI%%X!mykSJ=M?xPs?x$eDU8GS8)LTTDP0(?s=#WBQTRQ+cc)B){mHIJQ2si4pVJ z=IBTP;?7+jaE$$jkH}D+TQ|@F+(4!Ts)JkR@ooBbc2h8@1IW&g0@gn#R=K^jgZ!CMm8HF4v&ejCJ>wD*1}RRO{T|OTNsaZqFoO$$c;)^FUzVpP-y4)u^R8EHxfOX*)C(PYPN<@T;V()hcEHV(IuPu{U>)pZb7*$S!e762Qdsnb);ONAtFCUO zftu^3Du`R)!&^KLr9x0u?(QS5n4$u%QN=^*=C-e(V6KSC#%G_&l1p^vQI=!E`1l>S zWdsWdSE*xMdsF0ywrQ{(6J*8_VJd>de6D?o=zUjR6hg1YtUh8erg-!jL%Dugc5Uc2 z58?Te1Z*MZ@X`G;ZRFiN*R53CCPu!BK0Y=KqKP3lWuH1ixJR~Owk$z`vBf{`wTRmM zxd1z#^T3lUvZfoNAzI->wBNBL{)1AL-+@biJeCM!>8sBf;Tpv-r@C2@8zQfeHtDy) zP@2ee_Q05Zu>IbagCNdl@Z)Ux6$!d4#_4?D zdl~ATD*6tlXtNt&IYe->)~FH1Q)cj^%aa=tR>C!1^25Ix*y-=PrIAM8SbwIlwk-bUG>E!(Q5v>lhho9% zGNv3H&?1ml!7~c(V^1QEeJB{^AoQ}sRuNV0TAyHr*-8=Y5O-`5OngDiuE$Dc@#Ue1 zriTpsk(y^fch;EkMRvj6)sjgr<(^f75D+xU5fJeIAwD-Nh&_b!Ps#mfQlGwR+=2{A z`!TCrdqXU~@JRWaGXA-O(7pPSueI;`BSfo;NqtJw2OV6cfql|2G;oON9Vl?r-={Pt z8`Y?z$nF}goXRJ{EUHK9qY&zZ=(AzR{sBKJw|48$xTWP~6o!)OH=dX4L+Qn^t?=s0 zgkRkhriK=DD+ZFPDJ;CfvwpE~x$)S>Q#lzpk0sU>i8JLE`bm~ ziSS%p<`4(|HhN7x$OXw?gc>_JlqB_@Lv0!Ors_vo$yp}!Jz*``3-RB{ZVZxK2|A^y6W~+4C+N>dL~9^ z28yj`B)buZJ)}DR1f;6HaLL@Oj*B@^vL)W?eM8XZeY3v)ttKg4JV(Ua?dTAge2JAA z^5om_f!Ocw8EWG>Y8`$TKb3h6cYiMX|2mm50+I$&9{9qhl9qEsU&2l;1AKqaA}kYm zz3V~aW5oMP+rJ&yZ=4~CKpJ_hai3O&l|T@w0C84J1LS2JnT|0-qrLT&a2!FU^Lx@2 z;R|d!ZjCPpi!rPBoZ?)kzja%UIxs2=SSE&8d^62kd%z{eGq|69&)zXnict%H*g@uz z%|0)b|AC?u!O%X8n+CPoCt2rLv%Fsy!$TIgxV#5NaKIg z+hI9(4h}gj7s_N+lpc(8@h~ZPSkexazu4*ERvoqJiqIia}etFe?)qc<(<>#r*Ua??Lj5R&IwVG+v>Y_lU(f42~IU$3`aRe{{rDhBE!>-$YQomg5^wR+gx_)U19y1pUjcTK8t#1b&+tj6%(py&!TkZ zB2-<_EY+WzqI=G^zE*1=W1JW~EC}-Fv6V;*Ra`jIwjQ#Lx~T2itaYu(#KFMh%%HVb@9URErdzws*yJXMIZILyD13*N!e9#N9YmuJMN=JNp?(q$B%#d55ne- zw=6N3e7!!Ns)IblebaLB58oNz$030T&DS@oprXW zg;22KBgEYZB?STv=b9h(Hn_hB_zPb8EEcF5urS4bPrvr?*f4z=uX=(xjZvnQ%)k;U z!95cCLHVRgzc-HACHv7+^0HhRO2OLF{d*SZlSE6U{Kjnv#GrVWkaWDgHUSi~{xjTeLUn)HpT2y)nBLr(4`^52d68;AO|9x&(=V4ne;|1_$JlhYrY>Wn$!NfNYQ z>8hM9aj56k>(|qKnU{-^u@Y6YdrUS_{j?H4~X5!{+ZY`mI7d z6!5K*K#3hMb%bK6^`gCDxjp@*V_9KVV$>sQ`VBUSj3pf^l3b(ooX(KmSjDGBzW8Tp zLbDt9HRT8ks?3)wtAl;02|y8UD2!Ti8Z3?zd<-m77{mH)pH7s5qVROQAJwZEGOHsE zXC^CmCX+_VBr^VBHhv(;yF{Hk5!I-tF^TFfxU-u?e(|ArZBA$Y__uj0)&1Nbd zx}WXd+#FB0=+c>#Sz`U(2!{oZb{!l~CUK_;Q;d)P@q+Tnn-JnMj<$PUuff z?>%+7H~Nbp)9ObsK^U(9h)E2@#2q^+n=2T3b*j}imxfodC?mNzBaN z{1OO4&!M#xVwg6aG`7{!(MJ)VauU)hVU5JVs702W#4JSWP1k5>67nhk0!QlP0@~NT zvUP&~D(*WjYRH~aZ^fFq%++`~t|{f4eLpEz@EE-dp9B{Z!w%V^+@Twy-XDpxYVS zXF+#~(QLMCng064@(rfCG9nTI@@;MhAZrA1gp|Dnqgo(zyaEP>w`0S9eFB4SYX;Je z4sH+!H=}1Sogl6+Ze@QLaFnAszXKeq47dvBzgh#ofk7Jp*2o27=gP@(Ti$KhY7}r2 zbsMzs5D>`z*6cC5{SZor(6P0eQ+qe@F)&! zomc_Dc)g7DF69QW6&wm5JcMP)Z|p<_MnQfh!xgF|`z|3@idhfDdh?aE#2{-4h! zUkWbe&&uondrF2bT*{wC75?{>Bs;j2KMDE&Hwti>{{0CIiUK?bj&-@Y{$J+(J0>I* zJ_fFO|I@yI$M}`N$H0~Egs1MFCEx_uKq}?mF&>@pF>v+Ewc#R32w=tm4gVcO&;uU> zSHSpA*DPnZ0RnKzzhjJh;bY(`ctf*qCmFynBHzWt^ux!%m2gJ`xs5Gc+vp9#$H3Jv zaysUg9bC-iFnkPL5pxWvnUw>bI{dFrWsSnez*RBPo4s=QoeCa@i{XK*VlipOXdAel zqMwA1fve(0=xhc2v+13JkAbUVBB+ixe2nERd<_nY3kP4B*$`>Uyh&!47u-_zX%+9Im6ZG literal 0 HcmV?d00001 diff --git a/applications/ColossalChat/mstt_advisor_20250519174404.html b/applications/ColossalChat/mstt_advisor_20250519174404.html new file mode 100644 index 000000000000..028ccc63e770 --- /dev/null +++ b/applications/ColossalChat/mstt_advisor_20250519174404.html @@ -0,0 +1,7585 @@ + + + + + + + + +
+

Performance Optimization Suggestions

+ +
+ Optimization Priority: +
+ High +
+ Medium +
+ Low +
+ + + +
+

overall

+
+ +
+

Environment Variable Issues

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EnvironmentValueDescriptionSuggestion
ACLNN_CACHE_LIMIT缓存的aclnn算子的数量。在alcnn和host耗时过长时，可以设置一个较大的数字，例如'export ACLNN_CACHE_LIMIT=100000'。
HOST_CACHE_CAPACITY启用动态shape缓存。
默认值为0，表示数据缓存已禁用。
如果设置为非零正整数，例如10，系统将缓存最近频繁出现的10个输入形状的执行数据。
当缓存的形状再次出现时，host执行性能将得到提高，但host内存使用量会增加。
具体的增加与HOST_CACHE_CAPACITY的值和模型的大小成正比。		设置一个非零数字，例如'export HOST_CACHE_CAPACITY=20'
ASCEND_ENHANCE_ENABLE启用hccl ffts+模式。0-禁用，1-启用。建议通过执行命令'export ASCEND_ENHANCE_enable=1'启用hccl ffts+模式。
PYTORCH_NPU_ALLOC_CONF控制缓存分配器的行为。
可选参数为max_split_size_mb、garbage_collection_threshold和expandable_segments。
1.max_split_size_mb：v —— 大于v的内存块不会被分割。
2.garbage_collection_threshold：t —— 设置阈值后，如果NPU内存使用量超过阈值，缓存分配器将开始回收内存块。t的取值范围为（0.0，1.0）。
3.expandable_segments:True/False —— 默认值为False。如果为True，则此设置指示缓存分配器创建特定的内存块，这些内存块可以在以后扩展，以更好地处理频繁更改的内存使用情况。		export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
ASCEND_LAUNCH_BLOCKING是否在操作执行期间启用同步模式。
当设置为1时，强制算子同步运行，从而更容易调试和跟踪代码中的问题。
如果设置为0，则任务将以异步模式执行。		export ASCEND_LAUNCH_BLOCKING=1
+
+
+ +
+

slow rank

+
+
+ + +
Description
+
+ 集群中的通信有问题， +因为通信时间的最大差距已经达到 +103060.644ms。 +集群中的空闲有问题， +因为空闲时间的最大差距已经达到 +99304.586ms。 + +
+ + + + + +
details
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
steprank_idcompute(us)communication(us)free(us)
001556714.0687232581.3436829097.07
011560276.4578997510.1643698754.88
021558312.840868325.8279587869.88
031556942.2298397199.2425134636.69
041761254.8645969395.177693774.04
051765175.9256016250.6465924566.11
061762990.273041651.13117837748.23
071763501.6752671041.6269509934.54
081561748.0344449697.1176268659.57
091557930.15103663859.4920203496.84
0101558704.9887122155.735402682.33
0111557350.52105052622.0118533162.23
0121763293.7466816420.2348098511.97
0131760975.61991978.28111975738.82
0141759204.744457564.9268792652.51
0151762214.4758791678.6456870330.29
+ +
+ + +
+ +
+
+ +
+

slow link

+
+
+ + +
Description
+
+ RDMA bandwidth(GB/s)： + 平均值是 23.997， + 但最大值是 24.017GB/s ， + 最小值是 23.983GB/s。 + 差距为 0.034GB/s。 +SDMA bandwidth(GB/s)： + 平均值是 17.935， + 但最大值是 18.663GB/s ， + 最小值是 17.174GB/s。 + 差距为 1.49GB/s。 + +
+ + + + + +
details
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
steprank_idRDMA bandwidth(GB/s)RDMA size(mb)RDMA time(ms)SDMA bandwidth(GB/s)SDMA size(mb)SDMA time(ms)
0023.997616.22317.4618.4170229.433813.77
0124.027616.22317.1217.5270230.234008.1
0223.987616.22317.5518.5970230.233777.48
0324.017616.22317.2118.6670230.233763.05
0424.07616.24317.3717.1770229.434089.41
0524.017616.24317.2417.270231.314083.65
0624.07616.24317.2917.3870231.314041.88
0724.017616.24317.2218.4370231.313811.14
0823.997616.22317.4818.3970229.433819.49
0924.07616.22317.3517.670230.233990.1
01023.997616.22317.4518.6470230.233768.14
01123.997616.22317.4418.5770230.233782.16
01224.07616.24317.3417.2470229.434074.61
01323.987616.24317.5717.4170231.314035.02
01423.987616.24317.5617.2470231.314074.82
01523.997616.24317.518.5270231.313792.47
+ +
+ + +
+ +
+
+ +
+
+ + + +
+

comparison

+
+ + +
+

Kernel compare of Rank4 Step0 and Rank0 Step0

+
+ Issue: Kernel compare of Rank4 Step0 and Rank0 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
+ +
+
+ + + +
+

Kernel compare of Rank5 Step0 and Rank1 Step0

+
+ Issue: Kernel compare of Rank5 Step0 and Rank1 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
+ +
+
+ + + +
+

Kernel compare of Rank6 Step0 and Rank2 Step0

+
+ Issue: Kernel compare of Rank6 Step0 and Rank2 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
+ +
+
+ + + +
+

Kernel compare of Rank7 Step0 and Rank3 Step0

+
+ Issue: Kernel compare of Rank7 Step0 and Rank3 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
+ +
+
+ + + +
+

Kernel compare of Rank12 Step0 and Rank8 Step0

+
+ Issue: Kernel compare of Rank12 Step0 and Rank8 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
+ +
+
+ + + +
+

Kernel compare of Rank13 Step0 and Rank9 Step0

+
+ Issue: Kernel compare of Rank13 Step0 and Rank9 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
+ +
+
+ + + +
+

Kernel compare of Rank14 Step0 and Rank10 Step0

+
+ Issue: Kernel compare of Rank14 Step0 and Rank10 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
+ +
+
+ + + +
+

Kernel compare of Rank15 Step0 and Rank11 Step0

+
+ Issue: Kernel compare of Rank15 Step0 and Rank11 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
+ +
+
+ + + +
+

Api compare of Rank6 Step0 and Rank11 Step0

+
+ Issue: Api compare of Rank6 Step0 and Rank11 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Order Id api name Total Duration(ms) Self Time(ms) Avg Duration(ms) Calls Benchmark Total Duration(ms) Benchmark Self Time(ms) Benchmark Avg Duration(ms) Benchmark Calls Diff Total Ratio Diff Self Ratio Diff Avg Ratio Diff Calls Ratio
229aten::embedding0.00.00.0014.4913.977.252infinfinfinf
230_ReduceForward0.00.00.003.491.661.752infinfinfinf
231_SplitForwardGatherBackward0.00.00.004.241.032.122infinfinfinf
232autograd::engine::evaluate_function: _SplitForwardGatherBackwardBackward0.00.00.003.280.051.642infinfinfinf
233autograd::engine::evaluate_function: _ReduceForwardBackward0.00.00.000.080.020.042infinfinfinf
234autograd::engine::evaluate_function: torch::autograd::CopySlices0.00.00.0038440.370.2419220.192infinfinfinf
235autograd::engine::evaluate_function: EmbeddingBackward00.00.00.005.910.062.962infinfinfinf
236aclnnEmbedding0.00.00.000.180.180.092infinfinfinf
237_SplitForwardGatherBackwardBackward0.00.00.003.240.391.622infinfinfinf
238_ReduceForwardBackward0.00.00.000.060.060.032infinfinfinf
+ +
+
+ + +
+
+ + + + +
+

performance problem analysis

+
+ + + +
+

memory

+
+ +
+

Memory Operator Issues

+
+ + Analysis of rank 6. + + 发现了243个AscendCL@aclMallocMemInner算子，花费55469.200000000004us，这将导致大量的空闲时间。 + + + + + + + + + + +
Suggestions
1. For AscendCL@aclMallocMemInner: 请通过命令'export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True'设置环境变量，然后开始训练任务。
+ +
+
+ +
+
+ +
+

computation

+
+ + +
+

Pipeline Parallel Stages Issues

+
+ +
+

stage-0

+
+ Description: analysis for slow rank 4 in current stage +

+ +
+

Operator Dynamic Shape Issues

+
+ + Analysis of rank 4. + + + + + + + + + + +
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
+
+
+ +
+

AICPU Issues

+
+ + Analysis of rank 4. + + + + + + + + + + + + + + +
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： +IndexPut修改代码避免使用aicpu类算子12336.790.0003
+
+ +
IndexPut
+
+ + + + + + + + + + + +
Operator TypeCountsElapsed Time(us)
IndexPut212336.79
+
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12336.79 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+
+ +
+
+
+ + +
+

AI Core Frequency Issues

+
+ + Analysis of rank 4. + + Issue: 对于4号卡，在降频期间发现1个算子，频率降低比例超过了0.05。 Only show 10 operators here, see latest mstt_advisor.xlsx for details. +
+ Suggestion: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Operator name Count Total duration(us) AI CORE frequency decreased ratio Average frequency Max frequency Min frequency
aclnnInplaceFillScalar_FillAiCore_Fill1115.885.05%1709.091800.0800.0
+ +
+
+ + + + +
+

AI Core Performance Analysis

+
+ + + + + Cube算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.59%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.58%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.92%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.9%
+
不亲和算子集合 + + + +
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
+
+ + + + FA算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
+
+ + + +
+
+ + +
+
+ +
+

stage-1

+
+ Description: analysis for slow rank 5 in current stage +

+ +
+

Operator Dynamic Shape Issues

+
+ + Analysis of rank 5. + + + + + + + + + + +
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
+
+
+ +
+

AICPU Issues

+
+ + Analysis of rank 5. + + + + + + + + + + + + + + +
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： +IndexPut修改代码避免使用aicpu类算子12350.770.0002
+
+ +
IndexPut
+
+ + + + + + + + + + + +
Operator TypeCountsElapsed Time(us)
IndexPut212350.77
+
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12350.77 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+
+ +
+
+
+ + + +
+

AI Core Performance Analysis

+
+ + + + + Cube算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.65%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.49%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.55%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.85%
+
不亲和算子集合 + + + +
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
+
+ + + + FA算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
+
+ + + +
+
+ + +
+
+ +
+

stage-2

+
+ Description: analysis for slow rank 6 in current stage +

+ +
+

Operator Dynamic Shape Issues

+
+ + Analysis of rank 6. + + + + + + + + + + +
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
+
+
+ +
+

AICPU Issues

+
+ + Analysis of rank 6. + + + + + + + + + + + + + + +
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： +IndexPut修改代码避免使用aicpu类算子12259.620.0024
+
+ +
IndexPut
+
+ + + + + + + + + + + +
Operator TypeCountsElapsed Time(us)
IndexPut212259.62
+
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12259.62 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+
+ +
+
+
+ + + +
+

AI Core Performance Analysis

+
+ + + + + Cube算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.69%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.49%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.94%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.95%
+
不亲和算子集合 + + + +
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
+
+ + + + FA算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
+
+ + + + Vector算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnInplaceCopy_TensorMoveAiCore_TensorMove3584,4736DT_BF1670.0%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove896,3584DT_BF1669.9%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove3584,896DT_BF1669.88%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,1,1,16384,128DT_BF1669.82%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,1,16384,128DT_BF1669.8%
+
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,4096,3584DT_BF16vec_mte2_mte3
aclnnMul_MulAiCore_Mul1,16384,4736;1,16384,4736DT_BF16;DT_BF16vec_mte2_mte3
aclnnMul_MulAiCore_Mul1,4096,3584;1,4096,3584FLOAT;FLOATvec_mte2_mte3
aclnnInplaceMul_CastAiCore_Cast16383,38016FLOATvec_mte2_mte3
aclnnInplaceMuls_MulAiCore_Mul8486912;FLOAT;FLOATvec_mte2_mte3
+
+ +
+
+ + +
+
+ +
+

stage-3

+
+ Description: analysis for slow rank 7 in current stage +

+ +
+

Operator Dynamic Shape Issues

+
+ + Analysis of rank 7. + + + + + + + + + + +
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
+
+
+ +
+

AICPU Issues

+
+ + Analysis of rank 7. + + + + + + + + + + + + + + +
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： +IndexPut修改代码避免使用aicpu类算子12304.890.0002
+
+ +
IndexPut
+
+ + + + + + + + + + + +
Operator TypeCountsElapsed Time(us)
IndexPut212304.89
+
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12304.89 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+
+ +
+
+
+ + + +
+

AI Core Performance Analysis

+
+ + + + + Cube算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.68%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.51%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.76%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.85%
+
不亲和算子集合 + + + +
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
+
+ + + + FA算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
+
+ + + +
+
+ + +
+
+ +
+

stage-4

+
+ Description: analysis for slow rank 12 in current stage +

+ +
+

Operator Dynamic Shape Issues

+
+ + Analysis of rank 12. + + + + + + + + + + +
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
+
+
+ +
+

AICPU Issues

+
+ + Analysis of rank 12. + + + + + + + + + + + + + + +
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： +IndexPut修改代码避免使用aicpu类算子12328.430.0002
+
+ +
IndexPut
+
+ + + + + + + + + + + +
Operator TypeCountsElapsed Time(us)
IndexPut212328.43
+
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12328.43 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+
+ +
+
+
+ + + +
+

AI Core Performance Analysis

+
+ + + + + Cube算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.58%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.44%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.6%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF162.05%
+
不亲和算子集合 + + + +
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
+
+ + + + FA算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
+
+ + + +
+
+ + +
+
+ +
+

stage-5

+
+ Description: analysis for slow rank 13 in current stage +

+ +
+

Operator Dynamic Shape Issues

+
+ + Analysis of rank 13. + + + + + + + + + + +
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
+
+
+ +
+

AICPU Issues

+
+ + Analysis of rank 13. + + + + + + + + + + + + + + +
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： +IndexPut修改代码避免使用aicpu类算子12306.050.0031
+
+ +
IndexPut
+
+ + + + + + + + + + + +
Operator TypeCountsElapsed Time(us)
IndexPut212306.05
+
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12306.05 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+
+ +
+
+
+ + + +
+

AI Core Performance Analysis

+
+ + + + + Cube算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.61%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.45%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.73%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.85%
+
不亲和算子集合 + + + +
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
+
+ + + + FA算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
+
+ + + + Vector算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnInplaceCopy_TensorMoveAiCore_TensorMove3584,4736DT_BF1670.0%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove3584,896DT_BF1669.89%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove896,3584DT_BF1669.89%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,1,1,16384,128DT_BF1669.82%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,1,16384,128DT_BF1669.8%
+
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,4096,3584DT_BF16vec_mte2_mte3
aclnnMul_MulAiCore_Mul1,16384,4736;1,16384,4736DT_BF16;DT_BF16vec_mte2_mte3
aclnnMul_MulAiCore_Mul1,4096,3584;1,4096,3584FLOAT;FLOATvec_mte2_mte3
aclnnInplaceMul_CastAiCore_Cast16383,38016FLOATvec_mte2_mte3
aclnnInplaceMuls_MulAiCore_Mul8486912;FLOAT;FLOATvec_mte2_mte3
+
+ +
+
+ + +
+
+ +
+

stage-6

+
+ Description: analysis for slow rank 14 in current stage +

+ +
+

Operator Dynamic Shape Issues

+
+ + Analysis of rank 14. + + + + + + + + + + +
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
+
+
+ +
+

AICPU Issues

+
+ + Analysis of rank 14. + + + + + + + + + + + + + + +
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： +IndexPut修改代码避免使用aicpu类算子12324.210.0003
+
+ +
IndexPut
+
+ + + + + + + + + + + +
Operator TypeCountsElapsed Time(us)
IndexPut212324.21
+
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12324.21 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+
+ +
+
+
+ + + +
+

AI Core Performance Analysis

+
+ + + + + Cube算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.64%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.49%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.86%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.8%
+
不亲和算子集合 + + + +
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
+
+ + + + FA算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
+
+ + + +
+
+ + +
+
+ +
+

stage-7

+
+ Description: analysis for slow rank 15 in current stage +

+ +
+

Operator Dynamic Shape Issues

+
+ + Analysis of rank 15. + + + + + + + + + + +
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
+
+
+ +
+

AICPU Issues

+
+ + Analysis of rank 15. + + + + + + + + + + + + + + +
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： +IndexPut修改代码避免使用aicpu类算子12297.570.0002
+
+ +
IndexPut
+
+ + + + + + + + + + + +
Operator TypeCountsElapsed Time(us)
IndexPut212297.57
+
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:1 | Elapsed Time(us):11999.58 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(85): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+ IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:1 | Elapsed Time(us):297.99 +
+
+ +
+ +

+ Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 +

+ +
+ +
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(89): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): +
+ +
+
+ +
+
+
+ + + +
+

AI Core Performance Analysis

+
+ + + + + Cube算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
性能优化算子集合 + + + +
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.55%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.5%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.62%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.75%
+
不亲和算子集合 + + + +
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
+
+ + + + FA算子相关分析，参考如下: +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
类别描述及建议
bound算子集合 + + + +
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
+
+ + + +
+
+ + +
+
+ +
+
+ + +
+
+ +
+

schedule

+
+ + +
+

Conjectured GC Analysis

+
+ + Analysis of rank 6. + + 在34079031.859us的空闲时间内几乎没有主机任务，这可能是由Python的异常GC引起的 + + + + + + + + + + + + + + + + + + + + + + + + + + +
Suggestions
1. 实现高效的Python内存管理；不使用时及时释放内存，避免长期占用；避免对象之间的循环引用。
2. 使用 gc.set_threshold() 来调整垃圾回收阈值可以延迟垃圾收集，但这是一个临时解决方案。
3. 使用 gc.disable() 来关闭GC，注意这是个临时解决方案。
+ + The details of top 2 garbage collection events are as follows: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
timestamp duration(us)
1747647483551821.833818722.418
1747647606194246.2260309.441
+ +
+
+ + +
+

Affinity API Issues

+
+ + Analysis of rank 6. + + The analysis results of following affinity APIs are based on runtime env + cann-8.0.0 + and + pytorch-pytorch + +
+ + + + + + + +
torch_npu.npu_rms_norm
+
+ +
+ +
No.1 code stack, called 28 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(620): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.2 code stack, called 28 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.3 code stack, called 8 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.4 code stack, called 4 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.5 code stack, called 4 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.6 code stack, called 3 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.7 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.8 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/_ops.py(1116): __call__;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(582): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.9 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(620): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.10 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(158): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.11 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.12 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(250): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.13 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(598): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.14 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(272): collate_tensor_fn;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(155): collate;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(172): ;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(171): collate;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(398): default_collate;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py(55): fetch;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/dataloader.py(757): _next_data;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/dataloader.py(701): __next__;
/usr/local/python3.10/lib/python3.10/site-packages/torch_npu/profiler/_add_mstx_patch.py(28): wrapper;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(74): load_batch;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(373): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.15 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.16 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.17 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.18 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.19 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.20 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(166): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.21 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3762): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.22 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.23 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(235): repeat_kv;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(573): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.24 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(564): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.25 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(180): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.26 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.27 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(172): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.28 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(566): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.29 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.30 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(236): repeat_kv;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(574): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.31 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.32 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.33 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(526): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.34 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(179): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.35 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1073): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1359): gather_forward_split_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1390): gather_sp_output;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(253): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.36 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
+
+ + + + + + +
torch_npu.npu_confusion_transpose
+
+ +
+ +
No.1 code stack, called 32 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(598): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.2 code stack, called 9 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(766): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.3 code stack, called 7 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(636): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.4 code stack, called 6 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.5 code stack, called 5 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.6 code stack, called 5 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(595): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.7 code stack, called 4 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(585): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.8 code stack, called 4 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(590): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.9 code stack, called 3 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.10 code stack, called 3 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.11 code stack, called 3 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(580): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.12 code stack, called 3 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(589): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.13 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(235): repeat_kv;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(574): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.14 code stack, called 2 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.15 code stack, called 2 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.16 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.17 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.18 code stack, called 2 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(597): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.19 code stack, called 2 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(638): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.20 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(528): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.21 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(157): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.22 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.23 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.24 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(163): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.25 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(205): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.26 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(206): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.27 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(566): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.28 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/nn/functional.py(2380): silu;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/activation.py(432): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.29 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(179): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.30 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/_ops.py(1116): __call__;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(582): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.31 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.32 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.33 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.34 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(620): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
+
+ + + + + + +
torch_npu.npu_rotary_mul
+
+ +
+ +
No.1 code stack, called 28 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(179): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(206): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.2 code stack, called 28 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(179): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.3 code stack, called 10 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.4 code stack, called 5 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.5 code stack, called 4 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.6 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(166): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.7 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(235): repeat_kv;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(573): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.8 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.9 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.10 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(157): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.11 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.12 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/_ops.py(1116): __call__;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(582): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.13 code stack, called 2 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(766): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.14 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.15 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(163): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.16 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(564): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.17 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.18 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.19 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(564): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.20 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.21 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.22 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(170): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.23 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(206): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.24 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(178): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.25 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.26 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.27 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.28 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.29 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(250): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.30 code stack, called 1 times
+
+ /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(81): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(250): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> +
+ +
No.31 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(774): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
No.32 code stack, called 1 times
+
+ /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(811): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply +
+ +
+
+ + + + +
+ +
+
+ + + + +
+
+ +
+

dataloader

+
+ +
+

Slow Dataloader Issues

+
+ + Analysis of rank 6. + + dataloader加载数据速度较慢，一次迭代花费138000.9us，通常小于10000us。 + + + + + + + + + + + + + + +
Suggestions
1. 请检查数据目录的磁盘I/O。如果您正在ModelArts中训练模型，请将数据移动到“/cache”或装载更高效的云磁盘以获得更好的I/O。
2. 尝试调整dataloader参数'num_workers'。
+ +
+
+ +
+
+ +
+
+ + + +
+

Generated by Ascend Training Tools

+
+
+ + + + + \ No newline at end of file diff --git a/applications/ColossalChat/profile_log.txt b/applications/ColossalChat/profile_log.txt new file mode 100644 index 000000000000..20e56e0b2c33 --- /dev/null +++ b/applications/ColossalChat/profile_log.txt @@ -0,0 +1,278 @@ +[2025-05-19 17:44:04][INFO] cluster analysis is in the process, please wait... +[2025-05-19 17:44:04][INFO] Begin generate communication data. +[2025-05-19 17:44:08][INFO] Communication data read completed. +Cluster analyzing: 0%| | 0/5[2025-05-19 17:44:09][INFO] HostInfoAnalysis completed +[2025-05-19 17:44:09][INFO] ClusterBaseInfoAnalysis skipped, since data type is not db +Cluster analyzing: 40%|██████████████████████████▊ | 2/5[2025-05-19 17:44:10][INFO] CommMatrixAnalysis completed +Cluster analyzing: 80%|█████████████████████████████████████████████████████▌ | 4/5[2025-05-19 17:44:12][INFO] CommunicationAnalysis completed +Cluster analyzing: 100%|███████████████████████████████████████████████████████████████████ | 5/5[2025-05-19 17:44:13][WARNING] StepTraceTimeAnalysis completed +Cluster analyzing: 100%|███████████████████████████████████████████████████████████████████ | 5/5 + + + +Cluster analyzing: 100%|███████████████████████████████████████████████████████████████████ | 5/5 +[2025-05-19 17:52:51][INFO] The cluster analysis result file has been generated: /home/duanjunwen/ColossalAI/applications/ColossalChat/train_profiling_data +[2025-05-19 17:52:51][INFO] Cluster has been analyzed because of the existence of cluster analysis output directory. +[2025-05-19 17:52:51][INFO] Skip Cluster analyze backend. +[2025-05-19 17:52:52][INFO] Start cluster schedule analysis +[2025-05-19 17:52:52][INFO] For cluster schedule analysis, maximum free for rank 6 and step 0 +[2025-05-19 17:52:52][INFO] Enable schedule comparison of fast and slow rank/step +[2025-05-19 17:52:52][INFO] Start cluster computation analysis +[2025-05-19 17:52:52][INFO] Steps and ranks to be analyzed of different pipeline parallel stages are {"stage-0": {"maximum": {"rank_id": 4, "step": 0}, "minimum": {"rank_id": 0, "step": 0}}, "stage-1": {"maximum": {"rank_id": 5, "step": 0}, "minimum": {"rank_id": 1, "step": 0}}, "stage-2": {"maximum": {"rank_id": 6, "step": 0}, "minimum": {"rank_id": 2, "step": 0}}, "stage-3": {"maximum": {"rank_id": 7, "step": 0}, "minimum": {"rank_id": 3, "step": 0}}, "stage-4": {"maximum": {"rank_id": 12, "step": 0}, "minimum": {"rank_id": 8, "step": 0}}, "stage-5": {"maximum": {"rank_id": 13, "step": 0}, "minimum": {"rank_id": 9, "step": 0}}, "stage-6": {"maximum": {"rank_id": 14, "step": 0}, "minimum": {"rank_id": 10, "step": 0}}, "stage-7": {"maximum": {"rank_id": 15, "step": 0}, "minimum": {"rank_id": 11, "step": 0}}} +[2025-05-19 17:52:52][INFO] For stage-0, slow rank is 4 +[2025-05-19 17:52:52][INFO] For stage-1, slow rank is 5 +[2025-05-19 17:52:52][INFO] For stage-2, slow rank is 6 +[2025-05-19 17:52:52][INFO] For stage-3, slow rank is 7 +[2025-05-19 17:52:52][INFO] For stage-4, slow rank is 12 +[2025-05-19 17:52:52][INFO] For stage-5, slow rank is 13 +[2025-05-19 17:52:52][INFO] For stage-6, slow rank is 14 +[2025-05-19 17:52:52][INFO] For stage-7, slow rank is 15 +[2025-05-19 17:52:52][INFO] Enable computation comparison of fast and slow rank/step in different pp stages +[2025-05-19 17:52:52][INFO] Start cluster communication analysis +[2025-05-19 17:52:52][INFO] Minimum SDMA bandwidth for rank 4 +[2025-05-19 17:52:52][INFO] Minimum RDMA bandwidth for rank 0 +[2025-05-19 17:52:52][INFO] Minimum SDMA bandwidth for rank 4 +[2025-05-19 17:52:52][INFO] Minimum RDMA bandwidth for rank 0 +[2025-05-19 17:52:52][INFO] Minimum SDMA bandwidth for rank 4 +[2025-05-19 17:52:52][INFO] Minimum RDMA bandwidth for rank 0 +[2025-05-19 17:52:52][INFO] Start cluster memory analysis +[2025-05-19 17:52:52][INFO] For cluster memory analysis, maximum free for rank 6 and step 0 +[2025-05-19 17:52:52][INFO] Start analysis EnvironmentVariableAnalyzer with environment_variable_dataset +[2025-05-19 17:52:52][WARNING] convert_to_int_with_exception: an empty string was encountered. +[2025-05-19 17:52:52][WARNING] convert_to_int_with_exception: an empty string was encountered. +[2025-05-19 17:54:39][INFO] Start analysis MemoryAnalyzer with timeline_event_dataset +[2025-05-19 17:55:24][INFO] Start analysis ByteAlignmentAnalyzer with ProfilingDataset +[2025-05-19 17:56:09][INFO] Start analysis BandwidthContentionAnalyzer with communication_dataset +[2025-05-19 17:56:11][INFO] Start analysis RDMARetransmissionAnalyzer with ClusterCommunicationDataset +[2025-05-19 17:56:11][INFO] Start analysis PacketAnalyzer with communication_dataset +[2025-05-19 17:56:11][WARNING] Analyser: ComparisonAnalyzer don't rely on any dataset! +[2025-05-19 17:56:11][WARNING] Analyser: PPStageComputationAnalyzer don't rely on any dataset! +[2025-05-19 17:56:57][INFO] Start analysis DynamicShapeAnalyzer with ProfilingDataset +[2025-05-19 17:57:43][INFO] Start analysis AicpuAnalyzer with ProfilingDataset +Building dataset for timeline analysis: 0%| | 0/2315055 [00:00 1: data_iter = iter(dataloader) step_bar = tqdm( - range(len(dataloader)), - desc="Step", - disable=not is_master(), + range(len(dataloader)), + desc="Step", + disable=not is_master(), ) + print(f"len step_bar {len(step_bar)}") for step in step_bar: - print(f"data_iter {data_iter}") + print(f"Profile Start at step {step}") + prof.start() outputs = booster.execute_pipeline( - data_iter, - model, - criterion=lambda outputs, inputs: outputs[0], - optimizer=optimizer, - return_loss=True, + data_iter, + model, + criterion=lambda outputs, inputs: outputs[0], + optimizer=optimizer, + return_loss=True, ) loss = outputs["loss"] + print(f"step {step} loss {loss}") if booster.plugin.stage_manager.is_last_stage(): global_loss = all_reduce_mean(loss, plugin) @@ -119,9 +151,12 @@ def is_master(): optimizer.step() optimizer.zero_grad() + + prof.step() else: total_loss = 0 for step, batch in enumerate(dataloader): + prof.start() input_ids = batch["input_ids"].to(device=model.module.device) attention_mask = batch["attention_mask"].to(device=model.module.device) outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids) @@ -136,8 +171,11 @@ def is_master(): print(f"finish optimizer step") total_loss += loss.item() + prof.step() - print(f"Epoch {epoch + 1}, Loss: {total_loss / len(dataloader)}") + print(f"Epoch {epoch + 1}, Loss: {total_loss / len(dataloader)}") + print(f"Profile Stop") + prof.stop() if __name__ == "__main__": test_hybrid_qwen() diff --git a/applications/ColossalChat/tests/test_ray.py b/applications/ColossalChat/tests/test_ray.py index ca2f1456adef..16f5da507e88 100644 --- a/applications/ColossalChat/tests/test_ray.py +++ b/applications/ColossalChat/tests/test_ray.py @@ -32,7 +32,7 @@ def destroy_worker(self): { "precision": torch.bfloat16, "device": "npu", - "num_devices": 8, + "num_devices": 1, }, ], ) diff --git a/applications/ColossalChat/tests/test_ray_vllm.py b/applications/ColossalChat/tests/test_ray_vllm.py index 2deb048254fc..a0d1270db229 100644 --- a/applications/ColossalChat/tests/test_ray_vllm.py +++ b/applications/ColossalChat/tests/test_ray_vllm.py @@ -13,7 +13,7 @@ parser = argparse.ArgumentParser(description='VLLM args.') parser.add_argument("-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. ") parser.add_argument("-l", "--max_length", type=int, default=8192, help="Max sequence length") -parser.add_argument("-w", "--world_size", type=int, default=1, help="Gpu nums") +parser.add_argument("-w", "--world_size", type=int, default=8, help="Gpu nums") parser.add_argument("-t", "--temperature", type=float, default=0.8, help="Temperature") parser.add_argument("--top_p", type=float, default=0.95, help="Top p") parser.add_argument("-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. ") @@ -51,7 +51,7 @@ def destroy_worker(self): { "precision": torch.bfloat16, "device": "npu", - "num_devices": 8, + "num_devices": 1, }, ], ) From bf5b1537002d896b1244fd2fce0077bb8e2fa4a3 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Fri, 23 May 2025 11:38:21 +0800 Subject: [PATCH 09/24] [feat] support ColossalaiRL on Ascend --- .../ColossalChat/ColossalaiRL_On_Ascend.md | 95 + applications/ColossalChat/fusion_result.json | 1 - .../log/mstt_advisor_20250519174404.xlsx | Bin 102316 -> 0 bytes .../mstt_advisor_20250519174404.html | 7585 ----------------- applications/ColossalChat/profile_log.txt | 278 - applications/ColossalChat/rl_example.py | 10 +- .../ColossalChat/tests/test_log_prob.py | 58 + colossalai/shardformer/layer/loss.py | 3 +- colossalai/shardformer/modeling/qwen2.py | 1 + 9 files changed, 161 insertions(+), 7870 deletions(-) create mode 100644 applications/ColossalChat/ColossalaiRL_On_Ascend.md delete mode 100644 applications/ColossalChat/fusion_result.json delete mode 100644 applications/ColossalChat/log/mstt_advisor_20250519174404.xlsx delete mode 100644 applications/ColossalChat/mstt_advisor_20250519174404.html delete mode 100644 applications/ColossalChat/profile_log.txt create mode 100644 applications/ColossalChat/tests/test_log_prob.py diff --git a/applications/ColossalChat/ColossalaiRL_On_Ascend.md b/applications/ColossalChat/ColossalaiRL_On_Ascend.md new file mode 100644 index 000000000000..1b258137d40f --- /dev/null +++ b/applications/ColossalChat/ColossalaiRL_On_Ascend.md @@ -0,0 +1,95 @@ +# ColossalaiRL On Ascend +The document is the instructions for using ColossalRL on Ascend. + +## 1.Prepare Develop Environment + +### Install Colossalai & ColossalChat +```bash +git clone https://github.com/hpcaitech/ColossalAI.git +git checkout grpo-latest +pip install -e . + +cd ./applications/ColossalChat +pip install -e . +``` + +### Install Fuyao Ray +Please update CANN before install fuyao ray +```bash +# Install CANN +source /usr/local/Ascend/ascend-toolkit/set_env.sh +./Ascend-cann-kernels-910b_8.1.RC1.alpha001_linux-aarch64.run --devel + +# Clone Fuyao Ray +git clone https://gitee.com/openfuyao/ray.git +cd ray +git pull origin pull/5/head + +# Install ray +pip install ray==2.43.0 --no-cache-dir + +# Create soft-link from fuyao-ray to ray site-package +cd .. +ln -s ./ray/python/ray/ /usr/local/python3.10/lib/python3.10/site-packages/ray + +# Install Fuyao Ray +cd ray +python python/ray/setup-dev.py +``` +### Prepare Model & dataset + +```bash +huggingface-cli download --local-dir-use-symlinks False Qwen/Qwen2.5-7B --local-dir /models/Qwen/Qwen2.5-7B +``` + + +## 2.Set Distributed Config +Now, we need to set distributed config for multi-node. + +### Set Host IP Config +First, we set host ip config. +For example. I need to configure a cluster of 4 nodes, then I do +```bash +vim /etc/hosts +``` +Then write IP node map to /etc/hosts +```bash +10.0.0.3 npu-3 +10.0.0.4 npu-4 +10.0.0.5 npu-5 +10.0.0.6 npu-6 +``` + +### Set Ascend Multi-Node Config + +```bash +export ATB_LLM_HCCL_ENABLE=1 +export ATB_LLM_COMM_BACKEND="hccl" +export HCCL_CONNECT_TIMEOUT=7200 +export WORLD_SIZE=32 +export HCCL_EXEC_TIMEOUT=7200 +export HCCL_SOCKET_IFNAME=eno0 +export RAY_COLLECTIVE_MEET_TIMEOUT_SECONDS=7200 +``` + +## 3.Run task on ColossalaiRL-Ascend + +### Start Ray Cluster +Now we use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: +```bash +ray start --head --node-ip-address=10.0.0.3 +``` +Then, for each slave node (10.0.0.4/10.0.0.5/10.0.0.6), we add to the ray cluser by following code: +```bash +ray start --address='10.0.0.3:6379' +``` + +### Run Scripts +Then, run start command at master node +```bash +# Hint1: replace /models/Qwen/Qwen2.5-7B to your model path +# replace /datasets/train-alignment.jsonl to your dataset path +python rl_example.py -m /models/Qwen/Qwen2.5-7B -d /datasets/train-alignment.jsonl --master_address '10.0.0.3' -t 16 -i 16 -p GRPO-Train-Align-Debug -g 2 -ibs 1 -tbs 2 -tMbs 1 -tmbs 2 -imbs 1 -b vllm -e 2 -rt boxed -s "Please reason step by step, and put your final answer within \\boxed{}." &>run_log.log & +``` + + diff --git a/applications/ColossalChat/fusion_result.json b/applications/ColossalChat/fusion_result.json deleted file mode 100644 index ec747fa47ddb..000000000000 --- a/applications/ColossalChat/fusion_result.json +++ /dev/null @@ -1 +0,0 @@ -null \ No newline at end of file diff --git a/applications/ColossalChat/log/mstt_advisor_20250519174404.xlsx b/applications/ColossalChat/log/mstt_advisor_20250519174404.xlsx deleted file mode 100644 index 7d6f70c735bb1de84b3b47d809e90965ac236ee4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 102316 zcmeF)Wl$aMx+naFySux)ySux)OK>Maf&_PWcMTo_1b4UKu0ex4yyV$?*UWQH?Kv~` zews6@ibVrWH#dF%*L8LOkXDuh14jn{0MPFbAwcr!icup701$)*0KC5d^u!z;+$Xd zf@xZosf=CwTj%Fxr-p3KhOSYOxK77~1xR`9nt;0UtGy|q1hdB+V{Z(-15dM8Xdh>j z+rw46g;4v2c9nse_zeuQ^ORGi7mQuA*AV;!=s^&a7%@_ODRFK{aqgIKr!3?xZi*g~ zM|I&cYz<|Vio9)g5s8Yx2n`5~YFJuB4D~+RvNR1c&171^fVPDYMd3c;Ke6$TzjPuEX z)#&j!fAmqb>McGzeI+hB#OOrmMAzu6jWZH09AqdN6suDaeCEF8i#(ZZ`EW<94uJm%me#y|B!1*g5Dq z{)#^}DIm7eeIB7UmnR4h@SZ!*3enonr&XRhFwk6Rdlo3ok)F!WYtFJpQ7c#69@Xu1 z$6qmkIhc%+6K#2G#n1Q>Nam>nWA@BCL&Y`a1Ni$9X*cqJjeL*2GAaOo`+h{c?3n&> zP&^%7KA1W>e)tpozYb8xqMYq}`af^^Dw#Ao?&2KiY@4gWE;auG2^OeRup5>>B5SFY zu}&1m{tn&1nlOEDE;xfm;8C4o`3I$oJ`zTT3QvAf?1T#l<}@ z?`LJ`9$gb9qLt{{H3?40Vq7Kfz^lx$^4wfHvx~se*7(H3M4!noy0uJGN7|XR617t% ztFrm|F>|&35l30&;oGTMf6KPD8k)l;3?hOhuN}0<(FfnM@NMR~`v}vT*XBS$L1T7# zjYki7*g;~fM3_Ny{V{EB%dvj>U^7bnF?DXsk^Zq@GfVw3eQwK%e)?cDOnvyGnWJL6 z819~`9^!H4uZThHeOYa+KBFF8vLK$uZWtVC97nWt&I9lA9V51O5SlwnV=cmvq|h{YOCxgA=HIEh?vcI zc{R8|Xu44UA+fP|#Jc11)FNs00})?h8nfwOUEjDeh-DZT35}wZg9+mH_5)`$F;sY zOxfsn22@imi<3t!Yt||ZqmIY7caQU0m8A#Gczw49`#0a0-uRZQnxEpi_BunAv>BEU z+Un|1EHsybPrR;+ZI?~w&sGjLGMd+)U7gBzv_n-6xnZ*49Usdu^4NeR{k&2f`xMmv)JzaB4er$%;@|=jtZn&Fp z+iPLw=`$tDAFO9Mcl5kDwr=s{VzTh3D9@~tyJ*J}x;y$&jaNH)d-rtjaoIt%f9NF~ zipyJan3unULh$i0Ex#9@0D|{qYHKPUo`8_x*D%YkpT8Ql$r&+E+h%p{q8KoFU7EEI zmF@b8Hs}*u`nN>8AqFA*DdAyfO-*pA`hV;~b@%+(E4NJa+U<#ggTTVaYDYvj?GuHc zNT4_s`TDg?0f8wPj*Ssr-%0;BrE(YsTphO@=H2^Nmx37dG;o=F#m@PgfjfaM0W+v| zjo?)MOog!wv%&sS`f2}Ip#=J+$drU;BEjGkiAb-6=6~$YCNEB&o;*kgRR}_`S)R=h z8p~3j`rGEW%?4qTR&D8+hTW&d!4#VbxUGK<&4-X$y#KNle}_LjcZ}#$3Dsmdwz6d& zdLFm(b1Q2VwJ*&v1h z5n%n5$Q;x-c<4FzeUs$fg%5^|-snsVImvSEt5=eV`$@&&G4+ePYeahAOpRj-xnq_j z&e54eVMRg{%Ed_3tQVCbknDs42Y)AnV1g6_`UAE#!laVm$zh$vdiUtOxH)ABC!%4Q zA(BARp(3_%o+c-uTYQ9`n(MW3DZO4|6U!w_`yoTOolTCoj>a2g~|W*ScdS&F-jgTH3T&xx)O;x z8O}U3%BWKoCO`HqopLl|J}yW_!258~ML7~wqLJ(1g{mNsLSA6i3+2iXn&$l zlN?8c5-1~h2nch)=Hm{RNrq{Mu(fS=6GIb1i@MN23){a(fd5Nbw7D%=5kumo3j}G< zU+X^p8Kq2$C6wj)CGwyT5hJ7}4|LZ@bW?!@o!Gc(_*G}DGkFnYa3pY&HgxbV!N0!O zMJd)>l8zI3w5|@4^Xp$t0=r>6ehMmR(chv!E`aQ50m-FVHHZ!DaC$ z^i<;AU?Oq+{Ls8VkiijJM&6U>MbIP2V3sSX-|(9sQsA#cnm7N27#{a2ktDJ@#`MGE zXtwKA7yT=fAsERKGKp}$pN%Pa`zhMZHb z@^-hvn7p~SKX@HM$%4Gc`51Z0CjEpRHGL22`|j4{7`ReTZo!9dF#lRRVS{tPZmJEM z*|Hf>`YTZC3U`YV|DSs`F0o*qg;s~r_YDjcqKe~(sYP+<2~hnLL0oG)ms_&*)XVb6 zrd5IJ=5cEShPGdQDlOD)y+}2;wJVPnZxwGaifhd``=@KJ8jKYNZ#64@Deh}OI(`ZL zy5?(sdIS0US&N6W69?A&Ib6Vd+xs^ykM*yXmn;{X$^72(UNBPb`i%``z@theRNBha zHGHcCi~~77$DfkV`Wvl~+YD&iT7mtz&*}|jO55JD!0!r&9llrZp@t^OQ|r~^B#^z| zd3t?;1T7z$O9)VE4DKusMw)Ni9_w#7q0tbAlCKZ$P`yGVS-N&3XWeAWzLp>Cmri2r z2N~8;p3$AORj+4CdRn%v1=$F~+D0lS=y6~&mzwWdmPyx^ z9)YMxATT~XIoOz->DI@x`baLUfq1-^yN3o`Nxecq&Eq?dZ2zE9B=vW329kT!Kt~cC zWD1kpkg$7Hlnv4XYQ*1^(K9W5K5Jsij)VW51a9JhH`=*$yXKVwDb%MrQzp&!TPj4&6H{;g@}RwVbHA#-$mFUQ2R$^b(H-YU{ZHh2 zLYhbYvRbbShsMQ>DT6!gKW?SJ6470mT72+7F}C>Ehg6~9)_hCZc&3=d@fXRvh577Z zgYZG?PCj{5-P=i7Vr|O$KP*%%d=kJYnQ>GcZ2gXDDfR6|phD>HVFK`7g-w7100b!i zHcV`P!W6B&;;_Ss?0c>S{gG*W%R(Bm8B1?PK&tIQyUVnNRJ&GA1LGW51 z9gry9k#~w|P@B&htqd(alk#bN@@=SIIK915&ke0itP#`%v+opQPyL}k;fb)spIcD* zzE3iK`>v#hJ)h@Uf36!nA|FQn#7X<7{?6xSy`)t!E6>FPXDolW{+e~I+QBWgr`r!} z?f8(q9H*&vD`rBk^or3!BlX-G8v`4T;||>Pvi(&*VXLPtE?Un*4=l0AR~G!NmEQD4 zoLt@ANgWjX+R%C%w@~|HpW?WP9@n70z3K6{CE$eDR_5zYwB1%8O1k-%`?PP#%<4G4 zq!PWf6h=QkK>HZzW=;P1D8x0ygcBA{QVHKT5a$lLZ9241xP8xOF3G;$8b*)BQiHW4 zn-iDzwOSOy)S6(qM5Q@UpSei1>!CtFebrG z2p*;pf1PmbT=R+1_u+~uo0YjyBs2B|!LsR?S_W!c&%&^vq0JhzyKdH}V?T%>QnOo? zbrFdsY-J)xF2fX~;5d!w4iX_Z=$LxEf3_TFTRjU4t^GtW&RvFs2Z1StmhfN~^{|V* zCE5donQtHc9ydjy*A$UR40TONjhI$|g=-*-MEY@=TZZhYn?JH!pHx;(PqW-(NnE%aqJNa6!Hf(c zLrtE6N!VC6KwMIq1Dc$)54R?bjZ=;RlY|(Y!8obs2j{FJY5)ZVdA}kvO?i^VRDtP7 zZyZoHUUJrw2onrF8aH)M`CbjP^duwjk`k#VD)5~=RyICJJ%zC;B!$!CC4tS zxF2H5-bG3K7zb7~W$E>MHZYJw#Tl@T1^B~T{ zQH|E+#nqJIfYNKtDxzEe%d1COrNP<*SH+X>3gU**^Zeb55Z>t9f4GKppzlKPhX4So zasF)$WdBnGv(w+NAtjLq&mYi}&QNBr2(i94fq?N8(I$`VR6&@OL5R!LcRbwU!2Hx~ z3DKbwJR$X*JTA>Sd)*l>bu8UEq+qHPbIzfRt2a-|W{ z&bqR>bTp*MwG>h?jJ*8NGTKeo%TCS!yZ`Fpcwbr5b@ULbi=PF(Q5)CEe_)8#U{aFiy4Tk}yrx=#nr=b?@4H znKGL^M!-c9-8uDIk1-F{?BhIN{wAGmmZE(05gUPbFE4Fo?9HiXyY85-t4I~Dy1MNZ z*}=f*YdN@#;TCN57`JEn?lK1)n+&OGv}QB0s#y`Un+3QkxJU%_P)#)WZ2N~DR*Sr0 zOe|A?w1_b1f#%iiVN~BC>gmXV=)~~4CTI{k{S9+;BZ4$;n1Ru7nnD1}8|q3j3A`$# z$jF~v;9D`ui0k5F!SMi!$cdBH3#62{mDLf4D71$`z^4-i?)8kbnqu1d#FZgnDxkY- zqO-hftf^Tu9_9dn5y-n^Ql!kN$Xao*L36rO%IaXk1li>^1hS*V8Q?9Qtj2|%VBoNQF3bm^p?4);VnhHrq;I{Z6C*`Y?QWc9MGuDrLoe6d zG%~9CzDO*L2eQ}{zca=FK|Rc2um}RqKw;g)#;ae#07(-?N=SzXBBOe@Od2=iSv$b| zsdUy^B$zSq>xV*w{6c7(%w{JrWZsA=sk8)>V1td4*{u*#H!xF_YC90sI8^<{SrANz zMjdbp)`VkTsR2&5nV%Uq+UvE|Zz<2R8ftr?|KK8N7@JlT%c3 zJcHr-ygZ^~F2P3~caT|c z-9b)aL!MxgvR+^a_ZTn1;0r{~&QdbVIj{=#eP~toRjHm~-+P3k$_Y0jdIl<*!xeY1Pmv?`)+fNLi*hz7SPH*&{ECo$TK=z{{S@c`*;YqRhy3bvU8t-1QYXzY3D~aJ*q-l-Y;k(PpyoMr=|PH#|1T!vv;MwIrdle-j_C? zw9s$vLigeJS~?rYUcvr;;;M;M`?cr&Cm1mv|F&pz{3+VS1_rLH+zCFNM)|V>MB+Xu zo582i9OGx2Ty5UY&MkLUO)Jf&3{7G)QFpH|0rm834S*;NE~K`da=SceayX4)(w8Hq zxAq5@+rf*o*GIRfNBzqE=M%4|*Xymv+u7r*x7XVF&cUmOr_jn3BrPMwmcfOCw@>Xt zZ=;$1-Z!Uicd<{CgFC%CxSO}HHxCD)p|za``m3F9KJBgFwMuITSIi&0zPGi0l=g3b za&vyWIrF;Nx{%^BVw9h4#NyIgXgS!Bx<5VB8sPSQ_Ii1}Kf8Xt%iKBby%RZ$F{&X< zc)H82?Of08^uF2RUD`k9Y2)+r@qN{LS=n8C=+D%OReL*qyg&83IJde=etvO#9DEKP zMXH_5#Q8|_wNi@db?5A)_hM)M+=%e?>e}A5{q_2Rq&g$l|MK;E@Able(xK*{m|v)^ z_3>cm&A;vS_wR=rZ*|I#r+Pb+ldpd5{@MmY{23c>es50SA9Bx$U)bmFM8;-`_q~zL zl0C-d2EHnpEgrDARr*+0`E?VY*&yG!#{C`@x>XXo74kpPxL$8!dHxP}I!k?e$!pkN zuw|3<6?-q$A3xU*KiB`n=sVupEb`hU0Rrt|NduEM?AkN2)|G2os$+%Uewm@y)v?f9 zx1xQDVf($QAF}$&8B!TCP5Goy91#;#dAJ$IrP$(KcN?<}Vi|aJ7uDX;2>n+sB1TMIYgd#D@4_rWJzH zNJ6K>boQ7bS&s1sNG%h4x%>3@nF(1|@Jl4d7(??l2wkyt_k9t?j(vuV*{iRTdY|t?!603PTx`9%Bt&)cW~iY%~_}z9;Ri%PJ?h^RP)r1Ixi*6GyJth?t{1jV!-ji`=mR z)6{Dy#$UwBQkq>l8cWeb+gCRy!>Q%Wtea+1CQ-20XsxUt391SZMw)lKzlj$uitPK2 zcY_&bv7|VR%K+?hzyv2jI(M0!yR2?gV+I97O%Ul1q)EQ-teR+p7Ra{jToRdrnR>S@ zSU1lWnJ{r91w||F*GA!1|7<|_Lu=vs)7lJpX0s^FSDksB67cbL=kG8c zi81)N#zi3ah8tS#e2w4QmS?khH2`1VNIKt4aICD~#4H`deSHyopNKXe+t6dHi(5k| z21>Rwn!TDD2N#P`tVw?6w@2V%uQ}_2yy7g_%zL;Y=wXvU@J${-a4&k#JI`|l21Qin z3XB>58ie}I^0F8oga-0l-~vk6$B(pq*+)(&-Z~ia^XsHBI8ushQI#OYY6?=0Nq8NL zJ@_Cfwm4tv1VN1yNCB_zp+Z#xNVeND4GOaHXP)1)5+_4a)dgfkQkU^^liCzuuivJt zNaVqpi}R-ThNl=><;@RS&GR7M=TG;uC&8*44RKl5F*iL+wZm-phy zsX_%Vcj5{?iVwdVI=-qMDOby8i8BcSh+ERxkb+~fX%^eByshtr+Xwk%ioZ#C zGyq-UI=W-BTH~x=NHqEJQuP!+%^wJpN_Vm6+p8TZ#eon{RHi7+VLTpbY(6<6geAd5 zoVW);Gb%AoPsW8U{9bCekTNqi%qy(6mB=%e!c%m(b(cc2)>P;{=9x#9j!vj3Cjl{Y zIE3fYju;Bmj{NLx_kCWUPVF9~gT;WCjQ=>E4t!}qS8-&4T~U5VTO|ZE?7UUZW$`gC zrDbn5kRXOJnt{^0!TT&-5LY=d!fQ&Wk73gvqt7u? zPe%5@yCLE;q3$b?S8Ma8lbLM`Z7wacIEU#q20wh55`qy$FAa{KM)l=DJ(HWz7NEZ3 zLBp`;X89$}xc3=9QYUbys2GW!+Yvx!)APz%?Q)3&^N<;XA(-KD;#f*5*Gcw-r@s08 z!RY1=@RM?0oK&rvB#suMNiAq%;A81TSo(=YF4(X!Xkgw|wF-j}0xrN@73J_-421ac z?9Z>Xoda~L@^1(Rb$e0C0>1E4+hZU~D`Puo!ofQEd$P9XO07i{O0A0ApI@A>B*>l7 zWkf~RXCO$_gZKwj%O(!9@R^_RhdAYt(x?z-iXe8}907CsS5BwjLB*xZKOFnYsOy7q zTa0ASTIeJlSpJ-XSY%#cAua3|$#P+>-^3`=56NYYI`mX72wYvr7WnK<88c=>2TiUh zf;~@MRmWbz_MM7K))$d{XT&AvXYmbcMwn!{YGn+ORTcD1x!VSt4j>^g(QF8kWc#;> z((J|jMMph_5N`5_$v%{*sH~ZUzKGU=zI1R(MkUnOiMC0rM|Ck`A=sXB$nNN3wd`pL z9_2fpxLv3*vnGucEHi+-F5_TJFD%MccTqawXKVBgS3H0dft!p2XTumnAXKZ3_Tn-L zDXrfRzWP){&0Y>JB;)Ls>`cKDT+nb8WmS$a$AUI0vLqKH(OfVYQ~Cpza{6i7ugI)` zp+~oM=yLEqCzNP1lsRi9gz$F?_|G2tgUi36-C)B<#& zpI#yf*m>=+#o3h5k`le`0H$cvo8A~mmP=Ks%LFXHSO?fTf-yEGZK5>n=_ODWc@bTX zvI`Uzc}dPa@VvPFWk5 z_*KIv$Wt;h_PZp(N#AO}djpQ!Q`*(l5CX=isVgBHc{Mc^B-t$<-;b_2B<;8F@v-xe zL8c|B2w8NJPB(3m!m?uA=4lmKusF5z>^Tg1dksnkB~vP4O-Lu;B05WgcDySi^oCgMjrl%+NLMyk&eNQc&_hw&h|3J6qF zx|}W|51mKM5rKpQ{ff-O+7+JAw7&xqWz!3mdXOldgcu)aX)7FVHa=&*t1Ni*h%c=Q zx-au3xZbA-@$*GAEWK$pvWfX?9c16?2n`Ynwiq~^IwhK?-UMNDkCGp9r&#cW1l* zLXEu^xJ+qdnG!PVUEtu5y0b<_UM;0XMgY;4V{z@JwMdv&iZ8*JS_UlAPuCh#U)X0g zrH!HvBjStl(s8Y&%>Nqm#Oc%fn8C$=7zeTv2<3o{Wu^R{oy8!|QYePka`=ER9#(_r zCXP;PeOsORB$`gJTYBKPN$|jPC*SbhM7nWT7J>jJ8TM`UN>?FDn_pPc5?x#ctO75x zfU?ljMf=>9co;qoS7Mm+^BV0>d`)vJ#M%p9^T@Pvj;O$Cx)_cXAsnB<2{_Ena|jsQ z(}EClg~`FY_vaN9Y)>f&Ow1Rh#A)^i8u$0A!*9kDJNwHVBb|!VftCyo)$TcJBtq&A zIzDiFU@lW?dUbpcbR$jAO}#QD@sL0PHGyb*S6C{WUiOHfC?r^6Q0lY~`W*H>QMr)d z+?s+4l5@N6hpxmp1YbK@OFlE&YRrM+U33KlR#CqV@SxYuL!Pka3nLx0hZHmP66CGo zN)oyx^VElqK{_d$qZC>61c{})q4J`qt0)U19C}sxWJHzy(ta2VAZPo!;Oh5F>1XAp zT&ei34oJ`VCo)^KIP?RkVtFJ$*j?h(WGlW!3M$)DA>VMX;O{PQF+uTQSckU z_Mt~yloz)!_)6IqV90oA*R@TRAcUkh*nNH|f~tJ9G=+mLGdQwX{djS^^nDkKF7Ho0 zY?R|9VZg%j2QxM;5dZ^(3yOpwOkowbe-jbvyYkeJiBmVAk<4cDi$4TlPBx73(8x_pZSJnbh_IcAkB`;uFNDB&a882jo8U`-$+3#|3HXoL1 z#p%10qhg%)tdf76H%7+JKCGUL_TR`%4Rp|YMDB8{2|`xy~(M5HeTcuEKAX=g#t8PwQ$}q z!6Kp=l!AJ;)}YLWOXeW}`aVUa-aomyk>6g-er5Q)pPG1Y!QGz1oXWh1)R-rw($^*c zx(gG8YvWYlpM+u(Uq`G0OIrca5v!q|i6Z@w^FDkTbRRu|BB%pykx254LH`ZOmr2{^LqJ~m38h0_vP4+{lvL9iELWMtlcY~e1=7VJ@Z zFQrsL@F;ZQv!>~Lc{=y)&OFJU6Mz!7CNmP&VJWbXFo}lF{)HzY6o!UaY+l@t*BDcz zHW@x0gR;Gb&H|pwa0sE&lTUxd=jSG{b$`Wyt_x-$!tI{7R_AoYnn7K9_uiRl-aE6n z)t;0yWY<_mlJbC4QXC@M#d6h-rObZc%(|qXaOCmU&2zqP2}BgJ*)U=)SxHoe5H)js z>ir(GUoWoJuI;{X&_fp;A57o&xPFn0h>&~UM4jKGoxx1-%`Cq34ET38y>q<1```G0 zBv=rp2{gBsi*(Pg3*5x*c8gP`!WeR|NXwXk374aVnyg zS^`fURxaTsq{?v3B~Xi9gxG*b6qFY7vI)9Kh- zvM%Lxz9w@ZgXuiH3_?>m2X`j$f~g^wK1_$5*s)Cp78@)Z{>gTpmYP-O&%dU9Mr z$oyc$Wj7Md8MHaT=JQc0TJCU;_5Fh@3j{9uwF(qLbXDy4sOzPA!3?nQkKp@tG`pO# zK61u@I3=6J=O}XF?+3_ZcP4(CR9Y8ev;fYrXTobU zqPVSN#`X^t^?vn=pA6m_*`Ad+id7Iiq_h6 zsmjI58@AR*!a=^%VbDUb%!ve1MdmY7g+%<02MLdT=1ziE^T;j8--L>Yzlxj+@${z< zzFMH_j~6AzGbfsc6y83b&!c7)qa839=l&V>lyRZVrU*f%nn&!*oAf&k6h|X*$M}7a z&>&Z12^!(IW0Fs`98va$K)z~|##9N361K1Y##_Iq{JAbW} z*9d1Vwiu3`AC~-cQEJ^`8)Zykse}>|%YM1j6gttz@1-XHV;U81z9tf3@v!7AUlnvh z3{<`lvACxy*T#K@AB^PsL&o3t!Dlo@Ezqb|FZqk@(rmQe9g*CVFjLy1kr05Qjtoix z*fq=NKp+;U`Wi{F&_qy+?a8sD&35)-Kx~{FBkJ1@MC-H}6#2QAq|9#4*Y}L^#V`#- z!g~XR&zSIpKQEpYYfx_DW#zW4%r&=}uFgl9jeo1fLE_j^XWsU}LKly%^}O=1>Jwoc zd_NI%;0iOjN|bdD3ZjZw(bqJ`YF*9%WDN)D*m>@))D|xzOnEU?4@qT1cMcg$*e&y@$$fB#iip)2+{N)DbQMXdlyn?Y)V~PJ^ zz<1oM45u{k;>p2?{#CNKRx3B=sj^LJq2|f z33N}Odjj1P=$=6L1iB~CJ%R2CbWfmr0^Jkno2m_XN5p&^>|f33N}Odjj1P=$=6L1iB~CJ%R2CbWfmr{@?1J@Asqob5HvJbVGAy zAbbMh69}I`_yocy5I%wM34~7|d;;MU2%kXs1i~i}K7sHFgij!R0^t(~pFsHh?*^YN zK==g0ClEe?@Ck%ZAbbMh69}I`_yocy5I%wM34~7|d;;MU2%kXs1i~i}K7sK0-wi%l zf$#~0Pau2(;S&g-K==g0ClEe?@Ck%ZAbbMh69}I`_yocy5I%wM34~7|d;;O~|0ejH z_S8DH0tEn&U;u!B_owb`f8aAYWz?aN8C&x13VrAa=1z_@?}%L?QI;%P(p9Ie8(h6> z$RlB5B^DtV{LUy?A@JhjVvjB4eygx13eBXwNFng#%NO{W6B|v=Q1F-6rUzV-Ah=cY zZv_Z4Q$jt{VviJpYPS<`qQN8(2ujK%;%%%7KbWMW9|D_;;yHLMI56vn5GfsTx^))H zYUBs&K^q1vf}tnK7^8=ehkoW*k3h%=cK^@t+oVp)u;;93<|sa>&P52IaM*BTm_ zmpCt6^gh7D|hc$0wq@NX%&|;zt){%&5&cvFUF! zDrB6OB`q8vT6=396OSieFbZyNauXxMaSV=Bp!t6E5)Y^Qc1rR|`3BW0j=Rv@+f&ce z`Z&hk-qYlh=H}`~Cl_~nWv`J}43Xq5e58=}^Yk`%Ln-@lF2r?@;D>H;PQkMj#71f*du)WPK%j0WjQc# z^!I@a|+_$r<47!bS5dpW-=p(oWEeCoI}84OM_Q6XHuT#t7_cr zoIs#eK}6Y6ExkT1hxiEdkYoL3W>;b`~(=xePo4XPV z1FjlhLymB64f&=DlY5EuD?E}*N3G&Bq#J8clj_T|yM@YZigf5*<7|3FWC}GwLmehARG;A}T%N(n z@C5)clG08Ac|$VHh=t@oa#3TFelH`tIGsSmUI$4*|42dQW4YAq21^zvBmXWc3iV~3 zET@l*><4RLn}Bm@N3&}kNyKjXCKYj&?rq;Qf;GXh4E?QJxgd#FEqCx0Dv;& zzt0lKpDaZu8#opbqlcW|X(6A{g%!r2HQCI{f5*zS<9u>fz(`i_2e5g(y?`oOwUG^f zEX4PkeClGszj|uPhq9u?z%m2r1&DPG$jz)2NM*kcGtSBjxI|$@glg>< zV@If6Oe|9b!@Dsh)lq6%`f+q$WSqXRTD5cq7|s9{3=SPc7E|9xd`WE3@x=!Flz>HI zX#cyFF)y(o`S#G`LL)GP<9hDc(bUhf{j{-M*G`Rgn(hu9tfe^EIM4A+;7ZriGC0U7 z6b)7EcTbX_%c*8%;*9CN`?P)*CX&kqvA=*cdJ)M)!F4C2^-|o*GunLPSaDi_n=OaR z(|8XU%T2|}Zcw)Shk;4W3;mEKK`E;bZqxiV95Nos#Af~$$V@@sd@hm5+DxiA_Q{?| z;;~S@;d}7Hl*FFnva#w{oD!NsHiwYHJL0^+%&SSf4noNdrRzcJeU`GPoVR@xQa znocsGccgnOumAWJkKfJvrKEs=lm=whxKiF|+DPfC7Ed-SwIrDgx9pI}>ALwuic#&r3ZkJcOh$hf;(~*6VLN! zZ8W6wZJP>+4(pw|{S}4b7?s1|*3N;dW-Dm8%LBsgSwhl_eh8!A&tkEY%+F$I9(8l% zpYBQQYoo}>+yauVKFFT9@X*|H7gOto2rgddAZ&>QMlT^y5ES?;75E-d8NNaMeJVR! z@N7T)7iTzJ|2V_Zi!ETr{`0(#Td$pRnF~u?a>nOaGH+Pwb(e~W1Cd>>Xn6C}>jS%K zo5Cx;{WW|0DWBE%v5L-WGML^wk(B>3t~Uc=a>O#TV51Y$2yItdA~Q;}-Gt_Gko;qY zZ#v<(`#1=*k?BCmk}~Cr>~A%z=v5^yeR@9+bFv~cXxGs7w3-8_tMJsd7%$p~s%@6? z3Qb7QoPYB(aoJ69wPkP|QKM89$TAsWTF_0~Wh8%}JQ+lxSDI$asq)>OC|Mixm-UF# zT8LR1;yWNJsT)z<#hL3c!CE8rgQha$S<^V85~9~>Wk_PwwGOfKqG(epWmKSuSDT$Y zds4Qnkd7ycF+GWX;TE6ZOSDy~sQJ{87_s0Vi+rXb1@ntymmX(B;OF_^<3Zgb(NB_%@EMz~oGYv@L02>? zc@EaJs^}*&S})DL6YMJi4PZeA~ zu3aPLP3D^rxzp+7S?#rAklqkW=ewS*`^E3dxTo>Dis#^Hj`?8o!NBsC= z6<(n6u}ob<;S=TUq9lv)zxv^df!uh1j>&pE7g5|s=O`gTw|2fpfK7rac*RVUvVDM& z-`P1Qw$7l&_^0Z;F8{9Wr%O3O+bc^xKq!AeJ;YfSo=5Ov1=z5C0hF@fj5S&7RgFzJ z0sL=X%Hf=^>;@>qUwY7bCwxN0xT4E8WwfARMij^)Kc4wKG@}LeTNuI-!MzZJDFhH5 zo;iohq)Y7?fl7WdDV!Rr{OQ>Jpo_(gpzIQ~Hc0vdPhG4RspBbs&>;wEp*gJJaD+-D+0f)if?frlH;l;HT z0%v8nso;~8*{5RXNE0^>rusAtoiXGhEmGICoAx0WP4kCvMF*BBhZ^yr{*RFBJzNgD z`SeHm7dkfe14Hr)^vjA!)3Bop6HL#&$Ir8%X#5+xg-uURJrdkPKQEd7KFOFFr#1gY zr)2%>-`@Sb4hi$v1`nTb$7I;mFP&_C2a^~AP}t8e`FApG2Rs@aMyozv|Jo}4@-S=;X^Sh#i98;I$7 zCpqb&m4`5mRG?93A0$`IUe$3kpkAL$a1z$fdT(6iMiBPRHx*M|PArBpR$YPf;5Iof zS_p*^t&Z4;cr-l^Bhu;G%avm?OewHxM!qiC(sm>a*$!E~`IouqP!7RS#vjXfPz(R$ z!*J0V{=vHfGn~;WHAJ8;1w%-FPoKWN_7iXU<3V!`m$NTb-`(9g65`u7G^HS7!J=;$ zDtyYOUHAP~F*KBnmq0eLOS_aLDnh~`=>?HNqL))2melocyOJRDj{`Z%>8h24H)~eDZMEv69h0}^c!~$NhMAE2 zcwOQXY2E?Yo@4aHkg@4nVHX2qsHV>>SCA~-X*weE+?ManE;=2dug_{g6C_Km%L(=y z*JgaTJc#kvx*nz$cn^e0(BFL_2N(;FCi9KT)mh=Z!w(k>t44A^ zZ?qmy%v7^IOI6E%@KY~P%zx-k@p?^>c?0`#94pAV5SY6*8i8OsAPXv+xYx^C=a_X3ET z=o*q(^eY^V$x<+-9Pzdf1y#HOGv0>ff*nMWv!o)EXLdUzeA#zwZ{>#~`UZL*$JIwSBM2=ZIumNHxR1O6(;rB;AkzKFYDso)%X^Cwi-G3$2;ZH(4{zpPH|4Qhn zDUsvS>#ZTO%nXkeRODUbVVGappM>Jv)J{Gg{7{3racx#wbK-5eTV`k808qLNjKTGN zGqVI?@fvFvjpDLQLEec?=L&-v7=tzKVQC0SvmUb}rW9mrhTY5p4N>dr0EvaSgarge z!*;H+E}CP0rs7P~`jbh~_e|pTVgKoo((gS|QWY;Od=s6Y8{#|$9k5UNMGw-f@Z zKnwp7P@`|1DOJa{*Gq#%yyuG5svX|SHF~x`16jOn4_$vhGbX-5{XL+oVT-l@RSEs; z_eg;O{eK~#*m(1LfSkqO@6nE*Lp6b9|Eq-l6a4?@i;e%SCeHwD^8Y`Yyu!Xd z|HRsR32m|K?7$LFaGO}_-WFisEg-V>ESam2^1aj9IHOBe7P5MpLnw9qRI31XgSEPP-L(sY~6tP(ImW6TY)iuZfS zxCFqwsJ)f?UKGuK!gZnH<^0DQihYIq-u5|>(ca_f1iSvP%i)0mYplO6hgse)huI*) z{CE_)j&oWZ1>q z;lF6||3Lr0T@EMx6Kt~06UHr(76@oWA*pRDCtjcIQIO^w(K=LQa|H$3iv~rk}O`2e8q{`X|_A`~e#SbiBX7=EFPKbaT6W z(B(WdP7Bk+=bs$q$h^8~wv6VL2Cpoe=}nbbl4Z}dEUsa@at0q&yFOGf`~x-ylbv$S zsaOj!>3@OE*AdlVgd@YcNC)JNFNt+o4sUBCE=PrMI9fETb zcQ_Y1B$bh65Yg@?R%`fZZ` zkB+whifiq*MR9j`*FbQmacf+I1$PSx7F>h7ySqCC3+_(v;ND2^;BuSnbMO2Aed<-g zf};3r>X>7Gy{?ehN5ULjAWCo$GNBGp%#3Q)AqY&W&S$wLLb-qOZ^|-Hae^!0_uvZn z2o_A#fB`9OQvjI0(Xu*ymF)>pKmQDbG^$o>u|TK;ICVQELG>?zacI_m{D5$}txF2< zbP#BA?prTiN{n)Tmwc@0R;cXg;Pzhyah(Q;6zi`??IvV-uhj4I#P2@(1U28dHvrGq>8$PCMs~SBfm?+)t!*D{Ms(Fi z%591*Uwqt;5+&ONF@~M4r>m%4;iw2H5c#0|KfmsY_&)Dtyt=(!%ubQ}Jf13g$gX=` zy?pQTzS}z3czIdNczyJ(er@o%nkbn%^LgA_V+F=CGYt#QWvx(Qv)<`TBd#`s>5#>|Wp5neR#*xro=(pS6SMsTg11$HUv0 zvyJETy?jGo_v83NugBx|9AD2XyDe67^16$yA#&fBmB*L9jhE-e+b(Y(p3948)-&O= zQ{RWfgAI`%_b)5tKI?TQ8&?xUw<12b`-Zb!tNI2%+#jF68~%Fvp0n|KHG8~L@q~rBst?Tvj-q-i;;O^z1>-E~b>(jt2t?%Rc zSPARv<3x>!$l4h1#?!;8NY~S=i0{kRmGZ`qGS+IApZU%M8J}MsUSpG-x}9;s=_CUDMQ0Mb4`C`c20!}Z)FWpKDP7k!CdO0a6iS1BW@$wQ? zDD22ueFMZsI!=tBZA`@bp!PwW1~%deR!h5AYbhl$y|iv2uLzx%$Bd9^iGmVxfmEi)#5@Lc(V1Ndu)d;BzM@yNO>@4z#!wD$!Eaz6 zm1(hUOxs?-frJtAbF+<8^!Q5WxCnE4+!UkVpSQD=!I;yEk8T7ni9zt|L7?$-_==>cb`tK8;JZc zh}z_CTVt8pwfddtjM&#}hO3L2&L+(uxMBV>;C(w?9`SdqynXh?;6Q~zJy(i5=wINk zOc7a}=|OCj%O8S1o$D&x=;n;@$L}kC)WP~RtPy*7D3ik>+$41G&Wpq+K(@L6&e`N& zza|!_Rl&jUeMAt3ICbZT?(8Bj<6VQ;fS)0J4riB6r6L>mOTp!@o!&Dabc*C2T{@Sk z&-G6^W4#l=*o_@dBH|poQNAYG3oh{DiQ98b@USar`(jA$w;6Fo$m;IsWeR^sJ^uTO zz$Cm;#PpHKyw+D1i_EWju3K1jvhS=4^|wR+h((L*9THxhnuhV+ z$}gF7e+NxL2v?F%Z7p(@G2M3@5B=Af?EzcA0>0nGC)4`=D2R+A$Ubv@@as>4iK@*?||5u*SAJBID{Z9MU+!Dh~ME(q659kd`RXD=RT#8~Xc?1QRx<~bp0DNL09KyY>&gyRP57RdqiO^$k z6HTtS7`Or@5fZ+RE83mMMrPWjQ(d>BF&Hue*(<$^$Sx|3E|3*=>pQ2Rvwy2&M01Dp zGsEda*6>eLNXg>J^vY+m?wqAW^X>d0F}dz2WLFyom|CnK33V zuY0O$ogBwk2lL>au$#e1wHynz&T<+G+hWMyOtAMs^EGqSzPQZ3{)j1pQ@4DB1Zrw| zW*_b&@v|>@8=ZyX~z=x)S+D=62#vRZB+DJbm zXt3S5>H@gg*;Fm?4IIz>H|63;LU8sNb%fzMFsxBXOIeD>%X<9f0$amKV_o_9x_`Lz zWVGB`%rpe`^mRlLHlfwej1*5kbMi!bit@^fjT`SV{{dRU%3B&yF#%%uhjoH1C z3g+fKm>IuQay80VsVjsJaKa~3Ku6=Il*JOKXUaB#?C3JqE+C!p7ncsj7VIYfyy6eL zu_0Qj@l?EjPa0iyHG2YN8`B@eqzm4mJQ|D)@6vE}oN-H+Xf??F18h!_ zVPv9RLvAGcj{c5j(o%Z;#NIJiHEw%q5BEH{Xo%ROwyW)4htOYj_tGYN3EIb{0KUVb zf$wJr$;N0Bfoxw+O4KKu%okUB`iyckg(`hfA}5OHTCRP-PXc9c!H^@})|e*uuc(O} z2SqQC=hD?(CfJzb4A`yb_v2Zvw9@I92lm*+ ztDdK)kk8|=*PQTeSjU!=1vzyzGyY-Eo+Gm*WC)MWdU)KV<>FY5R(|-)_6YKqQ=t)h=8-*OZcWUR5D2^uPbcQ(PQ>{c} zwr}kza(Vr7Ta>9^IUTDuO&md7>F*T}TV=!;?x5_K@f#aWqRc|xYM$=R6udL=JiuGzyH2M%c) z{!xTQ;`C1Ebpes)H>tE=Ilcv>Ur*2-cd0claoT??kB5jIbG*26!KE zivSVS53yPtz3EHvWd3#NfSOU@#yj`RA4uPZRurZW_;<|e>2S{cB$Kl~;IBXYvLFkK zEa3q=)D;yg_dIb%97+=8In^hY%s*dXgFD(rLMjz@9|hylpFDDg;q|}fD3T{f!c?XQ zU2s!lD%6L6FO3rXg#aViD0a!*_yy(HctPxkdKoOK=6aRGR$o6(h!Q0tLM8o~a2S&5 z%3oc$fJ1WfO$E@<3z<=t56`nUhuU{XoyBmDW`>^ zsAQg&KA5QANMeUMhVMv@gRdk6VB{CIeRX;(eVA@>ch3 zNvA93Z|M`PgveT%@r@QVK1Hj@yv?e0Cr&cN{up2tpg7iR6OI8!3A&sGxh@*hlx|wo zxA8ajAQ9SL50LnfKl5^JXyjK$?dF^DDv6eIDb6_?jR0?K?!MJpRAz=Pz2|XrxQUJw zO@&U^*6cyuWb@==C)4_Bv@JW9jDDRK>lwU}bkCp{NR?j3CKJ5on%^U5AnE9>|ux;cT>B7Jvr7wC>Ii!Ghh$ z)XfnDw?d)|oyq*AxqARg3;CaH8xGmT1kD~gUp-#%)gfCMo-Q-e<^kmo%}XvC;iyo@ zA<~2s4lz?W9G}jnoqq&uHC~I(rVxaQf1F=&Qs0Z%LZf3$@qpBB`0_7_;54Y?V@wPh~B@ zZ_7dA(r-@9v6;E-S+LOhu6MYqp^(7Z8&29goLw1kg_g|~jLJnm=&=k0WtdNA7XY}}ZjfUA*fJcy!*?3|7#I}LfBi}Iz9)~g( zed)gTut>3)M(}5Rbc|0YIbf^ZqYg$7WqNEcHB>P6*Xwk4fuC-%E7xEgdijI(!`x z<%_|q22q8~{LEthU&?kaA$ez4cbe$wOz(5%%zq>oe>T~o;PUAy{bR4XJ)$z+zbp}Q z&hK19G#?XLIY`EB8&LtC?x*+-<_#>Pd?gfgwG>QsP!Rw(`O?fk0d^9vMSIi6KLd6h znqrVW7l2!<9Gz6sTL74^1mq#=>#_1FhV~IKb_Oku`F!z{mlY;!b)WRu-EW1VEoXW9^aGqA zIywBtBuiRAxhyqKkD06CzZ)u>a0(f@JIne#_{=DLZLPyFSikmGJmJZkqU4z}n-vq5Bc^srb8LC>Kh0$mpAR7eixPm$mN0(&1bj zZK&W_#Mb!9FHaFS8`_p5H?e*QrT#s&B+M$ty7ZzRq^pUaN@?j}7evd1%RDs?X`F{J zEC&mE&&-5NrV|!4TWy3GWU3I3ii{kHU!)U5u!ME?)5G!KzMKm;mk)Iea8MDl!DMav zYDA?hb$V78g&b2F0<}k_g9f1|$F&B*V!O;OLM7(vq5638s_=U(mS8q!G(IAxvL8q! zt7b~f{TeFBf3NLdNJYVdt8y7OP}ED!;_%=NPiLb%E$bC8lUClmvVDR(;eg)?$!u!w zC{-*O2X#5u+@BNiQMQepQ*viHbZhyCyi#yQ1r8)_c`cGsQ;lvnr)W{w zS)XXj2{wm*2c+(pt;y~}YclP37ILcwD!g3fD^j7RAmr{b2_^3Rrc$H~1h`fdkOr+? zs%cjFHAtgFVbLFvo^y!oT%$o%KECcwcJR395{i4u7M_}W%KbpS@Sd}?^HuaIW_w<&AwzFUIr}cp6zB|0N@KNv79NA>*bQ_IShcmyCvI1 zY5{`M0f_}fayT|__Yg)=JWZ46W5ZQrItp`bfT_tywlWREu1QUc!qwhE6hsbsZP;%l z81-_cssrMhW#XEHpPSGW3U%mQ*gedKlglB$(AWH7+T?(fIaUr}3`s*$mQAg5Ul6ox zxDTqb)I)aM%h@O>s1A zWaAAqq_uTvwP8Y)p@YY9oa2R1l?Vh@Z*L)m}TR`T;L9AxmWwBcNt#LFoRGqtGzjOv)0B91kP`SN(` z?<^NhP6HMVe}a5G(=RF9X{F=O3KZw;iFMNMD|9y8l7foR|eOWb6 zRDB_eQU#?y`L~~+d<$mMkPFRmha(5(B)MtBMG+NcMO+_N*p|(5Yd4RY&&CE*7eVk0 z8c-h>N6NCSLamHx=W*jh>CEs90QVhN2d4>z9@)VtR;s<90AWPTsz8l`!|@YeMiuMz z%I$Y{^pdc+-O8`_{pWzmBtxn`2>9KqBj&6xXdgbh@4gr2Xn@9U&7M-r6QFKA3LI0l zNh)GpD|KP%<%o9)t696V#=*(Kzl1-1F#_P(FqK5u$p*iF995&+>7ZS*@hVfvw#l?Zpp(|W zDm9`W6oyZBy@AHk!i{YyZ}959N@>0X>XA%T*V9%C2cQRzLE89T37kb$$$NeB3U3q! zi^17N{n9=F%;&sgnqq=5v4g(Ugq=^0Np>IkZ>bTgqvBd?kmxUY z40}#|28oste~@wRR@aRujYaqdpQ&7*3;#G48PxlMpt8@dciI8}GY(4DCtcXWYNXM` zhPe;!g)FDzq^VXvIodOM1)NsnGVxQ25q{*sQ>5Jp#$r#$DHtiEG-hApEPxNwxhZUiuQ}%hkad3O9&-h*|mKpqLKUS z@5Gxr`t)ITgM=e-0*XRvNPKD@!p9&4*NUS2)hdN|FIqI&w3XiWHsA2+(`GV-4KN-B za&zZvBVutIe--39j7q}OWh+gy1^@Dn!ookOMd3!Y%4{}VIW&RbjSm)Zyvh{CvGy6< z*KyFpSP$V~-l`m*#vS`5hfn>QPKORSb`h1Rpj->*22YmfFm($4Ad!uAI~f7XjKm)s zoO(AkC)f_7xPxFafcLl#qZE4PuCT%hq{uo5(ib~9J+VGG1iON5w+Kv_-u7}&|Ac&I zojc2K^#qgTs}NW7-~^}dsd-Yu!z0CC^F-n2nuv~a8`?BI=eU(@k8S1x-2f{x=XrEu z5km)l@&-mx_XhF%#69%(`n`5doztJ4)K1K!ICxCl5-%Pg6|2y!Z2suvZM7QDk*tcp=kmB?T(NR%_OT?MIMfvF$ncX}@arFV zr?aCva;M~CMCICc)&Vb_4)O_uBw(5UB&21g(lHZM9_u`-s<$c3Z49kbG*qd>!i-8E zdSSErtG1W+cv+SBcHf_-fsP`U+j1fZS*$5ahGM725uQ$uPH^0lVy-QBE%&s~`!Ia5 z!7N~6G5!kGt^6YrosGf1j>Q#n`i7+{b3IsJwz8b2(gRhi8Sp>H)BTtiGWYOUpyqkV zbRu0@oKZ)lb{pM0L?j{G+$t@taf#b0-@)ipg)n$_rf(Q&9C(+M5d zt(h+?ZrAKSMD1{Y*rS;iu1zlNtbiHuT0C~QiQVEWt(gX>Zgft+!#*C=@iencv6SiZ zKYr(s(5&P&p=s@`3Ue@+?InP}u7hKmaEcUqAaFge7Tc44HzalLf4-zZ1j2H%iCAQ$ zzU%KiM|)p!QA8OQvGb{yxbIyC1vSQjpTL4>Gj4@xlnm)kl@w{}m;2c|n94vNh7(Di zvs$r{KOImK$wiP8M;#;Agq9(E>$xRD=EdhEpWwm)d(IxFr-eqStrT* zS&@NcmwtK+qV6ooV?;4}gSgi1aJa>E4OA=G8C`PFI~-Nbax1d~E;c_{r0v7;TL}(8 z#M#UzvW&$0#0oS!pWrjJH)XEY0TdbBkjs_)$843?jZrc@JI{ioscyp62GKpZ(VAN1 zehI&o%Yi$?rF@_zY0WuD&@*W3++-3a6Ogj}5NSWmls(U`^z~ywB*Ap(fwgJj@^Psi?CPcFykYemaUV;L=KyUz-A{)qb zs!}%{S(;{YgppVF#F6uF!~~rly$p|zn1XkMqiVCHn#!Q^1g*Bp`K3Q+pap!SoO!;+ zcz$|WdGAaPih%^3=)iY-k3I#`NJ18-k?O^%#9+ zjs&731jxNpLOuZ~TUU6WQv~Mqyd=q4W0OFCMD|bc_kasiTeT@l>C}Gm>X={{vgfP4 z$KkA$shz|qMwcG-Ktc5Y)&RfqTNIKDOZ~I%sS8Wvn93I2=9p{XGQ5G~Iu-|jR<|^1 zobzc&s46U5I6Sdeb}ct-Z8MSJ27J?BH}t%*W$?IC&bZ3lG`z$F*|W_To{WJzhYoV< zjF7JlO5PW|DWuBhI$RyP!-7ai!u33gObrUl&#KsmLA0anqp2oJJ#;%G&amlbUl{6BCUd=G ztP!H=sJiq>U@RKIySaY|vkFnd7scL7LpIDA{r2tp)Ac7Xvz~q6Dls8vL7hzPUTvHY zJBjsAsTZL3_jEW@lZiqiJ4N5EIefqgTB|B{Q}LE0VFF(dQ|oRZ{dFcOy^QPtBe6O_ z5PNyH*e27JJLy3Tcq1>SyRGxtbMwH5qMlr5hC14_Fj2S518nJ=A80f(KR8pfW;e&% zDnqbd$|n;sx^2fP+>cPFT^GoBe&3dg{=iVAL8uabB4?p|ySzh9q(X!Rt&Wii=EH@~ zooX=HF)HV(MNG9uRM_{B40N zSB>mv5_FL3njZ*7j`gaxdd0YHc!c@d()ZPB;6sw88ok`Rk$ULXUd~>ot?U*FWW_B{ zVhAba@$@A`@ZFJm$P5(2OzGRe)e5HLx7Ivqai*u~!K2lKguSa5v_ zqKkd#ueGg^F+0<_9fzmh+e%6kJ55rzM+E& zjWuq1S!=A-0T2}5?I|gk?fTVBYOjinP`AxNG(dV!4$tjTur+q{(cKK+&HRfAzM10s zwGk!yP3M?+|GPwSrDC&iR&Hpg7U2vc{6KT|LP|1z$ctqNd*ykC^C1}?6n!s(@k*^jO**oJM04{D%N~V zU}x~FZPcRm2gL(qZ7r>7p+g|PbB7XtKD*R`gL!L zbVJLQ4oGk$SPhu_5`$Tsip<&CjU0@;CZEPw$=)js6UJpZS(I0&i}!PlL&g0LdIoHX ztNHDDxK%78Fp7nvKsgTHRl_)zJQ-6GSL7@OXP%h>NiFO(g=;c##$n31_s}E36;Vy_ zK4|r!c18o%!dzPbDWYN2>>Gm-KN#+yp$|jmbHefs*xy+r3N#-h+zhSgxYW&9JBZ>$ zMe?i7DP7(}X9X`Ju0V^xrt{`HTS?(ld(aJg2~vy64w_ntQbp}ciP7uHX*fK3rC2sR z2eB$6k?GcmGZta!td3?exqq_ojPl3y&YQQUgED z1lmB4Pv{E;?{14^zqto$DQguf!eqi9x!+edX9{>UOAMd)vL_}KX6!64dt^EKv!c3> z2w?#<1?JicN@k7Y)CCSKCLx-7M>EQGq~I&m6gb(j8SUDT^F68lN^}AlAd~!xFnc-= z^gNt}Sl#5ANqNKynfE*I^htH%dPRh=cTr4fK(eG-G$7k{u0EUGgs@4vwp5T!en!~3 z1Va#aVUW3-S&5Z#nhf$zU^-eA-+TBm7)5a3E^HgW7Z>rNQPV;j)tO^vSrQ*xU(uO! z#uK5aHdwL-MUr)W=@(eZl~f*ixXS*BR9^(4u}nT}Rs$C;7w0;d|22jpgsTQc0}n6Y z7qkex4i{@}e#y^k;I!f+^gSi=5O{kHE2+lZ(<73W*!l|2+%prBXOuX7=^>3(B$D*i z_w#Qn<$u~MRTv>&9)urs@qmFLWIrfQHqZO-6Txy9n(t{pDl}HE7MMHWHb+-lWi<+yo*ZzQm<|r$x)n&&56Zfz^-2!Ca=q8{ufO3Tc25e2)cy1imgxjrqMf) z0N!g*RtyekdQ_M&UP3MZEuqwtfo#g#JL;k$>9@e~aKI~d8Ge%7F!W2D(AD3o9s!eo z)|5UABY6O~M#b2Se>8y;<+Rk+sH#Z3<0E@-H_YUCM|^?WazypPhh*?Aq5f~fVOqnB zmSk0XuuKEsoHP@2LkbwvtiR-9%@@Xq|KSK|RyV6zrS`mFcOcbi{i38F1EbSTPH@oE zkBi8`)i*}_1SgONvdfPMtEOv90|^DHX-NFij80Fm`q&EQ!>-HGszA22U@xW|mPA6} zh#OV>G$#Jvevf1eTp6u%PWxv>f|X#snx5NCzKk>#t_nPj2L8IIhg87eJkIxj>$N4g zzlx;)JsCFGb)zdnKN``d!2Tjb?2NaOoK|cGm1@pGb;K6sv+wuPunhmAYNcSp2sMlk z*6PsA2j-eu8|<2>w{H$J(Rl7@uPFR;N(pf81>BlBL-mamr$QpOJ-A5)kbMbqi*s1$ z8D<&&%6ucLi9Z9a>4V>vqD;SVsGTvX$Z%iO5|$MCc|S|&)6pzn=r);JuOmQRG_iKI zAn+nM9xMT^rWnj%f~{~EZ{G~L)S&-%9)bm`!a&^Izze%tu)4IR5G9!Kl|G)vZSp6? zBI-fxgu@`(8rvA-{Y%c;61v*{@K~XEnr@A3lM~0oa{HLo{;KXA?MP~QyX`12Wj)?I zhE&J8)Q{a+pRvey`yz!wr|AJ^EJGtT6)L-^t?W`=le-5m$!O+qIF#&_5L-b~JvZIT z4hIbug7uDD`4~c2SR>HHsog;zs)5tFf>#ba|Lr7+y+rZtNPtN7-RfRySBKvN(Upl< zNo0hR>g&~^hx%4xx4?XK4{5e`iUYhbWcHp2K)kA}n8}Ah5gT*4{11`Kp)wa#UF4PK z9|PxVpq!bMGVIZxR%s=l-?5cN_-e<%Z1hJUw1oymJEpYNAp^7sT47%h6xs1DzKXR& ze2y|39Zl3gLtF}_+A3K~*m{>D1tek;CE6+Z4wmDv6cIF2=YW-+bWE9K02#{{seoi2 z*p^EiNeXl~`8zQ5{e|a2fz4_|QMNn+Mw-=*p}1Aw#G-Xo(HSjaZ4B}=amn@~Q?|f~ z-0Evd6ndb1caMxvi(l0WJ#J^Da9ZcYZpMP7m{*R&6PA`w{dlysC<=6Cs?F^J*$2tE?v$y7XxdEiTfmwp$l60=?q18$xCuab#A@AS z_ysb}Y%Y;T`Fm3|+RP2l~K(SR?sa`~pW+RF2`C60-)TC$u5xEhrmE0&2JKC{oETB;i(| zv3Db4P=}Gh_dX1^3iv3_+`fvAuBU;KC;4p(E0hnI#{X{IJz zFC$Jx{}V+aROC2W;j08+3T%oh30B;TIod?C1oGNJiG-Om(v<9H-S1>oKxMbxzyOZI zg(QlHbKVlLPb*_IfoI8fDv=)#F_m&P3Se1IY1V+RwXlg^gJUq?ac1*P6`p+o+ z&6D4lh(+T|^bV8@kkix#E?S;j52tgtt#Ohu`ek7mcsko+U#HkEm>oHiPn!&^Z z9M!fzZi+~hS|PP4k)9Ldc$%RUzXQi&A0DzM@Uvv8`f?;YRnf=|Wmr?BJj;beYnuMH z^{n`6~LfSGAL7gi#S z-IhOq3`pQgCFb7NP$k<*WZWNO_8GQ>BEDd_`^yapKppxWIAJi}QIgg|>4&wOg6(lL z0u4cvqpV`_jd<`%fOginYAJ=)t@qc8mP4t; zQVP%mSnp$ZwfqD7Z{KS4X{pN!pfWZlg*ZWKF5*3?boeJUQA zI*}4KCHE@;9OPYn;99|EqXN(i22S3!J@bL+i*1yY@=X|0f&W0Bsg=wr_RX}RH@+h- zGzDE`iPEJ8>PM(dCRl8JEJtM8%V#CkyT*cPCH96=N9oard6-j=Bo51RoblMLr(rbY z&M4@LZtMGn=d^8_tT_7J#^DD>*p74IUcloDQF0Sjumj2%Gz!;l%T;AZO(fR)2~hO8 zj>2Mi`NQk&wZ~;znSJF6!=~CdrOuw7jmv{QKCe*Y6LG)}w{M`5sTvMx{DG86S&gy) zMvj;Hzx|V#uk11^88DSAUOzCi7jRo)d_lNTDh1xZXiJ7M9Q-Dmrhz-7+~s5*wl zegL?F`?fGY+_^w5!0Cz$FJ;VYo0#08jeWGLlNdo(_?tN;#+(2!9z6J=bW%!Pz2YJ5 zXp$o|d^M>wd=(vKwPgEq14+m|nK4_DKw;(}i=;`M9P7enI2&S!s%q=pk|q`abwa|kc43*(mqn+lWT|4B1{*bs^Law zXKc%8x8h}1yDV;i8;A_0=Jke7=5hv?DrVSBK^FT9m8*M{d(~)+uH5$KxO>$%46cB5 zFzYmyv8wJ6&tIlc0=K2-MtLBIb}R#V-74UVbz)93T-dh7J`|YQp`?C4JTW|D@AsI+ z+B*9yp_dUr3duL|B7D_Bs4|UQ=I#QVs1W1dHV1*@(3h7g*=Yu3m4 z!=!7gDsd1EK{VK?uS_ghTlfaV1HH|9c)-;xb+}{}&W|=XGv2SHYPU8j(c0>75Cu_k z4VuY?lJSH08!PtR?_2fgMHep(m%t%GL8NR)>`x#jD<0K}sU8_z%^TsGhBp_(;Nt?k zs1W;T^1BS|8go(?&`j?8BRep`w!7^D6cUR{HNe;?e_qG@0p-$4k@VjZ%v2YDHZpl$LoC(s@0C{=9p|@-f??T0s`uCz+>idpUk8kp|i4Qli&T z-xO2`?G`BVqVmBx3I`VZWn3?$5v7-3ynCaJc9v!${oJTZfj5a~!+6=%?NP zX9n!bL%O@2a85UX){G^k!{z#!ei#vo;fg=fN}(Pim!EOj-Lc3`;&Xog6n}3(gKvT> zT{dtXn2-Iqk2z9iYAgQNNC3LP%zBy4!s(X=TO-2b$V04$zhV9>Uw5R@6&45{q zK3f$EEYz9{E4NA!SE~Wyy(QT)$xxYP0AEtZtZRp2ssU)q?xA5?e?4Odf9-D+^En@W zS*4@d=5A@wl}i5d3=(>-$o?H_^y!0E%#{WI+?^y6^vsYZ;|1tM2LG zjyocX(HP>e4bj9hkOooTb+@pLLWq5{$sV=&Ullb?FL&%kWtdtBwkc z?Ji^%0qIaxvWXJ~jLiSJ2<0dXB7^-@H>SDfDjJwa(p;Sm*Tg@5!-M%f6#nD}ZJE6` z`Tt$G2XRwW3qkt|!|3%@f)~=((UV}xR@I7RW23w|kKe%Bn~SXfH{-GA=LK%#s9M~g zbStY`l(k4O>HaLl4}e*&c6Fr{p)F6=v-sC4Qrr|NLeMfAIxK?cPT(gPZCWiOi}lG} z;?#J>6<{rlqg4ERl?A_tKPV)|=6{a6r@&9-D!?b&%|CW#IGru~7MnG|>He=alyN&! zOR!1`nfU72CBjEJ!^oa zG5(753{%l_8%k>GK%{45on_{sTUMKtSMFOX7U2E z#$rLVG5|~0X`urwUGQ^wH0Y|ulMa}rLj&rh<>=JopLybns(I}t;ETsd zBNVFoWhV9jPaV~TkLaeOtmvi(T(n9&sYB5K#-?kcbJc+$8HekfxgJ z7Vgwv&V})Pcwqdk##(8Z9@Bsfq561OT4pS>nT?5W%teODKcox!TFg;WUZLn%3T;Tt}QTb)52{wLQ%x=0A7~5SLQ! zz+oJr25wPJx0>kjfheWk!xn|Rx;4W4APJmR9ZiqEAmOiZy>*IE+ah4rjDg@+<^E4E zQF*%T`r{zh)!(?0+aB_b3~N6F2&NXec99U$a2sC)j zcO%p%e^r)l42%_$gfZ0nq(Mq%*q^<`J1N+p{}ICniPQ4+?*^0^+4BQE#DT@~Vs6_9 zZ!lFE16-UG*3W`VKSruMzraBH1+7bUE`hIp@X^#9@8TlqE<{Bxnx#WnPG%CfDHUXG z85w9a@6z5P=UJ>|%673*97rujawr2#`{$D%f?I^Cxz_JNrTr_y@L$1KOdKfUEo>ed zD2`Sz8V4CaX1*0z50phLutm}MeSgDeVjIpM>au5IYH;}U{WIl7x<0F^p281}rC zwbpRh9bh&1t%*C}bL2rZJ%;N8qJK+!tloiW_0IRV{#r|f-hqnX_U%XVw+RMG?=FAR z+XBSMOal*tUdD%0>_r#u=6}3>WO7%Yyfd>+4A|sIh_vbfCn7e|G&7prj~TE0W23x?SF<3o>Mt)v>T9teyv0!n#6rGfL~J`Mki8^YQl zvC`1%t9T2kQjtzHW`}*Y*l?7gqC4X!yp|SSIv}(BhA8 z)!bjkwY?Q^)A%b;4B??sH$&*q5%GUV$Jow&QMzqNbFc_%!#m+LvyXB@v%`Puwa6iF zXa5iZo>!*iA~z*&lP<%qkMV%{(h+!l7Kg%+4;^UV!KfC7_aJ5b1%da>9hD`eCANHD z^urEJ8(1dn_#YoRCh+E`0sL5`YUn8a9lrm-rBq1I?2nMn&Yk30nTWPY4W8>BS@&j4!l>*FM&VF~eX3SFP~S`5PcHM~ zFZ(BhD27T2o_sm)krB-Z5zIPP-zt=Is#yP|z;E_>-@*(OuY;}9J+abayC_b1GiIsA&hgkP4{TC}Ynyyt$v*@sabWnFIn0J1O+Ui9Msdip z%+xj(QL}8|F!5}@sN#xf9ZCpv0*KmxVZFYj#5f9nCGSj+WDT7MwT4=?G$rss2)nUe zJ80|t2JlPY!0Z2>w@n_HF)muWlWqb&Z+nDQf{|2vgg+Xc2lotdzE0M`yNF0L@lI`Q zte=CelJW5tZK^*%g=;xy;2Spl*iL^OtIo5>j_2i^K5?6M1Dmfs^(CG5jJ;{@|0sd7 zeMq`ZkM1}2v(HScCHT;chBN=YI^~4ujp^DOW$cO>gi~qd%0B%++qTsnM8L)_45ocM z?)<-wJ5ot#74jtg`R{QpVE-QPuv z@cwhrPHu2V0#U3!=>j~?+TcGn3)e1B_S4k~gZ2K<_V2126N=u8;_%o%(m|**2b8Q# z4Ag0{ec1Fg0Cabk&+7K&9rsKc>nZ;sC^dH{fTV=R`f9=DC1g6-RqRd6JMjLBI_M;q z?C?^XY+v%b#L?F&$=LNPq?y~jMZ21o{g-;%(%V4E4H7OzrA%w|r#a6SEZGpv#@p2V znjo;POP9(d8wZTnQ3s~k$2zBIg?6%b{eNaf^RVw0TpilkDs#Qpmn+zqzczs5{r<06 z-G0_s&~wu|2GTXM;cY(Bzx%|(4481vDO~IjfAqfRUSXt~3vZp^S_Q6^ zPr37-_ls|A|F4@lY+*hCQ#_NA&Tq%!8aXiw+y8qk%iCDl1l!}cu`K^J7IY;a%vmhM zanW5`9Gen7+3BQ?G-*f)+x3+G8+UQl@GI({{8*gKZ$C$9alq;2mMSg){2$H#*oQ=8 zEbI6%jM8-)=+^KKWg!jWr@p~|j&(`-XKY`1>e^WMetMkh@_yJ)xfMC(i667S*qQQu zJ&f^vecHNZeQoXfAx$x`7(8~Xz>zl(YX_`IT zwl!_rwr$(Sw9RREPusTL)3$AU+P1yteZSp(?%ikaf0b3gjEsy7Nk@8ulob(vg`}X1e z3ssMGB4Jo*s-oytwUOQmWQi4AE>3kJg;kKPg$!O9@R_N2mla0* z1}-mc|9}bzDKzUZjd?7H+9D$kBfa^$%-JN?3-Ui?@Ey=CZNrjosNE^7ilL>EH;ZI_ zr;1NLD}M}5Pg50BC!Iv2_`Gn4?l*x`Wf>>A{7%RzQdL2cE|DB7 zBGkMx!*QvK)Cg5(rf-c?VGd51#|LjPpGW%5Q6_ZbJ)eO?#v%ovo@5)JAVSpzjzjJL|CUm`Rj-{}OGkmN&J z{X>j(8RrIic8m^b!svDiA79PEsahAXT8WD(w23 zj#F@pQLwAp8RzpmA@^UV^ChW@vNN-N(;?_ulmnH7(QOC|lPdN;Fl>=htTI$lk2z|i ze_q(yl_|j_CRxJ>Lz}Mok0+tNgUFyXT|m@IvdTM9KcT9}+7!&y)4>kRvx)}wY4yQt z%aCQw>pWnBU8TSzJacyKFAyCjaXqk(PZ)|7kWk!gA2i~{TZJd!sZk3Z-?!VAlZt>Z zlsS!gmyOrpF}8usd~*K2OOFvXF>&NS5Qz;30@l?vby^t|ofnVaJ4y(D(dz=w!t+yu z4pDR4jeFC{{VkvtBG$$9#_8guIT$XX5g;zPJ)J%ZUfE#3jG}~KvmT{EhNiyuTg2L# zsFK#Ygp1b6_9wZa5!5_dy>0Ng=A5YX#AHK@;^RJ}-Do9BWQFY*(PMkuQgV3#@d$)! z^I~=QXkl|uEirC|qdWxi%Q$#N30{ewr<$-*vM*Or3}(Z!eP!8by}airyPl;RD0$1F zEGRzM&s-oi1B8X)PU`%u@nt;ajmj71SW(yEF(r9!G>idNFy;gKJfaLFjY;B~F#oTg zxd>{72n+IE)cKR+`ahIWCw0aWL|>^*#0m_5?NxrUDS_4Mn)l-pJ1!d=fvNxImgRY@ z2v(-{c9oQ?+mkWDnIac59<<9k0WB3pgBpfH#6z8ve2~7mT)?PM|54KAr6%0MGhzNi z1-I&DK30^?;60ua%SH32y66G=VEOOtVj>h+h=w7fD7BUtHX1(&wPZy_vzC~0=%Z>E zsRAvT(h;2UKlA~_#CFk>76K)6miJQ_Rqd`-TDPzjh#jsz7KquL_S^ZEN!N}El$I|0 zsEqQFri$T&vg#_LBuy}{Cj%8~BUY9A$%@Kml`(_%7KOFVN?pWCibB#Fl+^7OHPxu0 z9Mlz#d0l^FiL|S5T(j8E)S$)G*w5154EQPNq*Q5QB42FG8b@et3VzbNQSqP8vAmqf zHEJPNV*Dtj1I&*onU1?;TElTis({?C94x~L8zeG1)ZvPr!2QaHRxhhw?oM+wjDh2 zcHn8McoSqif7(`a7=jJX9dMg8@;v%Kw-pQ{Rt5l3$eS6&4BnbAsooFJp!{%`3NRr~ z!L1So_NfiAHVxK0QOM^$DX{}+8bML2T^q4qDu4QRa)ZXV~)A4bH^@v zF^iGIi6jLfE4Oi95e!1J;Vj0!5Lm00!kg0-5I4MEbqIryZCS+;qNhOZI1S@hqOOEE zqb2ta05?#-JB#r)PpSLzPG^HejO;lr6*uNudrQC&M9NW zIObs_t(pB+tp7v+PgD!GND@-_zOg>GFMe^*Hst#kc)&IMvhn_W8Bz5KC}%m81U$ z*2mMw_kQ>E^jv|#fWQ0m=kx26W81gabFtX=j(|_MueHDL%jMzzN?6B+dn&`*$7wMO zdDn-}`|ZS33(dzv-7(Jd^^U)fw|hGv-Peq}K~A^c!S2<x=aCOB&8b3X+LuKoNw-|nL>C0-xDz8p3TzAm@UaU1g)z&rH2 ze}8*;++JOMUhW9+eI9>a_iXxpTn?xt(%Bo_J$)_gANm{ke6?D6-tXOC3NUp z5a{oawhY>Ky51fQ)w;(joR5^syj<=Okcv0h~nnncNJZ0ND0&!=T_T( z`NjMA##=-xcN-My>X|$ZG4#MEPQ-C<@o#l|KK}fEj|zkRe0g;1NWM@ghTm`55cqO^ zUy-TObHLsF_CS^Pdb}gp2{`$5ukS{#|3zW*B0;#77RtfggBy<=|P6m3honS8b_ojSk2@l^`_CZAm5;$0msDkwv#&N z+b?h;bzZ@#6@ffq&*R7Bw5o-$rsTo`n`xyTORuqJt<7-pyx%*lO+AQ+@z+JI=*#1V zM7%5sh$u{j4v$R`e}yK@Zkw&_vcJ+Qe2f*14VMbWHF|5yz;)u>&~S2>R6!e+nDCar>dD|@ulJ7YL z`O!Cm|7M;=Yc#-H%RqMD1EDy5VqcyY6PA!@j@0R(MITc}HZt)Gck-2Fu^3ZB4wqiY zX|77<+zL1Iq0aA-oSTJtt+nZ+oSqhYyPPXBS=wWdFvke>(U#p4{?VNiZ!-(HY9pqu{V{@` zkeBo+{+4(r;!tnD$77Xxunz4TJK=$^WSous)}0qbo*RL^{L*dEFhW;*cBU)%DRjw( zYo=fvZ38nFLzZelYizyK^ii-z+0yV6C6aX@!A#$uV{i(5^-g8OrlT4Fee2WiEpt_w z%}7tqj~FBeYxJCM1&Fr*#--F9i^;=L5c8|Fk?z-X$S^o6&R@+ ze;}td@?*>a4D{N;ptiPRjkp}c_F{L_Q!b7HNBJ4S&9IB=JIlc+Ra7T^4-8(=gLyZD zD4J>yGNRSZ1pwD|^4p*5q!ZCRoL&Rs{rLm*e)ng>bgkkl{fk5QrV5Hyu*D%gnM?5S zU75H5UdoGcz1%MoLazJEu(VI=ROr`dgLFF@n@|6G)KHAt8hbXE1J$Fyu1&#zA_0y} zi(J?IX7;WC6%Wz0Y|tB%LO#c7nDTCK#d>-GJIv@z+folG(sWi35LKkzTIjGzoQ(Md zQkoFhpUWTEcB4$P;@|Vt{H660=tN%jx=O73eHI$JJq6zt^l&AzH4DJgvTHF>hhnt< z#Pz`T0O?;>l+%w^gvqDNuD`jq1DA}cY)c0SM7cfPeFY+&g&8tjdjOgqss$M}+o=36 z2t9{P3z#D;v8pqIj0^1&BkJ{$yuOf^@~Ql`-iFDxCcmJnCtA?AJt9}FE%F|lE~^}! zK7B6qcprE@r}h>j$@P&s0B(J%zO9#OvTZCNsHz0J+0w#O)vBqvzuMXB`M`B?62t-;! zEAV^+Gmk3seDk%Ab}#%4q!bSoT-LbenVujq%qo_+*)LcYjPq&iY4~a+y*+c;5%ln9 z{;~t|ogQL&wOgbWdbh&3;ReTU^3QRW*}J^rjh!*-(-z`oNA89@U>TRn6Z^8-8E?=q zBUECO8C^G4P`r>$tdl)e0@l%T<#0=kUHXze06%}xCNJd2ytn$^1H5io;`csq1%!PT zh^f4YRIJC0nVccYPM502`xQ>mYe^q6GCjZn&LPu-K#76L-AEU$R>)`Kfoy1mWpPB<~otG4Oe^bz`lox7iRM9n4z? z`^N!g9086y5wPO<-<>m@&4aqJj&7zBv0TsQG-g}KEc~k>&XE?eo}QR?=xeo0i1&AX zS4ZeY9DiImu9>cl z#Uizkkroz#U`K98P<_C~MX@!RmF|wwz%ASv*`J~VT%Y|Z=t0|SJ=VpE`C#o-cilVh z+*0B12HgZKoeN{nb2hk3pR|ovq9E>5H-5oeSj?Y7tWeo_XJ{4bMR0L;vCzAP)4jS+ z=80_nA0zn0aB{wBnQufv8K))ux*Mi`tptSQm7OG_b`?35^$LQl5-CHGuW*v`V`E(W zaFsUwo8yScs^u!5es(c@4pBx)EJ#RAW7sv;{6upY5lZP50ZcU>yL8KpRn^eWumBsR z*y@L*^xoy=73G@ogwu3i=!|EtHV_$Vksw={%LbNmqZv4`wZba-JFwY~h>RxPas{l* z(lN*Q=jx?t?uGDKTmdSiRc$d`>qq+E1lk}(k#qc`W#8~T)pl3JxL+3T=!}pvT##@Q z`xVN4aI-0;wRTdg=?ybL$Cn8chJFMkhXb3VG?FtsC*u6N?|2z&W+_Y-T@L5;Y4L#*nLK1 zg5w$h$I9(Cu`7O>q>Y5sp{SzOwM!V~Xryl4Oq{72XIygQ!;bR+!2Ps-M~3|CVLF{T9vv#K!} zv~%LyF1GGnmpT7jI@)glG`)nowmQhpA`xhg&;?}UD$xV`157k=jJ6YEP=S(YjLi^F z`-!vVLIg}~p3Xa%*^MdJKmyc3PH^$9t~m5?qU+`C{0Sqm^|~%;8?4^FemLJK)_H(m zRFQJ4PGYiHrks&}2xR7JLpV+*8mu0=uOu-atFtUicb{kM59S`BBE~gl0rY>r4R8a{hsEQTL zTl9HPtcUKFE1a?|BW)%CFKH;{egR#%DHapEXB=R(xsm6g)R)%WzrO@8qSM9Fv)oc* z)XX>K3y}W9ysviyXD{czpd(W+ir4cIEm|TOp~h-@E;#Bh-Z5aJ{FTjafS+O5>ZWM8 zwv}2K>^9lVR7}1dl67n-Ny@j3PV0$7cAEMG&^nm^^uD8n6!^h4jgP;-tyyWi2|J3G zo@`WRyAhJrCq1y~ozVgCaL^?kFTrNaSq1`%y4$g7dbMoiT-|BV()Fama*IVpDGPPc zgeUs`ssOJ;hYjwP~@&);8LVuUT1f$I&Rx6Bm-f!7#k zb5I(K)KqhDSp`9aL4IKZ16v9?Gy0+$w7RAlh)i?F+Vb>|W>ow3yR`eTAC0Y!lctv6 z>7&WUT^A!IRXXSkZ$=GVcxPk+Xu{KNZBj5lXQ|4gI>+8#J^&xCQDR-Hla>wA>KkY+ z%x;#pwE-fdetVn33q<9r)gfo9mC@#qZ88TtwzOKex0899oIb2hCL8?1t(E;1dJ&1{ z@{}*R^Gu~gOrs%|d;U%B=iuPeUN$G`ue_5#yc+m+2p?I;%m2p1W&Zv5N@J=)$9#;z zEw_MJJbjRLeGn4{A;ndMX5@}A&$NsgXNOuszCIU+-28%23x;qAdV|n1*XbK*6AD7* z2Bj4GeGbDG>!0jM0g}l_DSqL3((T1FmTN;W;rqs#b2gPrQeo6s`NhVTZc3KzR$S|NbsKeOR3l}U&{7GoPaJV0>N$ud3&bjZ6%GAcz72*in>7->Q;g7R!P}xI>UtludAk;r5s;wa5l8p7FMX zBozDeBVeB*`}_!QV2O?>6&a<oo(Mo+Brcr&P*R_tQ5^BEL_`G|F6Q;GZoF z8OCX=;(Y(}cRGO5x6Uxj z(l8-xHJM`PEPwG@gT+Pk0Kyy z@6w-PzyLC|vkskTkR8R!EZhtS7q4nO-<(54PLgteNS?p8RivUPrwPZb#J37qfm*}y zjGRf&J6W$oxr7NFd*yz- zOuu`&>tC(w@*%68-cH)TkR!?w*)<_k0MblcU`MZ49eu&drcBg1pB-8GXP=s*-X2?f zu)y_8#L<(W-|Bwj^jLZYsj6F{`xTh$&q=WcW@S}Cr3j?KF#^Eiy-pFj6GSk*jIt1~ zd4zd4sMjC=g8S%@tPj=uT1fIzEptyUthBlRyAyY>g}!Fqt~uhgP$tTeI~-Y*0=hIx zJ16+AH-;LiW-vz7KcAggX-frwJ=#-i6ZbN?Z|{(1>_U~NX$EXw4k9n=-N;XzVp=^M zTM?oh&Ne6i_3o1zvJ=i$`+@ty3?4SF8}G*TSV*stEJXE#A@BXwt|$BFoBwP#%jKge zc{xK}abAx}L`7nh@RcDX*S38fn^NPrK(qAbxy6JcdVW^$c!}{9YN^)hps`!+aeWd3 zIbsO_?amX4Hb?aj9JZ*;hG>b6Vq5&;oyn*nmEY$8RNY|e`T{RmU>^U=#cceA8E_H< zEEt}0e2S`9pxaPWzyl#TZ7VSr2_b2oR{xERFV);*l{6>!)#6XU%M9K-jUHI176Ual zdt{Mn22LczkEx{>8%?~nd1520(ICG=`Ac74_5d8(N!5Lkycavb|#_>b)4_lIRZhz8om`Sxu=aTZ}RSGU~fj2*n!ULYi zXfCzEOdEWV69_RmBxqG}bHeUPc$Mw^2dEZKInB;4zC;P7m-0?kacz|9+cc*aaz+aR zkFoQuTlOea-WBE@pQ%_3NTadO;?}ju6c;qt;9@$aVP5M@sx}bP*oE=0jG;`n26{*H#t!kCnc0{5art^uB;f|qrV!u zAR1?z)z0}P0}60d7gi=A@3PWX<f-!TLK>p&>~bY9t5zApzl)4!o_C{#%bAO4 zUWvr$=pgq~Iwu68!Uig~D#1c9UjL(hPzXdsaKL3tBOGbXvkdzFKDRC}qJ3_uBFL!& z5p2Cm`s5^;?-K+=IWBGm#bwe$Zd|qAN>;r!m#A*iUg5e7y6C1(ztzT2ZMhaBC{++1 z8ByK8ez%M!FOBeiIvshYl)<5WyXDFa;2JdC|+^AcdTj`Ar%FvFs2`LTJn8=9zLK}e({>oH=1{Vr+5oNTu2nUyN*!%$^oPLbErwif`p?h3iQXLKm(QIARH-~71$Kl&el z>L^F~sgp%eoTQzg@|A5>n@cu51`H)Qbbg2eZ@etp%ypdKBW=t_H%k2=`-wJusV?fbR5Nr>qFKDA!f!t_|Djmt4<)(QxC z9>+BuZB-Omx{?pbl94~~)CelF=1Bp6f8Se|H_<&GdyFDQ2KIsJj|ajmbP6XFw%0MF z)}e)3o3Ki~1LoGpxUh`>SE>^r3LC(8z}gnS0HXm{{z}Ki*-3N& z*afHwrecqF8_!Tc8KH15TH#jF>fLgPdC1`t)8nxnYES|d4#!LrPy4a8tVZ+n9?<%n z^@a-AXurcnT!&p5?3AHEo*dmJFxA};sHX4EhPj|zGWDzfz#FH>12X=O|Hgn26-NO! z@q`f|+N~;|SIvTU%DhMVeR$efWX_`w0rQW^GUuhVI`w3GA4yOBjV73h9YzJIp~6TC z!um+dX!e^{Zg51Ni)n8EcQwd+uUe)4M4JU{eIN&(Wi_W557V}Nm*CG29nhk5SN>>O zZ{qQ~1n}E6eFxOOYEAVMBq;)39-dh@(W_yxamBV$kqC>KEHeZ%y|TEbQ@H*|6llmP z^|BTo*d}TmE*;$HEkrE@SS71CQnV6GYW!Ne@csKO4RfS9PYT34&m};>da!i1nWRoh z$?4`d?R6^hKhY$Uwy-0lmgsngYEm-Z0VnZjVV~SLo3c|nFy413__(J69mp50f^A zlL0d>y*01nbS5H?EQ#`D+yQ3SIlT^qvu*881sv&dgW%KwnOqF~^jyt9t}+_Y*4oaf z=%B0dr%wROZXNUje>2_j2O2PelV}(`V=RKgZlLNum z%l*RV%0%uu$=k|%Y3cinKj6*Q)@4UW*rmf1eRoff|JyzT!-mC&-}g<+#Bt}V|9>wm z2mh=Av;Zg(Ob7NqziRxqiuG?5z1S>5G~er5!cGmakBo@+wQ2KtepagtKi z7W;2sV2e_>sCb&etIySDq(*=`?f)nu2)2NH_HfO;{Tse&23Bxk7pHxHiiZ z@t8N5D>pShsWZxar!mf;RlQ}ftJo};$n$C=kR_m8w{zw=?laz~_Cm;4`03-D4Hb=v9jSh> zYhsiwK|~u$DPf$eAj*CY@mXs{RGU|e5^W~y+52HPW;kW2MJI{R_0h4~P4J}Qm@l%k zi;jbFF+eS6>3ft`8b{yLLx=ya2}JAVUq2uLTEO`~Yr=nuMQ_MBY>1Gh?X z_8zYLQ3M_2rj$qdrpba~2OBRMXu9{XqmlF@we7i%KXWi4=7dD%jgtjOuj{par@H&k zXWb>jN^)CSrJ8r^CHuy}r|aV}0fFD-k~LQ5K*^eG*i-vk>5mEy4gp<0x6jhue7ljQ zQ=qWjb}y`v{X;LlP5wXc=B+0&m%A<#dMnqNw=VTP28biwcI?MIm+m$?D~??b&0%M! zP^o;*(N|aY-W%Q5_5#-%11~Od-CJz?0~g0j4~qP5TcxM{44LN@IR!S!pFeAEb$ZtSxKAB#y?8&rICpgUV=4GI zGj1I=Z@?e>YrJ1Rv{b3TADzE`k6fDjwAN&G+^`qf+LoykY#4pi{g`Rl@xvds|9(~( z=Os8lrF?p&d^)5YUViA>y6@WZf0oB2T8b&F^Yh|89{K9V&djXr_SStoe)wrya=fYA zzOeJ|)_!;Ca{1c4a9d)rJ0Vv#u=Fz-HBFT7tTk}>*pRo z8<{7IEY@vSqGLAlqdaf54p4awo1mHT_@~;Iv1`L=cAriRYu%;5>B#ehH zVLwPbB@pv)1jy|p2sDPIC~zE;KAL1B2mlxIY$11xMabMwm9>AF!FguaX?ElLo%sX7H3!rtpP=Dtq0!1U z4x*C(;P3tmzVpA}8HNCOgs4+2CZSQ&M+~w-7fo46#s~crK@-`>jI^jz403#g#M6M# zc5oP}maVMGAQ=zY5=fj16h_jmCue3%b!a~F1yFcbXjVFnC{t(VePQT-o}ZyuYSZr& z`pKJXQU;DaNQ$U|gdA)P@BpL}Nj9&TEJ)-DaS!O8a=}Z!KmxA@jh0}+s6eahC0il^ zAU?TXoSWEI=XS!BjvrSp!+4&WquKkACOZ=sAuNe(gQYYYJp z35>N+w&p0D=43BG%0k_Yp!IRsb!mYB0WDroXcmBBurL}I#A)jy4hoz9r-$;nSRkMW zZ0dsLj4DG$@8D< z{(of5l_TlQl-hhlDr+u`Q-Lw6_KGhzDP9mfB^w~J6|^Rk)XmbUZy7izJ1Vjiw5FJJ zWJyn}L>g@lf`5iGImpY@5sA@^4vgpu{ovV#w^GEF0c)}qJSW>Lq6d;+u8%f&RRCy> z0@fCuWvL@+)hLN61KXoaIxj9G*v!~C=iio0o>2*J*ngVoLI7?fm6}DNZOzRmMTyvo zMh6FZO_*6k0GzWu$;wW$V6d3vY34opT5O-83N7U`o7GO{q( z4g;h~)D%zhe;!a4vHO@Ar)kxth5XY)JVqFf1s0?6p9NpPQ42RR?=!{#h*?=CeFz0$ zs{l~wL0cgb>V41@L8YZo|(O@tUl{wKMt>$DuK#Q!owV{~)4;?Vg|9hPg zgCRWGP#2Q@?_vqdsxa7@SWE_&7rDu=1ZR}!ua4`p*!Qa{ca+))%d41h=hpalK}nJ| z{c0zF$m$jhjbils=V{sbH>wNYJMnUTn}fgI+xw-F=H+_F0@Hca0#CL78~gg}aXn9t ze@6Gw(r=TNZ@1HYJ9aw(J?_uujvCw^y|nJT$Ght8On>129#dVw9J&cWfq){Rfq)Q! zfPg$~8QdM5Y>XWoZ0J4gY=5UXC@cu#jpV*l)BDn&|F9c@`X!5|$107sdpozSS=?o_ zKHbNHs_K!KcL%fw`kQ4~BIq`d^nNEZKe`LHmTrJx(>8_G>U#a!;`yB4eWg5|ZATGY>*_0z)p^yogk^Y)h^khMkeIQvGEjXH6DY1U_Sz2_ZIzd6N<}i5dCt9c_t8S(q zrrU%C?=%t7y(O�oxYgd`T%z%?QSQ5E6V)$71S${ypo#9A)=zE0c3Y22p0g6unDu z^hp~rpvDBtQ|NK7@%PG27;cjXq zp+1f0zcAR@7N=`cbt&w}5oBt!p0|vzq*-dL(I@6rJfLSApE)>5a?n?Hcuw4(yaoJn zzFw)ns~_a`)C0|=5^Fl4&JpCf7#BA9Ww~+3rv3b#Db~FF?yk@!QQ}Bb6Del6$0iV^ z&GAf!!%W{gdK6+2lGA{VhYwfnFr1EvbY6hru~Fb%a_`hY`NFmGNMIPrd(#SyCW)|M z4XfrDmyu+#4aWB!2A~u1?Odl&e8dB`sn&2e=SC|SqDS=Y?2uTG$ z1c{PD>7WYy%3p)|s#i-quUNF6)pLjM%AP|E?b)s8nMbFqMd7}lRIjc6qTCyHF@Q!^$M1k{Ax{91fcIt6QER^9^Sl93MK_Z}9(aOJE79krZS=Kn8Un zKxqH5C1*<`Co@wO7bh!w3+I0f>Qe9A9d#t(d)gp>Ag3d)F`{Zt2WgdU4>=b$^q17r zt=Y6D&IU{&wi!Oe6$R)ncJDL<2yj0Q!`0MKT=2vI`%Ta0(&%032dvH(UxJxy>_^-| z^x^mw+mnleyQ`Y(Qf<3;g}}#iq8J@(O@?~kth-%ue|$BO+6?w<5P!SP`N+tVf7~`s zgRR&MO-6oYM{|yHb@g|*_e5BVS=e8-t@S3?$z~azx3{XzG;P(YIwfUSN^GXd0^@<) zBgeIAIwuD`6qfX)yYBH+I%)gD~=o)G=VeJOgS=Z>% zmM&u+@iky(L`p!a*;CO1%=X+hBVMXsZ4|a7EVjmUU@^(JqK;{vOEPR!$tfzYbJ}VR zCTINi_0!5Xvngk`GI~Be_cLLSYo>eB-7G9B78a(iWp))GPbbbEbjpZ)($8-;NuJL{ zh?qn-d8!f;KI2oAuawoF{(t6%ABw{mseoGf2EYY6LX z@*s}ans+8-GGN&AheZlTzJDrQO`M)hIBcc%OwY)x?{vE|@Wa~06W2>JqsR~Mi=BUy zPX{-%HVb^Oq=#hoM6N$er#J{CMu_e0igKT_F<^c7YdBOg% zm~0?koHlO1`(C+O%0PWN8+;%6dG}?fRc?CWL=N|-%SO)2jgLD_dnb7f-ToAnj(-*b zUMQ!1p~C@mj&!b#Oe?8A+sC#&bFir1MRCpOvOvwCN|+>77) zMj$E6O>&Nv1kw)iq}_U2VaFv`7I^FGascG0&%tSIE*W=N`IEACTl@i|wfyUp?P9GOQxH4a2MBVbiXI;IVJo zrIfK*SVs6ras90?tNbs+6+f}Y#phmmI$sKl6OxcQ#&lnbgCKiQ(7rI17vy#H0tdT& z__z_O;y4;EWBg?*D1EVu8*gvxA5LHr`T-$l?X~N;SUwa}J1_B=LdbOUnF$IYX z37^MT7P|%9bA5Guy?!AxeIHEs50Zg2bbt0aAY5$k^zc4Lpv(6SHo&Cm8p({A>AZ{WH;_hU0ZPg3QqS>)wydeSzk&lJ;fUbMOskd`$oj^v zyJ30J%5H@=XGoWoEGGqWDVLPhALV-n>E&8AGDpeBrAmn=ABdNXRL5eW&6Lx4>h^vF zPRoD-(xRkYKbtQ_4(JWCnzF?NwYW;uFoGmOZYz)@WT1u$phCm6@aOoisASxk-a5wny|u#Q z*v&TFd&JVM(+&N12T59W-#1UsE9}Mc54VBrZpb1~PuD0>%lBK^!y{0PAtdgdE#O>` z0DHeDDZMn)jCr?a(Lms5i{54q*bIc^c(I=sr|0)QQz!)#J+w2Etp?lL!Xpe87~Q_A zey*Nl9e-a-{i--7>iKO>a)SuN+eLNEc)Z_Xbk?|m$hqQbhtM*i! zF2BDH!-~T6>sD(MdI)G*)I5CV`bnOXNjA>znU3VK>vZ_0qyGCv_$(6Bz7x~|dL4>w zm+lD(&*m{T>~O`P?(e+sJ4ev4(9H@#VjkpR;emiFZAV^g=mis-3nA<`kT4F*pO1H< zWjjk&i9We5!*_3@pR9Sh0EupSTWobWZ1Fr(uNcs7wnHT%bAqmpW&0}my5}Y*6!0)6 z&>-alpNEV8^7nacxvM;8OgmH9X*>3>^!aSRyCKe}!az8L#fTse$PmZrEUj{f7(*}} z)BV15JiH&%9@taQ1ZM&Zp!uUwk3OY%SD^sa!edYilvek41i!En67dd%RpBrr+$#t1 zXkBpk9Jb`HV2D$1C>L2wBn)doSr^0!;{sA3p9Y2!v59BH^;6h8y@%zpyG>~Eu4Cce z-5&noN9fAvL5Dl$tjyt8At)^$OJDk$SS>Y-)}Aj$ELekY253`CnFLpl24A>$dSB-=_;`3p zjh}0=Ph*gf2=I9Os~pehOn5B^wE zLoPLG4F|*Iaz43=97i?VBSmkXeaAkN2Z2k9M{mmp~R zDtT_g>ThiY4|P@fadj!0)dsbfC~C|wA}BnyNh34WWarQl;3D9Ox|1lOq@Y8N^#-&f zoL?_~ybm)G=QrO(NW85@js+m9)te3uC`C-}9d9QVc}ranGUVDK7 zvoRm#mg}B zy8ogSb74H`BwQ5AP_sqi(m~Ocb!d7BRwOd%ew5rDcstu?Z(LQ;pjPlJo58n+ zJWtWM%bM7POUHv-Sa_SfRf*B9l-DJaP50Aldqg*cZ&<$V(Wqd!)&SmJ+9PUk5CKJw z_l;%TWtv6c7E9xIFOf$tT+59$7QaYn-QtF_$;?ehx*ZSkxi>)};c5O^2_G35qdyv7 z&90|vG^iTF5Xt^C*fkt{RUZ!nb6#(N_`AfCL0ogB_Ay_*nt+|^;ppX{nuLHyZEumq zCat3GFV@;8+@-6m0m4&kY%{GuQI@Tq{;W=l1-jfmaX4Q^4lWJ#(XFDTCk!q%gM_Wf z(&34HRr@3oP6doaV+$km-tYY~OO><~)cKHNNVIUqDTRWMx4Mt9Zdy&cblOy>(RR+N zqp_4PeYKWF(2|5>9ZuF81E=p^iXh*1qU3>nNLjUdHRm0Rm;$!cAF6crv!Aa%H77sS z#`b-TI$!H|&bZ*1y$?*}ZTgfo$p-coJ$p(tO+K69-Ed6VL@{Q^W`{s$oEmHLxC+jK z)WHY#C_;Tl=YpM3p>0n@`*8Npv_+b=Mk{sqwL~BbTT(0i8wP)*kK$z6hQj!~fDLCh zI$Cd!m4~T@RQe8b_Oaby@AL|ppxHLMtscNg{|ePQ@N`~8`Ew^F#23%@5(sY_JUr!%D*TdVyf)G*#bbH$U6`?HGCW6AI?7a(R~ac~ z5g0%d$v+Ms#_I5c7Vj7)rXsRAHP=WhFHz+pX!v&zJL-3%VQfT_@hJAupJ_aB8)XnaC%CUCkgVCUo=3bKZNLSry@8#~cx;#*n zcq`GVQxqB!8lFFTxlY&7XcEyT?=2VTZyc9uH&z=B$#{RMeS=jb+P9TZUj*xm#+n37 zcP=L_NfTfw3O7>*)v2&$M%=EF+~y6Morno|Q=+dA{G9H#I~t$9_wWbd@26M=PH|Zx ztqr!79u$6j23I_r!B5rEik%ajOnCNChw#S*q*FwzPVvieiTMdDwYpe)bAN5eASGo5 z!0;R<#!H&kb{5anOTsfc`PxeGVx$oJY8$oQDpVR)5+n&iAt7k{x**pmINF+^`N-{r z7cTsbbBUKE%R?+yotLM3nx)^G^MJsa3T&z)G;EQpD{4r()_t$XGLr2!T)RtaBg`M@G0|_tbB{+0{3Z^p&LLo z-{fa3H9rH5I#_!o_E{bM`+;*XJV6B;mFk+9cuG&pc_0)xKVwZ}K5H&$o5Rf->n2=R zSvF5$$@D3$LQD1R;Nt||C<%tFV1$~FN=ExS@kdAAoF|Ex6oIN_CqiwLr-;TOa{32e ziLew~xIQ_P`$$K)#3-YzL@m59J_Lfh8SAa;A0yT#`(@TD^vWi*Sfu;%998F->Tx!l zmG#``>%Cwtm4QJKk0^ZgPfs`5`ghx<2D&#DbCUv-JdF0;eAx?24KL|Lrq4?AYtzJ2 z1BuJ*sTfX-so-MM1f8X_AF${~O;slc>``U+)ywn{!la8Zzthev5Wr4Lg+19I9jl6u zhDI9SNB&o3XB`$**ZqBJ2odQH>5vWy>24{hLFu8pr9oooMv#F$=65@`gaL5U#+ z1b+vgxSyM6-uL~zGuO3e|8e$b&R*xc*4gJ=d+i+r)o_P_CTzk3OeD`aPbzg3ktbE0 z&T&Wxdy@w%u0hpebW!78v=oZJBFGyN>Z{pt4hryJkkj{*7QInZiHJMVn5P~J@_JDY z64uYx<2$*`DILK9y%Ux~iRfL9WB3fqa;{7{X;9BLo}_%+*1s-#Rk;52V0rx4j8W>= z=D^z?IlLW{((}1Fj&@^tX8fn-H?^U7s8#M&+ql)qAKr0l@2FJYL}8l_$8j+15L6`k@VL+|7=O`1VI-K8!+HD|SbHlSB7yHr@nEiM#`+ z?|{lw(4Mj4Yv$JRq}>0Ya~52tf}+j329vObNTl#65n=GRYBCn)K7!DWE2T$*v{-$5 zo-rAF(&E~ZnkBvJol>9Mba6jTzZVeT>^ z^7b`ePN+oE59v4My}Z(2cgt?|IvQOV1N&Ac%lenRx6KrNuOPTO^8Imw7Vk{fTgG$@ z%lWrqJmH=cw#AHD4&f9#ewP7{3->6rC+gqtil~+#qS_V+Q^6+qyX;;^)sr!Zo5rN> z;y6ElYClHezos`?*)w$*o-@4ry#KUnk083HOR1t&Ic~7P(#3b``B(po zcJL)P3x%2sT3mj*xnNHY6H+{WRJa*lxW6e&=Yj?BzY*<6t-wf*`ZMEOC@ zCqXpMgmrq=7uavC5=L;XJ_sQ*=0D+3*!7fRnO#kca}lzSGu`S7dnOw2Y{dBjQ_s`5 zv1t+GrMl>(AcMo}wp3kvhJMJ5F{H?bbY5_9vM^x2Ic&5z>`;$+NTw^YGafxrppr+v zLCp2pDW_QP8YpGBx=TsP_YzA-onU?UaC-$m+3OX4;o>29t;TGCO#?J|+IjT(ho#)C zvK=BS5`jaUWxO9>ALq-cVq3noRBTDNWy9yFFRlmpTeDUC03MB3u!*J(J8ns?^U0v{ zM`4-jZx@fN6(l;`mBt|Ct}YkM`8w*pUWHdqGpnveKSCAgl#ycm*hUZ2FMS^oRP^e& zVi!flHPM@$<|;L^QvLzQ-OtGdsXeQSmN_M$X+QhR=0W zS5)USiQ?GgW*YRut|G!ePh;=#gi_3cDNYQhMB%Rcm8%fi50Wo)F|Bcs|sl-1m;gE?d6)w6IEj--nKV-SJP z#U`68FS{}>(_uQ4mWW|_zI*NKB7@1>y^<791{D)T5INo`TE0@#jcw2Cvq>v=8!D!( zAk0a?vpvA{@CBdb@j#QGg-UB)DXc#$S<_OG5^$b8N;91x&j6-M$~2k%FlBrqrV&~9 zM$b)_Jf(j}^l`OA8ijsLRmNE50Qb^nVh=-$I`JwXcjAc#q=SJ$AJt&j67%P$EG)T?o{#&A zS9l)=U;N4_T|s$Q;x&(vm@=ImbvhtC%cx0ilE~oyty3J`h@;)mdV7)QkptyneVD&3 zhL$dAP&rMb9ic!uGrb+4kgALaG!LUbk;_Gnl=hEYour@25V{@ z2^fT2<|H;pyIq0kRFyD6I7v_%Bajtluf1>U_CT|7$U&QeAW&O8ZQiZj`4^wb+U5ZKvp>HU}U~B1EPiXgj;{jNtm&P9&EtDyXqIA~VyGFc`efAt1L9FoI z=O`U8-VX6e0^9g1Pe3Yx&^HZc8qQY?=%UI+b=aLxni@J=UVf1BzzlKf#1>=2KtbtD z55}{cMa#$V^%vc#sg#K?%Mkrt3W5cfF4TQx5FbYb`R}TqiKTxfAYFRXS3CZ@oz7_k$?0AFK(0LTYqO9h6 z4zsE6jb=?2vr46oX38iMoWn4)HxSk$N<7rz581@TvnL_AZG(#OIcZrJNZPP6jauzx z9^_!b$!G3C;{&!QQ9YYTE|htdGQU_%p)bgu18?8Xi#c0a`}f7fMmgTlLSIvue9~WA zh)J{)>diCFrVVJ9=OE1&V5nJ02*_#C^LL+Wa~aLLY`Q$Gy1I6C878+)*3LhVeF4T& zO=%Ht8CxpPDoQ!6Ti;!e(D#_y^5Dng6zCb*!ffc!XMlwedGNM)1{BV?SlP#@T(@01 z*x1K-5!>8cn{4#e5Vj9ERMEZWn19p~=~Ktav4k@!n;Ypb^3e!Uz6e33gBDHIz~XoL zSLKBO^S~&{m{zj9sI^}k3hUO;%oFY7w-kyZquOsblR_{}f+^Y9)svp zHU*oiWkR1xIDHnF&@hsGxs$} zI#Mp8$hBu%<$Hv=JiGh(u@@gV6A?qHrM##p(Ud*5FrAigf1{2vy~3%aPd=FFz$&#m z_o``sb&r{lfu}~gOWB*38~@eor6eL$PGHo4%1P2{d>Gt?d0RQD&asX8il^zqTUENy zugb688cw5hpol4*jDUeGO>kd>OQs74bX)PB8)GNVv-=Pla9;zdD1amnMHvhf3M3?X z#?WAzdtuDCmCwWe4ox@^EkC^=8l}z?Z0TgB9;-L^C ziyhxHX>*3L$^mRU|hw{%#<%b-eE4;nJ3@3zc(}9rlE&f~uS@ zIE*jDy&mg*mTD!VHla3@ZH@FI#D2aSq%7ie(eIN#jm{b|$iNlfDS=cN8YFWV5)_1j zibtZb;vUs?wi1EveTbllDe46b<)|n`1-+e~-+SHz2GdIGSp01IGEvv5N3U&w`6DfD z)*`^Lig0W;LBlq(jOUe1QRTTrz4limBeA>&@z&HfrHGHF6<_zGV&heUd}%DVqD>=I zg3CQ0l`SBcJ>x#!g|1rFFZPPbKNA*^D@IZskiFTp5?BwnV1CK}Ivp18ep}fB$*hM=!*+Q1`DI(HtCA zgGDB+n=DP{devZx{SzjdjG-l(eW}IJMoj0KJ{;6lj*l;(EpME3!^P5}?k598{yag` zWa8-(xs=bZrE@9StZBv{#B+@&(Yt_?3Ryw#!VRu)JUcS4PVzvnd2M!fgs?U> z0=mJiILofZsX$dAf$HK`&>FsINS0O1dnlS%4IH(WC`i@YD`5)Etw4^rbsW4~#{sUZ zG4(`B4U=Xv(!Q?;-W9RAs0!X4f=ejbL^Z1N8dlYQ7AGK4)!w-N!%? zD8TLeinSK5Ag7E?~8fCAT*5*cq^wpS)g8STL-g_QLNEGhtl%7=+{dN*R;pR?#>V zG=frx`v8O8X6H$|CH~kzhuFczpr3WBz$*D2rD0bf6Vr`V@vGr_^04$qyQqYZsIQ5D z!~i^0IXn_YWBQ4=3P{)jC)JX&$*m@75zM3-N5BDj>p zS*aka)|;6I7;lHnr=&gw_)`BRzUiR+aJ6MX4?X1*| z+%(!Kc9*ZxwR7I7QWh#rtjZ>lqfOe5Fe&gcBOx$#`~_6ZNz%nVNWMNjmE62vblEAI z9>|S%N(znO5$&oxQd64ZcSyHB*tmAN%48Jc9%H671-5Ir7^@x zy8(*S|Mab1d?K%IBsD!0q%sD2l}P9Wk{l*8do&Re)BR{7E^w!niJxvm5~cgM#^#uk z1wew`Dj!P#NB>v?T=Z4#TRF}M=t~#0r$sf@;(CG-)1fUu|Di2_nQ!9=j%OL~z7j1E zUkI%%X!mykSJ=M?xPs?x$eDU8GS8)LTTDP0(?s=#WBQTRQ+cc)B){mHIJQ2si4pVJ z=IBTP;?7+jaE$$jkH}D+TQ|@F+(4!Ts)JkR@ooBbc2h8@1IW&g0@gn#R=K^jgZ!CMm8HF4v&ejCJ>wD*1}RRO{T|OTNsaZqFoO$$c;)^FUzVpP-y4)u^R8EHxfOX*)C(PYPN<@T;V()hcEHV(IuPu{U>)pZb7*$S!e762Qdsnb);ONAtFCUO zftu^3Du`R)!&^KLr9x0u?(QS5n4$u%QN=^*=C-e(V6KSC#%G_&l1p^vQI=!E`1l>S zWdsWdSE*xMdsF0ywrQ{(6J*8_VJd>de6D?o=zUjR6hg1YtUh8erg-!jL%Dugc5Uc2 z58?Te1Z*MZ@X`G;ZRFiN*R53CCPu!BK0Y=KqKP3lWuH1ixJR~Owk$z`vBf{`wTRmM zxd1z#^T3lUvZfoNAzI->wBNBL{)1AL-+@biJeCM!>8sBf;Tpv-r@C2@8zQfeHtDy) zP@2ee_Q05Zu>IbagCNdl@Z)Ux6$!d4#_4?D zdl~ATD*6tlXtNt&IYe->)~FH1Q)cj^%aa=tR>C!1^25Ix*y-=PrIAM8SbwIlwk-bUG>E!(Q5v>lhho9% zGNv3H&?1ml!7~c(V^1QEeJB{^AoQ}sRuNV0TAyHr*-8=Y5O-`5OngDiuE$Dc@#Ue1 zriTpsk(y^fch;EkMRvj6)sjgr<(^f75D+xU5fJeIAwD-Nh&_b!Ps#mfQlGwR+=2{A z`!TCrdqXU~@JRWaGXA-O(7pPSueI;`BSfo;NqtJw2OV6cfql|2G;oON9Vl?r-={Pt z8`Y?z$nF}goXRJ{EUHK9qY&zZ=(AzR{sBKJw|48$xTWP~6o!)OH=dX4L+Qn^t?=s0 zgkRkhriK=DD+ZFPDJ;CfvwpE~x$)S>Q#lzpk0sU>i8JLE`bm~ ziSS%p<`4(|HhN7x$OXw?gc>_JlqB_@Lv0!Ors_vo$yp}!Jz*``3-RB{ZVZxK2|A^y6W~+4C+N>dL~9^ z28yj`B)buZJ)}DR1f;6HaLL@Oj*B@^vL)W?eM8XZeY3v)ttKg4JV(Ua?dTAge2JAA z^5om_f!Ocw8EWG>Y8`$TKb3h6cYiMX|2mm50+I$&9{9qhl9qEsU&2l;1AKqaA}kYm zz3V~aW5oMP+rJ&yZ=4~CKpJ_hai3O&l|T@w0C84J1LS2JnT|0-qrLT&a2!FU^Lx@2 z;R|d!ZjCPpi!rPBoZ?)kzja%UIxs2=SSE&8d^62kd%z{eGq|69&)zXnict%H*g@uz z%|0)b|AC?u!O%X8n+CPoCt2rLv%Fsy!$TIgxV#5NaKIg z+hI9(4h}gj7s_N+lpc(8@h~ZPSkexazu4*ERvoqJiqIia}etFe?)qc<(<>#r*Ua??Lj5R&IwVG+v>Y_lU(f42~IU$3`aRe{{rDhBE!>-$YQomg5^wR+gx_)U19y1pUjcTK8t#1b&+tj6%(py&!TkZ zB2-<_EY+WzqI=G^zE*1=W1JW~EC}-Fv6V;*Ra`jIwjQ#Lx~T2itaYu(#KFMh%%HVb@9URErdzws*yJXMIZILyD13*N!e9#N9YmuJMN=JNp?(q$B%#d55ne- zw=6N3e7!!Ns)IblebaLB58oNz$030T&DS@oprXW zg;22KBgEYZB?STv=b9h(Hn_hB_zPb8EEcF5urS4bPrvr?*f4z=uX=(xjZvnQ%)k;U z!95cCLHVRgzc-HACHv7+^0HhRO2OLF{d*SZlSE6U{Kjnv#GrVWkaWDgHUSi~{xjTeLUn)HpT2y)nBLr(4`^52d68;AO|9x&(=V4ne;|1_$JlhYrY>Wn$!NfNYQ z>8hM9aj56k>(|qKnU{-^u@Y6YdrUS_{j?H4~X5!{+ZY`mI7d z6!5K*K#3hMb%bK6^`gCDxjp@*V_9KVV$>sQ`VBUSj3pf^l3b(ooX(KmSjDGBzW8Tp zLbDt9HRT8ks?3)wtAl;02|y8UD2!Ti8Z3?zd<-m77{mH)pH7s5qVROQAJwZEGOHsE zXC^CmCX+_VBr^VBHhv(;yF{Hk5!I-tF^TFfxU-u?e(|ArZBA$Y__uj0)&1Nbd zx}WXd+#FB0=+c>#Sz`U(2!{oZb{!l~CUK_;Q;d)P@q+Tnn-JnMj<$PUuff z?>%+7H~Nbp)9ObsK^U(9h)E2@#2q^+n=2T3b*j}imxfodC?mNzBaN z{1OO4&!M#xVwg6aG`7{!(MJ)VauU)hVU5JVs702W#4JSWP1k5>67nhk0!QlP0@~NT zvUP&~D(*WjYRH~aZ^fFq%++`~t|{f4eLpEz@EE-dp9B{Z!w%V^+@Twy-XDpxYVS zXF+#~(QLMCng064@(rfCG9nTI@@;MhAZrA1gp|Dnqgo(zyaEP>w`0S9eFB4SYX;Je z4sH+!H=}1Sogl6+Ze@QLaFnAszXKeq47dvBzgh#ofk7Jp*2o27=gP@(Ti$KhY7}r2 zbsMzs5D>`z*6cC5{SZor(6P0eQ+qe@F)&! zomc_Dc)g7DF69QW6&wm5JcMP)Z|p<_MnQfh!xgF|`z|3@idhfDdh?aE#2{-4h! zUkWbe&&uondrF2bT*{wC75?{>Bs;j2KMDE&Hwti>{{0CIiUK?bj&-@Y{$J+(J0>I* zJ_fFO|I@yI$M}`N$H0~Egs1MFCEx_uKq}?mF&>@pF>v+Ewc#R32w=tm4gVcO&;uU> zSHSpA*DPnZ0RnKzzhjJh;bY(`ctf*qCmFynBHzWt^ux!%m2gJ`xs5Gc+vp9#$H3Jv zaysUg9bC-iFnkPL5pxWvnUw>bI{dFrWsSnez*RBPo4s=QoeCa@i{XK*VlipOXdAel zqMwA1fve(0=xhc2v+13JkAbUVBB+ixe2nERd<_nY3kP4B*$`>Uyh&!47u-_zX%+9Im6ZG diff --git a/applications/ColossalChat/mstt_advisor_20250519174404.html b/applications/ColossalChat/mstt_advisor_20250519174404.html deleted file mode 100644 index 028ccc63e770..000000000000 --- a/applications/ColossalChat/mstt_advisor_20250519174404.html +++ /dev/null @@ -1,7585 +0,0 @@ - - - - - - - - -
-

Performance Optimization Suggestions

- -
- Optimization Priority: -
- High -
- Medium -
- Low -
- - - -
-

overall

-
- -
-

Environment Variable Issues

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
EnvironmentValueDescriptionSuggestion
ACLNN_CACHE_LIMIT缓存的aclnn算子的数量。在alcnn和host耗时过长时，可以设置一个较大的数字，例如'export ACLNN_CACHE_LIMIT=100000'。
HOST_CACHE_CAPACITY启用动态shape缓存。
默认值为0，表示数据缓存已禁用。
如果设置为非零正整数，例如10，系统将缓存最近频繁出现的10个输入形状的执行数据。
当缓存的形状再次出现时，host执行性能将得到提高，但host内存使用量会增加。
具体的增加与HOST_CACHE_CAPACITY的值和模型的大小成正比。		设置一个非零数字，例如'export HOST_CACHE_CAPACITY=20'
ASCEND_ENHANCE_ENABLE启用hccl ffts+模式。0-禁用，1-启用。建议通过执行命令'export ASCEND_ENHANCE_enable=1'启用hccl ffts+模式。
PYTORCH_NPU_ALLOC_CONF控制缓存分配器的行为。
可选参数为max_split_size_mb、garbage_collection_threshold和expandable_segments。
1.max_split_size_mb：v —— 大于v的内存块不会被分割。
2.garbage_collection_threshold：t —— 设置阈值后，如果NPU内存使用量超过阈值，缓存分配器将开始回收内存块。t的取值范围为（0.0，1.0）。
3.expandable_segments:True/False —— 默认值为False。如果为True，则此设置指示缓存分配器创建特定的内存块，这些内存块可以在以后扩展，以更好地处理频繁更改的内存使用情况。		export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
ASCEND_LAUNCH_BLOCKING是否在操作执行期间启用同步模式。
当设置为1时，强制算子同步运行，从而更容易调试和跟踪代码中的问题。
如果设置为0，则任务将以异步模式执行。		export ASCEND_LAUNCH_BLOCKING=1
-
-
- -
-

slow rank

-
-
- - -
Description
-
- 集群中的通信有问题， -因为通信时间的最大差距已经达到 -103060.644ms。 -集群中的空闲有问题， -因为空闲时间的最大差距已经达到 -99304.586ms。 - -
- - - - - -
details
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
steprank_idcompute(us)communication(us)free(us)
001556714.0687232581.3436829097.07
011560276.4578997510.1643698754.88
021558312.840868325.8279587869.88
031556942.2298397199.2425134636.69
041761254.8645969395.177693774.04
051765175.9256016250.6465924566.11
061762990.273041651.13117837748.23
071763501.6752671041.6269509934.54
081561748.0344449697.1176268659.57
091557930.15103663859.4920203496.84
0101558704.9887122155.735402682.33
0111557350.52105052622.0118533162.23
0121763293.7466816420.2348098511.97
0131760975.61991978.28111975738.82
0141759204.744457564.9268792652.51
0151762214.4758791678.6456870330.29
- -
- - -
- -
-
- -
-

slow link

-
-
- - -
Description
-
- RDMA bandwidth(GB/s)： - 平均值是 23.997， - 但最大值是 24.017GB/s ， - 最小值是 23.983GB/s。 - 差距为 0.034GB/s。 -SDMA bandwidth(GB/s)： - 平均值是 17.935， - 但最大值是 18.663GB/s ， - 最小值是 17.174GB/s。 - 差距为 1.49GB/s。 - -
- - - - - -
details
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
steprank_idRDMA bandwidth(GB/s)RDMA size(mb)RDMA time(ms)SDMA bandwidth(GB/s)SDMA size(mb)SDMA time(ms)
0023.997616.22317.4618.4170229.433813.77
0124.027616.22317.1217.5270230.234008.1
0223.987616.22317.5518.5970230.233777.48
0324.017616.22317.2118.6670230.233763.05
0424.07616.24317.3717.1770229.434089.41
0524.017616.24317.2417.270231.314083.65
0624.07616.24317.2917.3870231.314041.88
0724.017616.24317.2218.4370231.313811.14
0823.997616.22317.4818.3970229.433819.49
0924.07616.22317.3517.670230.233990.1
01023.997616.22317.4518.6470230.233768.14
01123.997616.22317.4418.5770230.233782.16
01224.07616.24317.3417.2470229.434074.61
01323.987616.24317.5717.4170231.314035.02
01423.987616.24317.5617.2470231.314074.82
01523.997616.24317.518.5270231.313792.47
- -
- - -
- -
-
- -
-
- - - -
-

comparison

-
- - -
-

Kernel compare of Rank4 Step0 and Rank0 Step0

-
- Issue: Kernel compare of Rank4 Step0 and Rank0 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
- -
-
- - - -
-

Kernel compare of Rank5 Step0 and Rank1 Step0

-
- Issue: Kernel compare of Rank5 Step0 and Rank1 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
- -
-
- - - -
-

Kernel compare of Rank6 Step0 and Rank2 Step0

-
- Issue: Kernel compare of Rank6 Step0 and Rank2 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
- -
-
- - - -
-

Kernel compare of Rank7 Step0 and Rank3 Step0

-
- Issue: Kernel compare of Rank7 Step0 and Rank3 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
- -
-
- - - -
-

Kernel compare of Rank12 Step0 and Rank8 Step0

-
- Issue: Kernel compare of Rank12 Step0 and Rank8 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
- -
-
- - - -
-

Kernel compare of Rank13 Step0 and Rank9 Step0

-
- Issue: Kernel compare of Rank13 Step0 and Rank9 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
- -
-
- - - -
-

Kernel compare of Rank14 Step0 and Rank10 Step0

-
- Issue: Kernel compare of Rank14 Step0 and Rank10 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
- -
-
- - - -
-

Kernel compare of Rank15 Step0 and Rank11 Step0

-
- Issue: Kernel compare of Rank15 Step0 and Rank11 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id Kernel Type Core Type Total Duration(us) Avg Duration(us) Max Duration(us) Min Duration(us) Calls Benchmark Total Duration(us) Benchmark Avg Duration(us) Benchmark Max Duration(us) Benchmark Min Duration(us) Benchmark Calls Diff Total Ratio Diff Avg Ratio
1GatherV2AI_VECTOR_CORE0.00.00.00.001316.306658.153660.833655.4732infinf
2EmbeddingDenseGradV2MIX_AIV0.00.00.00.00899.178449.589451.049448.1292infinf
3MemSetAI_VECTOR_CORE122.30210.19212.686.5612761.13563.428337.3665.46126.22346.2233
39RangeAI_VECTOR_CORE49.38112.34512.66111.921429.18114.59114.6414.5420.59091.1819
4GreaterEqualAI_VECTOR_CORE16.9018.458.568.341219.0819.549.6819.421.1291.129
43FillAI_VECTOR_CORE15.8811.4441.61.3116.521.631.881.4240.41061.1288
38LinearIndexV2MIX_AIV121.50220.2520.8618.881690.68222.67124.3421.54140.74631.1196
5LessAI_VECTOR_CORE21.44110.7211.2810.161223.92111.9612.58111.3421.11571.1157
6AddcmulAI_VECTOR_CORE11961.33635.18889.3372.034012491.69336.958895.9982.043381.04431.0505
7AddcdivAI_VECTOR_CORE13155.31738.6921144.9021.9434013414.2139.6871141.8421.983381.01971.0257
- -
-
- - - -
-

Api compare of Rank6 Step0 and Rank11 Step0

-
- Issue: Api compare of Rank6 Step0 and Rank11 Step0. Only show 10 rows here, see mstt_advisor*.xlsx for details -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Order Id api name Total Duration(ms) Self Time(ms) Avg Duration(ms) Calls Benchmark Total Duration(ms) Benchmark Self Time(ms) Benchmark Avg Duration(ms) Benchmark Calls Diff Total Ratio Diff Self Ratio Diff Avg Ratio Diff Calls Ratio
229aten::embedding0.00.00.0014.4913.977.252infinfinfinf
230_ReduceForward0.00.00.003.491.661.752infinfinfinf
231_SplitForwardGatherBackward0.00.00.004.241.032.122infinfinfinf
232autograd::engine::evaluate_function: _SplitForwardGatherBackwardBackward0.00.00.003.280.051.642infinfinfinf
233autograd::engine::evaluate_function: _ReduceForwardBackward0.00.00.000.080.020.042infinfinfinf
234autograd::engine::evaluate_function: torch::autograd::CopySlices0.00.00.0038440.370.2419220.192infinfinfinf
235autograd::engine::evaluate_function: EmbeddingBackward00.00.00.005.910.062.962infinfinfinf
236aclnnEmbedding0.00.00.000.180.180.092infinfinfinf
237_SplitForwardGatherBackwardBackward0.00.00.003.240.391.622infinfinfinf
238_ReduceForwardBackward0.00.00.000.060.060.032infinfinfinf
- -
-
- - -
-
- - - - -
-

performance problem analysis

-
- - - -
-

memory

-
- -
-

Memory Operator Issues

-
- - Analysis of rank 6. - - 发现了243个AscendCL@aclMallocMemInner算子，花费55469.200000000004us，这将导致大量的空闲时间。 - - - - - - - - - - -
Suggestions
1. For AscendCL@aclMallocMemInner: 请通过命令'export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True'设置环境变量，然后开始训练任务。
- -
-
- -
-
- -
-

computation

-
- - -
-

Pipeline Parallel Stages Issues

-
- -
-

stage-0

-
- Description: analysis for slow rank 4 in current stage -

- -
-

Operator Dynamic Shape Issues

-
- - Analysis of rank 4. - - - - - - - - - - -
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
-
-
- -
-

AICPU Issues

-
- - Analysis of rank 4. - - - - - - - - - - - - - - -
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： -IndexPut修改代码避免使用aicpu类算子12336.790.0003
-
- -
IndexPut
-
- - - - - - - - - - - -
Operator TypeCountsElapsed Time(us)
IndexPut212336.79
-
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12336.79 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
-
- -
-
-
- - -
-

AI Core Frequency Issues

-
- - Analysis of rank 4. - - Issue: 对于4号卡，在降频期间发现1个算子，频率降低比例超过了0.05。 Only show 10 operators here, see latest mstt_advisor.xlsx for details. -
- Suggestion: -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Operator name Count Total duration(us) AI CORE frequency decreased ratio Average frequency Max frequency Min frequency
aclnnInplaceFillScalar_FillAiCore_Fill1115.885.05%1709.091800.0800.0
- -
-
- - - - -
-

AI Core Performance Analysis

-
- - - - - Cube算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.59%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.58%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.92%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.9%
-
不亲和算子集合 - - - -
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
-
- - - - FA算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
-
- - - -
-
- - -
-
- -
-

stage-1

-
- Description: analysis for slow rank 5 in current stage -

- -
-

Operator Dynamic Shape Issues

-
- - Analysis of rank 5. - - - - - - - - - - -
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
-
-
- -
-

AICPU Issues

-
- - Analysis of rank 5. - - - - - - - - - - - - - - -
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： -IndexPut修改代码避免使用aicpu类算子12350.770.0002
-
- -
IndexPut
-
- - - - - - - - - - - -
Operator TypeCountsElapsed Time(us)
IndexPut212350.77
-
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12350.77 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
-
- -
-
-
- - - -
-

AI Core Performance Analysis

-
- - - - - Cube算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.65%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.49%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.55%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.85%
-
不亲和算子集合 - - - -
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
-
- - - - FA算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
-
- - - -
-
- - -
-
- -
-

stage-2

-
- Description: analysis for slow rank 6 in current stage -

- -
-

Operator Dynamic Shape Issues

-
- - Analysis of rank 6. - - - - - - - - - - -
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
-
-
- -
-

AICPU Issues

-
- - Analysis of rank 6. - - - - - - - - - - - - - - -
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： -IndexPut修改代码避免使用aicpu类算子12259.620.0024
-
- -
IndexPut
-
- - - - - - - - - - - -
Operator TypeCountsElapsed Time(us)
IndexPut212259.62
-
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12259.62 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
-
- -
-
-
- - - -
-

AI Core Performance Analysis

-
- - - - - Cube算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.69%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.49%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.94%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.95%
-
不亲和算子集合 - - - -
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
-
- - - - FA算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
-
- - - - Vector算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnInplaceCopy_TensorMoveAiCore_TensorMove3584,4736DT_BF1670.0%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove896,3584DT_BF1669.9%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove3584,896DT_BF1669.88%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,1,1,16384,128DT_BF1669.82%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,1,16384,128DT_BF1669.8%
-
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,4096,3584DT_BF16vec_mte2_mte3
aclnnMul_MulAiCore_Mul1,16384,4736;1,16384,4736DT_BF16;DT_BF16vec_mte2_mte3
aclnnMul_MulAiCore_Mul1,4096,3584;1,4096,3584FLOAT;FLOATvec_mte2_mte3
aclnnInplaceMul_CastAiCore_Cast16383,38016FLOATvec_mte2_mte3
aclnnInplaceMuls_MulAiCore_Mul8486912;FLOAT;FLOATvec_mte2_mte3
-
- -
-
- - -
-
- -
-

stage-3

-
- Description: analysis for slow rank 7 in current stage -

- -
-

Operator Dynamic Shape Issues

-
- - Analysis of rank 7. - - - - - - - - - - -
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
-
-
- -
-

AICPU Issues

-
- - Analysis of rank 7. - - - - - - - - - - - - - - -
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： -IndexPut修改代码避免使用aicpu类算子12304.890.0002
-
- -
IndexPut
-
- - - - - - - - - - - -
Operator TypeCountsElapsed Time(us)
IndexPut212304.89
-
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12304.89 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
-
- -
-
-
- - - -
-

AI Core Performance Analysis

-
- - - - - Cube算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.68%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.51%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.76%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.85%
-
不亲和算子集合 - - - -
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
-
- - - - FA算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
-
- - - -
-
- - -
-
- -
-

stage-4

-
- Description: analysis for slow rank 12 in current stage -

- -
-

Operator Dynamic Shape Issues

-
- - Analysis of rank 12. - - - - - - - - - - -
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
-
-
- -
-

AICPU Issues

-
- - Analysis of rank 12. - - - - - - - - - - - - - - -
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： -IndexPut修改代码避免使用aicpu类算子12328.430.0002
-
- -
IndexPut
-
- - - - - - - - - - - -
Operator TypeCountsElapsed Time(us)
IndexPut212328.43
-
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12328.43 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
-
- -
-
-
- - - -
-

AI Core Performance Analysis

-
- - - - - Cube算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.58%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.44%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.6%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF162.05%
-
不亲和算子集合 - - - -
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
-
- - - - FA算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
-
- - - -
-
- - -
-
- -
-

stage-5

-
- Description: analysis for slow rank 13 in current stage -

- -
-

Operator Dynamic Shape Issues

-
- - Analysis of rank 13. - - - - - - - - - - -
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
-
-
- -
-

AICPU Issues

-
- - Analysis of rank 13. - - - - - - - - - - - - - - -
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： -IndexPut修改代码避免使用aicpu类算子12306.050.0031
-
- -
IndexPut
-
- - - - - - - - - - - -
Operator TypeCountsElapsed Time(us)
IndexPut212306.05
-
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12306.05 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
-
- -
-
-
- - - -
-

AI Core Performance Analysis

-
- - - - - Cube算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.61%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.45%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.73%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.85%
-
不亲和算子集合 - - - -
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
-
- - - - FA算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
-
- - - - Vector算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnInplaceCopy_TensorMoveAiCore_TensorMove3584,4736DT_BF1670.0%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove3584,896DT_BF1669.89%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove896,3584DT_BF1669.89%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,1,1,16384,128DT_BF1669.82%
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,1,16384,128DT_BF1669.8%
-
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnInplaceCopy_TensorMoveAiCore_TensorMove1,4096,3584DT_BF16vec_mte2_mte3
aclnnMul_MulAiCore_Mul1,16384,4736;1,16384,4736DT_BF16;DT_BF16vec_mte2_mte3
aclnnMul_MulAiCore_Mul1,4096,3584;1,4096,3584FLOAT;FLOATvec_mte2_mte3
aclnnInplaceMul_CastAiCore_Cast16383,38016FLOATvec_mte2_mte3
aclnnInplaceMuls_MulAiCore_Mul8486912;FLOAT;FLOATvec_mte2_mte3
-
- -
-
- - -
-
- -
-

stage-6

-
- Description: analysis for slow rank 14 in current stage -

- -
-

Operator Dynamic Shape Issues

-
- - Analysis of rank 14. - - - - - - - - - - -
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
-
-
- -
-

AICPU Issues

-
- - Analysis of rank 14. - - - - - - - - - - - - - - -
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： -IndexPut修改代码避免使用aicpu类算子12324.210.0003
-
- -
IndexPut
-
- - - - - - - - - - - -
Operator TypeCountsElapsed Time(us)
IndexPut212324.21
-
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:2 | Elapsed Time(us):12324.21 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
-
- -
-
-
- - - -
-

AI Core Performance Analysis

-
- - - - - Cube算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.64%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.49%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.86%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.8%
-
不亲和算子集合 - - - -
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
-
- - - - FA算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
-
- - - -
-
- - -
-
- -
-

stage-7

-
- Description: analysis for slow rank 15 in current stage -

- -
-

Operator Dynamic Shape Issues

-
- - Analysis of rank 15. - - - - - - - - - - -
DescriptionSuggestion
找到所有是动态shape的算子在python脚本入口加入以下代码关闭在线编译：
'torch_npu.npu.set_compile_mode(jit_compile=False)
torch_npu.npu.config.allow_internal_format = False'
详细信息请参考：链接
-
-
- -
-

AICPU Issues

-
- - Analysis of rank 15. - - - - - - - - - - - - - - -
DescriptionSuggestionElapsed Time(us)Time Ratio
一些算子和任务执行时间超过了20us，比如： -IndexPut修改代码避免使用aicpu类算子12297.570.0002
-
- -
IndexPut
-
- - - - - - - - - - - -
Operator TypeCountsElapsed Time(us)
IndexPut212297.57
-
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:1 | Elapsed Time(us):11999.58 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(85): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
- IndexPut | Input DType:(INT64;INT64;INT64;INT64) | Output DType:(INT64) | Counts:1 | Elapsed Time(us):297.99 -
-
- -
- -

- Suggestion 1: 请参考链接修改源码，尝试用等价的算子替换indexput算子。 -

- -
- -
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(89): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): -
- -
-
- -
-
-
- - - -
-

AI Core Performance Analysis

-
- - - - - Cube算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
性能优化算子集合 - - - -
nameshapedtype 参考性能优化空间
aclnnAddmm_MatMulCommon_MatMulV216384,3584;896,3584;896DT_BF16;DT_BF16;FLOAT9.55%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;4736,3584DT_BF16;DT_BF169.5%
aclnnAddmm_MatMulCommon_MatMulV216384,3584;128,3584;128DT_BF16;DT_BF16;FLOAT4.62%
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;38016,3584DT_BF16;DT_BF161.75%
-
不亲和算子集合 - - - -
nameshapedtype 不亲和类型为
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;16384,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;4736,3584DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;16384,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,4736;3584,4736DT_BF16;DT_BF16内轴无法被256整除
aclnnMatmul_MatMulV3Common_MatMulV316384,3584;3584,4736DT_BF16;DT_BF16内轴无法被256整除
-
- - - - FA算子相关分析，参考如下: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
类别描述及建议
bound算子集合 - - - -
nameshapedtype bound类型为
aclnnFlashAttentionScoreGrad_FlashAttentionScoreGrad_FlashAttentionScoreGrad1,7,16384,128;1,7,16384,128;1,7,16384,128;1,7,16384,128;16384,16384;1,7,16384,8;1,7,16384,8;;1,7,16384,128;DT_BF16;DT_BF16;DT_BF16;DT_BF16;BOOL;FLOAT;FLOAT;DT_BF16;DT_BF16;INT64fixpipe
aclnnFlashAttentionScore_FlashAttentionScore_FlashAttentionScore1,7,16384,128;1,7,16384,128;1,7,16384,128;;;;16384,16384;;;;;Nonevec
-
- - - -
-
- - -
-
- -
-
- - -
-
- -
-

schedule

-
- - -
-

Conjectured GC Analysis

-
- - Analysis of rank 6. - - 在34079031.859us的空闲时间内几乎没有主机任务，这可能是由Python的异常GC引起的 - - - - - - - - - - - - - - - - - - - - - - - - - - -
Suggestions
1. 实现高效的Python内存管理；不使用时及时释放内存，避免长期占用；避免对象之间的循环引用。
2. 使用 gc.set_threshold() 来调整垃圾回收阈值可以延迟垃圾收集，但这是一个临时解决方案。
3. 使用 gc.disable() 来关闭GC，注意这是个临时解决方案。
- - The details of top 2 garbage collection events are as follows: -

- - - - - - - - - - - - - - - - - - - - - - - - - - -
timestamp duration(us)
1747647483551821.833818722.418
1747647606194246.2260309.441
- -
-
- - -
-

Affinity API Issues

-
- - Analysis of rank 6. - - The analysis results of following affinity APIs are based on runtime env - cann-8.0.0 - and - pytorch-pytorch - -
- - - - - - - -
torch_npu.npu_rms_norm
-
- -
- -
No.1 code stack, called 28 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(620): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.2 code stack, called 28 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.3 code stack, called 8 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.4 code stack, called 4 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.5 code stack, called 4 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.6 code stack, called 3 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.7 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.8 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/_ops.py(1116): __call__;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(582): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.9 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(620): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.10 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(158): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.11 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.12 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(250): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.13 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(598): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.14 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(272): collate_tensor_fn;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(155): collate;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(172): ;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(171): collate;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py(398): default_collate;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py(55): fetch;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/dataloader.py(757): _next_data;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/data/dataloader.py(701): __next__;
/usr/local/python3.10/lib/python3.10/site-packages/torch_npu/profiler/_add_mstx_patch.py(28): wrapper;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(74): load_batch;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(373): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.15 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.16 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.17 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.18 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.19 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.20 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(166): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.21 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3762): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.22 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.23 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(235): repeat_kv;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(573): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.24 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(564): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.25 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(180): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.26 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.27 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(172): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.28 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(566): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.29 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.30 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(236): repeat_kv;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(574): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.31 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.32 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.33 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(526): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.34 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(179): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.35 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1073): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1359): gather_forward_split_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1390): gather_sp_output;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(253): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.36 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(84): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(249): cross_entropy_1d;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/loss.py(334): dist_cross_entropy;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(360): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
-
- - - - - - -
torch_npu.npu_confusion_transpose
-
- -
- -
No.1 code stack, called 32 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(598): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.2 code stack, called 9 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(766): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.3 code stack, called 7 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(636): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.4 code stack, called 6 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.5 code stack, called 5 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.6 code stack, called 5 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(595): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.7 code stack, called 4 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(585): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.8 code stack, called 4 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(590): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.9 code stack, called 3 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.10 code stack, called 3 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.11 code stack, called 3 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(580): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.12 code stack, called 3 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(589): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.13 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(235): repeat_kv;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(574): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.14 code stack, called 2 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.15 code stack, called 2 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.16 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.17 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.18 code stack, called 2 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(597): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.19 code stack, called 2 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(638): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.20 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(528): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.21 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(157): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.22 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.23 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.24 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(163): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.25 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(205): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.26 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(206): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.27 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(566): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.28 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/nn/functional.py(2380): silu;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/activation.py(432): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.29 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(179): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.30 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/_ops.py(1116): __call__;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(582): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.31 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.32 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.33 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.34 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(79): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(620): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
-
- - - - - - -
torch_npu.npu_rotary_mul
-
- -
- -
No.1 code stack, called 28 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(179): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(206): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.2 code stack, called 28 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(179): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.3 code stack, called 10 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.4 code stack, called 5 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.5 code stack, called 4 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(518): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.6 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(166): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.7 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(235): repeat_kv;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(573): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.8 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.9 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(637): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.10 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(157): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.11 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.12 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/_ops.py(1116): __call__;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(582): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.13 code stack, called 2 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3341): all_gather;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(766): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.14 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.15 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(163): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.16 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(564): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(516): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.17 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): ;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1218): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.18 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1222): _gather;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(561): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.19 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(564): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1323): linear_gather_forward_reducescatter_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(345): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(517): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.20 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.21 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.22 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(170): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py(116): decorate_context;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(540): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.23 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(206): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.24 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(178): rotate_half;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(207): apply_rotary_pos_emb;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(541): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.25 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(739): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.26 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(748): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(223): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(638): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.27 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py(3757): reduce_scatter;
/usr/local/python3.10/lib/python3.10/site-packages/torch/distributed/c10d_logger.py(83): wrapper;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(751): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.28 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(575): apply;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(1339): linear_reducescatter_forward_gather_backward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/layer/linear.py(578): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(618): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(623): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(233): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.29 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(80): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(250): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.30 code stack, called 1 times
-
- /usr/local/python3.10/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py(81): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(250): qwen2_model_forward;
/home/duanjunwen/ColossalAI/colossalai/shardformer/modeling/qwen2.py(334): qwen2_for_causal_lm_forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/interface/model.py(30): forward;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(221): forward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1747): _call_impl;
/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py(1736): _wrapped_call_impl;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/_utils.py(126): model_forward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(270): forward_step;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(406): run_forward_backward;
/home/duanjunwen/ColossalAI/colossalai/pipeline/schedule/one_f_one_b.py(472): forward_backward_step;
/home/duanjunwen/ColossalAI/colossalai/booster/plugin/hybrid_parallel_plugin.py(1409): execute_pipeline;
/home/duanjunwen/ColossalAI/colossalai/booster/booster.py(221): execute_pipeline;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(134): test_hybrid_qwen;
/home/duanjunwen/ColossalAI/applications/ColossalChat/tests/test_hybrid.py(181): <module> -
- -
No.31 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(774): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
No.32 code stack, called 1 times
-
- /home/duanjunwen/ColossalAI/colossalai/shardformer/layer/_operation.py(811): backward;
/usr/local/python3.10/lib/python3.10/site-packages/torch/autograd/function.py(307): apply -
- -
-
- - - - -
- -
-
- - - - -
-
- -
-

dataloader

-
- -
-

Slow Dataloader Issues

-
- - Analysis of rank 6. - - dataloader加载数据速度较慢，一次迭代花费138000.9us，通常小于10000us。 - - - - - - - - - - - - - - -
Suggestions
1. 请检查数据目录的磁盘I/O。如果您正在ModelArts中训练模型，请将数据移动到“/cache”或装载更高效的云磁盘以获得更好的I/O。
2. 尝试调整dataloader参数'num_workers'。
- -
-
- -
-
- -
-
- - - -
-

Generated by Ascend Training Tools

-
-
- - - - - \ No newline at end of file diff --git a/applications/ColossalChat/profile_log.txt b/applications/ColossalChat/profile_log.txt deleted file mode 100644 index 20e56e0b2c33..000000000000 --- a/applications/ColossalChat/profile_log.txt +++ /dev/null @@ -1,278 +0,0 @@ -[2025-05-19 17:44:04][INFO] cluster analysis is in the process, please wait... -[2025-05-19 17:44:04][INFO] Begin generate communication data. -[2025-05-19 17:44:08][INFO] Communication data read completed. -Cluster analyzing: 0%| | 0/5[2025-05-19 17:44:09][INFO] HostInfoAnalysis completed -[2025-05-19 17:44:09][INFO] ClusterBaseInfoAnalysis skipped, since data type is not db -Cluster analyzing: 40%|██████████████████████████▊ | 2/5[2025-05-19 17:44:10][INFO] CommMatrixAnalysis completed -Cluster analyzing: 80%|█████████████████████████████████████████████████████▌ | 4/5[2025-05-19 17:44:12][INFO] CommunicationAnalysis completed -Cluster analyzing: 100%|███████████████████████████████████████████████████████████████████ | 5/5[2025-05-19 17:44:13][WARNING] StepTraceTimeAnalysis completed -Cluster analyzing: 100%|███████████████████████████████████████████████████████████████████ | 5/5 - - - -Cluster analyzing: 100%|███████████████████████████████████████████████████████████████████ | 5/5 -[2025-05-19 17:52:51][INFO] The cluster analysis result file has been generated: /home/duanjunwen/ColossalAI/applications/ColossalChat/train_profiling_data -[2025-05-19 17:52:51][INFO] Cluster has been analyzed because of the existence of cluster analysis output directory. -[2025-05-19 17:52:51][INFO] Skip Cluster analyze backend. -[2025-05-19 17:52:52][INFO] Start cluster schedule analysis -[2025-05-19 17:52:52][INFO] For cluster schedule analysis, maximum free for rank 6 and step 0 -[2025-05-19 17:52:52][INFO] Enable schedule comparison of fast and slow rank/step -[2025-05-19 17:52:52][INFO] Start cluster computation analysis -[2025-05-19 17:52:52][INFO] Steps and ranks to be analyzed of different pipeline parallel stages are {"stage-0": {"maximum": {"rank_id": 4, "step": 0}, "minimum": {"rank_id": 0, "step": 0}}, "stage-1": {"maximum": {"rank_id": 5, "step": 0}, "minimum": {"rank_id": 1, "step": 0}}, "stage-2": {"maximum": {"rank_id": 6, "step": 0}, "minimum": {"rank_id": 2, "step": 0}}, "stage-3": {"maximum": {"rank_id": 7, "step": 0}, "minimum": {"rank_id": 3, "step": 0}}, "stage-4": {"maximum": {"rank_id": 12, "step": 0}, "minimum": {"rank_id": 8, "step": 0}}, "stage-5": {"maximum": {"rank_id": 13, "step": 0}, "minimum": {"rank_id": 9, "step": 0}}, "stage-6": {"maximum": {"rank_id": 14, "step": 0}, "minimum": {"rank_id": 10, "step": 0}}, "stage-7": {"maximum": {"rank_id": 15, "step": 0}, "minimum": {"rank_id": 11, "step": 0}}} -[2025-05-19 17:52:52][INFO] For stage-0, slow rank is 4 -[2025-05-19 17:52:52][INFO] For stage-1, slow rank is 5 -[2025-05-19 17:52:52][INFO] For stage-2, slow rank is 6 -[2025-05-19 17:52:52][INFO] For stage-3, slow rank is 7 -[2025-05-19 17:52:52][INFO] For stage-4, slow rank is 12 -[2025-05-19 17:52:52][INFO] For stage-5, slow rank is 13 -[2025-05-19 17:52:52][INFO] For stage-6, slow rank is 14 -[2025-05-19 17:52:52][INFO] For stage-7, slow rank is 15 -[2025-05-19 17:52:52][INFO] Enable computation comparison of fast and slow rank/step in different pp stages -[2025-05-19 17:52:52][INFO] Start cluster communication analysis -[2025-05-19 17:52:52][INFO] Minimum SDMA bandwidth for rank 4 -[2025-05-19 17:52:52][INFO] Minimum RDMA bandwidth for rank 0 -[2025-05-19 17:52:52][INFO] Minimum SDMA bandwidth for rank 4 -[2025-05-19 17:52:52][INFO] Minimum RDMA bandwidth for rank 0 -[2025-05-19 17:52:52][INFO] Minimum SDMA bandwidth for rank 4 -[2025-05-19 17:52:52][INFO] Minimum RDMA bandwidth for rank 0 -[2025-05-19 17:52:52][INFO] Start cluster memory analysis -[2025-05-19 17:52:52][INFO] For cluster memory analysis, maximum free for rank 6 and step 0 -[2025-05-19 17:52:52][INFO] Start analysis EnvironmentVariableAnalyzer with environment_variable_dataset -[2025-05-19 17:52:52][WARNING] convert_to_int_with_exception: an empty string was encountered. -[2025-05-19 17:52:52][WARNING] convert_to_int_with_exception: an empty string was encountered. -[2025-05-19 17:54:39][INFO] Start analysis MemoryAnalyzer with timeline_event_dataset -[2025-05-19 17:55:24][INFO] Start analysis ByteAlignmentAnalyzer with ProfilingDataset -[2025-05-19 17:56:09][INFO] Start analysis BandwidthContentionAnalyzer with communication_dataset -[2025-05-19 17:56:11][INFO] Start analysis RDMARetransmissionAnalyzer with ClusterCommunicationDataset -[2025-05-19 17:56:11][INFO] Start analysis PacketAnalyzer with communication_dataset -[2025-05-19 17:56:11][WARNING] Analyser: ComparisonAnalyzer don't rely on any dataset! -[2025-05-19 17:56:11][WARNING] Analyser: PPStageComputationAnalyzer don't rely on any dataset! -[2025-05-19 17:56:57][INFO] Start analysis DynamicShapeAnalyzer with ProfilingDataset -[2025-05-19 17:57:43][INFO] Start analysis AicpuAnalyzer with ProfilingDataset -Building dataset for timeline analysis: 0%| | 0/2315055 [00:00= vocab_end_index) + masked_target = target.clone() - vocab_start_index + masked_target[target_mask] = 0 + return masked_target + +def code2(target, vocab_start_index, vocab_end_index): + """bool multiply""" + target_mask = (target < vocab_start_index) | (target >= vocab_end_index) + masked_target = target.clone() - vocab_start_index + masked_target *= ~target_mask + return masked_target + +def test_performance(): + batch_size = 8 + sizes = [4096, 8192, 16384, 32768, 131072] + code1_times = [] + code2_times = [] + + for size in sizes: + target = torch.randint(0, size, (batch_size, size,)).to("npu") + vocab_start_index = random.randint(0, size//2) + vocab_end_index = random.randint(size//2, size) + + # warmup + for _ in range(5): + code1(target, vocab_start_index, vocab_end_index) + code2(target, vocab_start_index, vocab_end_index) + + # Code 1: index input + start_time = time.time() + for _ in range(10): + code1(target, vocab_start_index, vocab_end_index) + code1_time = (time.time() - start_time) / 10 + code1_times.append(code1_time) + + # Code 2: bool multiply + start_time = time.time() + for _ in range(10): + code2(target, vocab_start_index, vocab_end_index) + code2_time = (time.time() - start_time) / 10 + code2_times.append(code2_time) + + print(f"DataSize: {size}") + print(f" Code 1:index input AvgRuntime: {code1_time:.6f} s") + print(f" Code 2:bool multiply AvgRuntime {code2_time:.6f} s") + # print(f" acceleration ratio: {(code1_time/code2_time-1)*100:.2f}%") + print(f" acceleration ratio: {(code1_time/code2_time - 1)*100:.2f}%") + + +if __name__ == "__main__": + print("\n===== Performance Benchmark =====") + test_performance() \ No newline at end of file diff --git a/colossalai/shardformer/layer/loss.py b/colossalai/shardformer/layer/loss.py index 7c43e3659901..cf138aeb0bb2 100644 --- a/colossalai/shardformer/layer/loss.py +++ b/colossalai/shardformer/layer/loss.py @@ -190,7 +190,8 @@ def forward( # mask mask = (target < down_threshold) | (target >= up_threshold) masked_target = target.clone() - down_threshold - masked_target[mask] = 0 + # masked_target[mask] = 0 + masked_target *= ~mask masked_target_1d = masked_target.view(-1).contiguous() handle.wait() diff --git a/colossalai/shardformer/modeling/qwen2.py b/colossalai/shardformer/modeling/qwen2.py index bb7d14966cb5..a8e561409f5d 100644 --- a/colossalai/shardformer/modeling/qwen2.py +++ b/colossalai/shardformer/modeling/qwen2.py @@ -218,6 +218,7 @@ def qwen2_model_forward( all_hidden_states += (hidden_states,) past_key_value = past_key_values[idx] if past_key_values is not None else None + print(f"######debug idx: {idx}") if idx - start_idx < num_ckpt_layers: layer_outputs = self._gradient_checkpointing_func( From 5085c5eaf117ce45dd6d98fbb1d57585013a127c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 May 2025 03:46:40 +0000 Subject: [PATCH 10/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../ColossalChat/ColossalaiRL_On_Ascend.md | 10 +-- .../coati/distributed/consumer.py | 14 ++-- .../coati/distributed/inference_backend.py | 2 +- .../ColossalChat/coati/distributed/launch.py | 21 +++--- .../coati/distributed/producer.py | 16 +++-- .../ColossalChat/coati/distributed/utils.py | 1 - applications/ColossalChat/rl_example.py | 4 +- .../ColossalChat/tests/test_hybrid.py | 70 +++++++++++-------- .../ColossalChat/tests/test_log_prob.py | 35 ++++++---- applications/ColossalChat/tests/test_ray.py | 11 ++- .../ColossalChat/tests/test_ray_vllm.py | 31 ++++---- applications/ColossalChat/tests/test_vllm.py | 23 ++++-- .../ColossalChat/tests/test_vllm_multinode.py | 19 ++--- colossalai/shardformer/layer/loss.py | 2 +- colossalai/shardformer/modeling/qwen2.py | 21 ++++-- colossalai/shardformer/policies/qwen2.py | 1 - 16 files changed, 164 insertions(+), 117 deletions(-) diff --git a/applications/ColossalChat/ColossalaiRL_On_Ascend.md b/applications/ColossalChat/ColossalaiRL_On_Ascend.md index 1b258137d40f..28cf637fd5a9 100644 --- a/applications/ColossalChat/ColossalaiRL_On_Ascend.md +++ b/applications/ColossalChat/ColossalaiRL_On_Ascend.md @@ -30,7 +30,7 @@ pip install ray==2.43.0 --no-cache-dir # Create soft-link from fuyao-ray to ray site-package cd .. -ln -s ./ray/python/ray/ /usr/local/python3.10/lib/python3.10/site-packages/ray +ln -s ./ray/python/ray/ /usr/local/python3.10/lib/python3.10/site-packages/ray # Install Fuyao Ray cd ray @@ -60,19 +60,19 @@ Then write IP node map to /etc/hosts 10.0.0.6 npu-6 ``` -### Set Ascend Multi-Node Config +### Set Ascend Multi-Node Config ```bash export ATB_LLM_HCCL_ENABLE=1 export ATB_LLM_COMM_BACKEND="hccl" export HCCL_CONNECT_TIMEOUT=7200 export WORLD_SIZE=32 -export HCCL_EXEC_TIMEOUT=7200 +export HCCL_EXEC_TIMEOUT=7200 export HCCL_SOCKET_IFNAME=eno0 -export RAY_COLLECTIVE_MEET_TIMEOUT_SECONDS=7200 +export RAY_COLLECTIVE_MEET_TIMEOUT_SECONDS=7200 ``` -## 3.Run task on ColossalaiRL-Ascend +## 3.Run task on ColossalaiRL-Ascend ### Start Ray Cluster Now we use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: diff --git a/applications/ColossalChat/coati/distributed/consumer.py b/applications/ColossalChat/coati/distributed/consumer.py index 9828ce6f15cd..f92dc6c06968 100644 --- a/applications/ColossalChat/coati/distributed/consumer.py +++ b/applications/ColossalChat/coati/distributed/consumer.py @@ -13,11 +13,11 @@ from colossalai.booster.plugin import HybridParallelPlugin from colossalai.initialize import launch from colossalai.nn.optimizer import HybridAdam -from colossalai.utils import get_current_device from .comm import ray_broadcast_tensor_dict from .utils import bind_batch, post_recv, unbind_batch + class BaseConsumer: def __init__( self, @@ -56,7 +56,7 @@ def __init__( self.plugin_config = plugin_config # self.device = get_current_device() - self.device = 'npu' + self.device = "npu" # self.device = torch.device(f"npu:{torch.npu.current_device()}") self.lr_scheduler = None self.generate_config = generate_config @@ -86,16 +86,18 @@ def setup(self) -> None: # Init Hybrid ray process group for i in range(self.num_producers): - cc.init_collective_group(self.world_size + 1, self.rank + 1, backend='hccl',group_name=f"sync_data_{i}") + cc.init_collective_group(self.world_size + 1, self.rank + 1, backend="hccl", group_name=f"sync_data_{i}") if self.pp_size > 1: # use hybrid tp + pp if self.tp_rank == 0 and self.dp_rank == 0: cc.init_collective_group( - self.num_producers + 1, self.num_producers, backend='hccl', group_name=f"sync_model_{self.pp_rank}" + self.num_producers + 1, self.num_producers, backend="hccl", group_name=f"sync_model_{self.pp_rank}" ) else: if self.rank == 0: - cc.init_collective_group(self.num_producers + 1, self.num_producers, backend='hccl', group_name="sync_model") + cc.init_collective_group( + self.num_producers + 1, self.num_producers, backend="hccl", group_name="sync_model" + ) self.buffer = [] self.recv_cnt = 0 @@ -161,7 +163,7 @@ def loop(self) -> None: f"[T{dist.get_rank()}] Sync model PP stage {self.pp_rank} episode {episode} step {step}" ) else: - print(f"[T{dist.get_rank()}] Sync model episode {episode} step {step}") + print(f"[T{dist.get_rank()}] Sync model episode {episode} step {step}") torch.cuda.empty_cache() state_dict = self.state_dict() if self.pp_size > 1: diff --git a/applications/ColossalChat/coati/distributed/inference_backend.py b/applications/ColossalChat/coati/distributed/inference_backend.py index a2bc3c000e67..7988802a3eaa 100644 --- a/applications/ColossalChat/coati/distributed/inference_backend.py +++ b/applications/ColossalChat/coati/distributed/inference_backend.py @@ -210,7 +210,7 @@ def __init__( self.model_config = model_config self.tokenizer = tokenizer self.num_generations = num_generations - self.max_length = generate_config['max_tokens'] + self.max_length = generate_config["max_tokens"] @torch.no_grad() def generate(self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwargs) -> Dict[str, torch.Tensor]: diff --git a/applications/ColossalChat/coati/distributed/launch.py b/applications/ColossalChat/coati/distributed/launch.py index 1d1b440a3590..6bb10f9e7ac1 100644 --- a/applications/ColossalChat/coati/distributed/launch.py +++ b/applications/ColossalChat/coati/distributed/launch.py @@ -65,7 +65,9 @@ def launch_distributed( core_consumer = ALGO_MAP.get(core_algo, SimpleConsumer) train_dp_size = get_dp_size_fast(num_consumer_procs, plugin_config) - print(f"inference_batch_size {inference_batch_size} num_producers {num_producers} train_batch_size {train_batch_size} train_dp_size {train_dp_size}") + print( + f"inference_batch_size {inference_batch_size} num_producers {num_producers} train_batch_size {train_batch_size} train_dp_size {train_dp_size}" + ) assert (inference_batch_size * num_producers) % (train_batch_size * train_dp_size) == 0 dataset_path = train_dataset_config["path"] @@ -73,7 +75,7 @@ def launch_distributed( global_inference_batch_size = inference_batch_size * num_producers num_update_per_episode = num_samples // global_inference_batch_size num_recv_per_update = inference_batch_size // inference_microbatch_size - + run_name = f"{inference_backend}_bs_{train_batch_size * train_dp_size}_temp_{generate_config['temperature']:.01f}_top_p_{generate_config['top_p']:.02f}" wandb_group_name = str(uuid.uuid4()) rollout_log_file = os.path.join( @@ -81,7 +83,6 @@ def launch_distributed( f"{project_name.replace(' ','_')}_run_{wandb_group_name}.jsonl", ) - # ########################################### # # Old version, may lead colossalai init stuck in multinodes # ############################################ @@ -136,7 +137,7 @@ def launch_distributed( # procs.append(consumer) # ray.get([p.setup.remote() for p in procs]) # ray.get([p.loop.remote() for p in procs]) - + ########################################### # New version, assign master ip for colossalai & vllm respectively ########################################### @@ -153,13 +154,13 @@ def launch_distributed( gpu_to_node_id = [] gpu_to_ip_address = [] for node_id in node_info: - for idx in range(int(node_info[node_id]["num_gpus"])): # use num_gpus instead of num_npus + for idx in range(int(node_info[node_id]["num_gpus"])): # use num_gpus instead of num_npus gpu_to_node_id.append(node_id) gpu_to_ip_address.append(node_info[node_id]["address"]) print(f"node_info {node_info} \n gpu_to_node_id {gpu_to_node_id} \n gpu_to_ip_address {gpu_to_ip_address} \n") producer_procs = [] - + for i in range(num_producers): node_id = gpu_to_node_id[0] producer_ip_address = gpu_to_ip_address[0] @@ -167,12 +168,12 @@ def launch_distributed( gpu_to_node_id.pop(0) gpu_to_ip_address.pop(0) print(f"Schedual Producer P[{i}] which requires {num_proc_per_producer} GPUs on node {producer_ip_address}") - + producer = SimpleProducer.options( # num_cpus=1, - # num_cpus=num_proc_per_producer, + # num_cpus=num_proc_per_producer, num_gpus=0, - resources={"NPU":num_proc_per_producer}, + resources={"NPU": num_proc_per_producer}, scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( node_id=node_id, soft=False, @@ -221,7 +222,7 @@ def launch_distributed( gpu_to_ip_address.pop(0) print(f"Schedual Consumer T[{i}] which requires 1 GPUs on node {consumer_ip_address}") consumer = core_consumer.options( - resources={"NPU":1}, + resources={"NPU": 1}, scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( node_id=node_id, soft=False, diff --git a/applications/ColossalChat/coati/distributed/producer.py b/applications/ColossalChat/coati/distributed/producer.py index 75dd2ee8858e..2911559929c2 100644 --- a/applications/ColossalChat/coati/distributed/producer.py +++ b/applications/ColossalChat/coati/distributed/producer.py @@ -11,7 +11,7 @@ from coati.dataset.loader import RawConversationDataset from coati.distributed.reward.reward_fn import boxed_math_reward_fn, math_reward_fn from ray.util.collective import allreduce -from ray.util.collective.types import Backend, ReduceOp +from ray.util.collective.types import ReduceOp from torch.utils.data import DataLoader, DistributedSampler from transformers import AutoTokenizer @@ -19,7 +19,7 @@ from .comm import ray_broadcast_tensor_dict from .inference_backend import BACKEND_MAP -from .utils import pre_send, safe_append_to_jsonl_file +from .utils import safe_append_to_jsonl_file try: from vllm import SamplingParams @@ -152,7 +152,7 @@ def __init__( print("No eval dataset provided, skip eval") self.device = get_current_device() # self.device = get_current_device() - self.device = 'npu' + self.device = "npu" # self.device = torch.device(f"npu:{torch.npu.current_device()}") # init backend @@ -164,12 +164,16 @@ def __init__( self.consumer_pp_size = consumer_plugin_config.get("pp_size", 1) # consumer pp size def setup(self) -> None: - cc.init_collective_group(1 + self.num_consumer_procs, 0, backend='hccl', group_name=f"sync_data_{self.producer_idx}") + cc.init_collective_group( + 1 + self.num_consumer_procs, 0, backend="hccl", group_name=f"sync_data_{self.producer_idx}" + ) if self.consumer_pp_size > 1: for i in range(self.consumer_pp_size): - cc.init_collective_group(self.num_producers + 1, self.producer_idx, backend='hccl', group_name=f"sync_model_{i}") + cc.init_collective_group( + self.num_producers + 1, self.producer_idx, backend="hccl", group_name=f"sync_model_{i}" + ) else: - cc.init_collective_group(self.num_producers + 1, self.producer_idx, backend='hccl', group_name="sync_model") + cc.init_collective_group(self.num_producers + 1, self.producer_idx, backend="hccl", group_name="sync_model") def rollout(self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwargs) -> Dict[str, torch.Tensor]: raise NotImplementedError diff --git a/applications/ColossalChat/coati/distributed/utils.py b/applications/ColossalChat/coati/distributed/utils.py index ce4685b3226b..a40ebbcfbe92 100644 --- a/applications/ColossalChat/coati/distributed/utils.py +++ b/applications/ColossalChat/coati/distributed/utils.py @@ -3,7 +3,6 @@ from typing import Any, Dict, List import torch -import math from filelock import FileLock from colossalai.shardformer.layer.loss import dist_log_prob diff --git a/applications/ColossalChat/rl_example.py b/applications/ColossalChat/rl_example.py index 016946e7f93a..472c13e94ff0 100644 --- a/applications/ColossalChat/rl_example.py +++ b/applications/ColossalChat/rl_example.py @@ -151,7 +151,9 @@ args.top_k = -1 inference_model_config = dict(path=args.model) - train_model_config = dict(path=args.model, use_flash_attention_2=False, use_cache=False, attn_implementation="eager") + train_model_config = dict( + path=args.model, use_flash_attention_2=False, use_cache=False, attn_implementation="eager" + ) generate_config = dict(top_k=args.top_k, top_p=args.top_p, temperature=args.temperature) if args.backend == "transformers": diff --git a/applications/ColossalChat/tests/test_hybrid.py b/applications/ColossalChat/tests/test_hybrid.py index d66999afd542..ec1bf4c3e312 100644 --- a/applications/ColossalChat/tests/test_hybrid.py +++ b/applications/ColossalChat/tests/test_hybrid.py @@ -1,10 +1,11 @@ import torch import torch.distributed as dist +import torch_npu from coati.dataset.loader import RawConversationDataset from torch.utils.data import Dataset from tqdm import tqdm from transformers import AutoTokenizer, Qwen2ForCausalLM -import torch_npu + import colossalai from colossalai.accelerator import get_accelerator from colossalai.booster import Booster @@ -21,6 +22,7 @@ MODEL_PATH = "/home/grpo/models/DeepSeek-R1-Distill-Qwen-7B" Device = torch.device("npu" if torch.npu.is_available() else "cpu") + class RandomDataset(Dataset): def __init__(self, num_samples, sequence_length, vocab_size=10000): self.num_samples = num_samples @@ -35,6 +37,7 @@ def __len__(self): def __getitem__(self, idx): return {"input_ids": self.input_idx[idx], "attention_mask": self.attention_mask[idx]} + def load_model_and_tokenizer(): attn_impl = "eager" if get_accelerator().name == "npu" else "flash_attention_2" tokenizer = AutoTokenizer.from_pretrained( @@ -45,40 +48,47 @@ def load_model_and_tokenizer(): model = Qwen2ForCausalLM.from_pretrained(MODEL_PATH, trust_remote_code=True) return tokenizer, model + def all_reduce_mean(loss: torch.Tensor, plugin: Plugin) -> torch.Tensor: loss = loss.data group = getattr(plugin, "dp_group", None) dist.all_reduce(loss, group=group) return loss / dist.get_world_size(group) + def test_hybrid_qwen(): colossalai.launch_from_torch() get_accelerator() coordinator = DistCoordinator() tokenizer, model = load_model_and_tokenizer() # dataset = RandomDataset(num_samples=100, sequence_length=2304) - dataset = RawConversationDataset(tokenizer, DATA_PATH, 16 * 1024, system_prompt="Please reason step by step, and put your final answer within \\boxed{}.") + dataset = RawConversationDataset( + tokenizer, + DATA_PATH, + 16 * 1024, + system_prompt="Please reason step by step, and put your final answer within \\boxed{}.", + ) # dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True) optimizer = HybridAdam(model.parameters(), lr=LEARNING_RATE) # plugin = HybridParallelPlugin( - # tp_size=8, - # pp_size=1, - # precision="bf16", - # zero_stage=2, + # tp_size=8, + # pp_size=1, + # precision="bf16", + # zero_stage=2, # cpu_offload=True, # ) plugin = HybridParallelPlugin( - tp_size=4, - pp_size=2, - sp_size=2, - enable_sequence_parallelism=True, - sequence_parallelism_mode="split_gather", - precision="bf16", - zero_stage=1, - microbatch_size=1, - max_norm= 1.0, - enable_flash_attention=True + tp_size=4, + pp_size=2, + sp_size=2, + enable_sequence_parallelism=True, + sequence_parallelism_mode="split_gather", + precision="bf16", + zero_stage=1, + microbatch_size=1, + max_norm=1.0, + enable_flash_attention=True, ) dataloader = plugin.prepare_dataloader( @@ -105,38 +115,35 @@ def is_master(): experimental_config = torch_npu.profiler._ExperimentalConfig( aic_metrics=torch_npu.profiler.AiCMetrics.PipeUtilization, profiler_level=torch_npu.profiler.ProfilerLevel.Level1, - l2_cache=False + l2_cache=False, ) prof = torch_npu.profiler.profile( - activities=[ - torch_npu.profiler.ProfilerActivity.CPU, - torch_npu.profiler.ProfilerActivity.NPU - ], + activities=[torch_npu.profiler.ProfilerActivity.CPU, torch_npu.profiler.ProfilerActivity.NPU], record_shapes=True, profile_memory=True, with_stack=True, experimental_config=experimental_config, schedule=torch_npu.profiler.schedule(wait=0, warmup=2, active=1, repeat=1), - on_trace_ready=torch_npu.profiler.tensorboard_trace_handler("./train_profiling_data") + on_trace_ready=torch_npu.profiler.tensorboard_trace_handler("./train_profiling_data"), ) for epoch in range(NUM_EPOCHS): if booster.plugin.pp_size > 1: data_iter = iter(dataloader) step_bar = tqdm( - range(len(dataloader)), - desc="Step", - disable=not is_master(), + range(len(dataloader)), + desc="Step", + disable=not is_master(), ) print(f"len step_bar {len(step_bar)}") for step in step_bar: print(f"Profile Start at step {step}") prof.start() outputs = booster.execute_pipeline( - data_iter, - model, - criterion=lambda outputs, inputs: outputs[0], - optimizer=optimizer, - return_loss=True, + data_iter, + model, + criterion=lambda outputs, inputs: outputs[0], + optimizer=optimizer, + return_loss=True, ) loss = outputs["loss"] print(f"step {step} loss {loss}") @@ -151,7 +158,7 @@ def is_master(): optimizer.step() optimizer.zero_grad() - + prof.step() else: total_loss = 0 @@ -177,5 +184,6 @@ def is_master(): print(f"Profile Stop") prof.stop() + if __name__ == "__main__": test_hybrid_qwen() diff --git a/applications/ColossalChat/tests/test_log_prob.py b/applications/ColossalChat/tests/test_log_prob.py index 0d7454070bdf..5e71f089cb4d 100644 --- a/applications/ColossalChat/tests/test_log_prob.py +++ b/applications/ColossalChat/tests/test_log_prob.py @@ -1,6 +1,8 @@ -import torch -import time import random +import time + +import torch + def code1(target, vocab_start_index, vocab_end_index): """index Put""" @@ -9,6 +11,7 @@ def code1(target, vocab_start_index, vocab_end_index): masked_target[target_mask] = 0 return masked_target + def code2(target, vocab_start_index, vocab_end_index): """bool multiply""" target_mask = (target < vocab_start_index) | (target >= vocab_end_index) @@ -16,36 +19,44 @@ def code2(target, vocab_start_index, vocab_end_index): masked_target *= ~target_mask return masked_target + def test_performance(): batch_size = 8 sizes = [4096, 8192, 16384, 32768, 131072] code1_times = [] code2_times = [] - + for size in sizes: - target = torch.randint(0, size, (batch_size, size,)).to("npu") - vocab_start_index = random.randint(0, size//2) - vocab_end_index = random.randint(size//2, size) - + target = torch.randint( + 0, + size, + ( + batch_size, + size, + ), + ).to("npu") + vocab_start_index = random.randint(0, size // 2) + vocab_end_index = random.randint(size // 2, size) + # warmup for _ in range(5): code1(target, vocab_start_index, vocab_end_index) code2(target, vocab_start_index, vocab_end_index) - + # Code 1: index input start_time = time.time() for _ in range(10): code1(target, vocab_start_index, vocab_end_index) code1_time = (time.time() - start_time) / 10 code1_times.append(code1_time) - + # Code 2: bool multiply start_time = time.time() for _ in range(10): code2(target, vocab_start_index, vocab_end_index) code2_time = (time.time() - start_time) / 10 code2_times.append(code2_time) - + print(f"DataSize: {size}") print(f" Code 1:index input AvgRuntime: {code1_time:.6f} s") print(f" Code 2:bool multiply AvgRuntime {code2_time:.6f} s") @@ -53,6 +64,6 @@ def test_performance(): print(f" acceleration ratio: {(code1_time/code2_time - 1)*100:.2f}%") -if __name__ == "__main__": +if __name__ == "__main__": print("\n===== Performance Benchmark =====") - test_performance() \ No newline at end of file + test_performance() diff --git a/applications/ColossalChat/tests/test_ray.py b/applications/ColossalChat/tests/test_ray.py index 16f5da507e88..9868c6ed56b9 100644 --- a/applications/ColossalChat/tests/test_ray.py +++ b/applications/ColossalChat/tests/test_ray.py @@ -1,11 +1,13 @@ -import ray import time + +import ray import ray.util.collective as cc import torch from coati.distributed.comm import ray_broadcast_object, ray_broadcast_tensor_dict from colossalai.testing import parameterize + @ray.remote(num_cpus=1, num_gpus=0, resources={"NPU": 1}) class Worker: def __init__(self, rank, world_size): @@ -13,6 +15,7 @@ def __init__(self, rank, world_size): self.world_size = world_size self.group_name = "default" cc.init_collective_group(world_size, rank, backend="hccl", group_name=self.group_name) + def run_ray_broadcast_object(self, obj, src, device): # ray_broadcast_object received_obj = ray_broadcast_object(obj, src, device, group_name=self.group_name) @@ -26,6 +29,7 @@ def run_ray_broadcast_tensor_dict(self, tensor_dict, src, device): def destroy_worker(self): cc.destroy_collective_group(self.group_name) + @parameterize( "test_config", [ @@ -37,7 +41,7 @@ def destroy_worker(self): ], ) def test_comm(test_config): - #ray.init() + # ray.init() ray.init(address="local", namespace="ray-example") # ray.init(_node_ip_address='10.0.0.5', namespace="ray-example") @@ -84,5 +88,6 @@ def test_comm(test_config): worker.destroy_worker.remote() ray.shutdown() + if __name__ == "__main__": - test_comm() \ No newline at end of file + test_comm() diff --git a/applications/ColossalChat/tests/test_ray_vllm.py b/applications/ColossalChat/tests/test_ray_vllm.py index a0d1270db229..37ea241dee4c 100644 --- a/applications/ColossalChat/tests/test_ray_vllm.py +++ b/applications/ColossalChat/tests/test_ray_vllm.py @@ -1,22 +1,25 @@ -import ray +import argparse import time + +import ray import ray.util.collective as cc import torch -from coati.distributed.comm import ray_broadcast_object, ray_broadcast_tensor_dict +from coati.distributed.comm import ray_broadcast_tensor_dict +from vllm import LLM, SamplingParams from colossalai.testing import parameterize -from vllm import LLM, SamplingParams -import torch -import argparse - -parser = argparse.ArgumentParser(description='VLLM args.') -parser.add_argument("-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. ") +parser = argparse.ArgumentParser(description="VLLM args.") +parser.add_argument( + "-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. " +) parser.add_argument("-l", "--max_length", type=int, default=8192, help="Max sequence length") parser.add_argument("-w", "--world_size", type=int, default=8, help="Gpu nums") parser.add_argument("-t", "--temperature", type=float, default=0.8, help="Temperature") parser.add_argument("--top_p", type=float, default=0.95, help="Top p") -parser.add_argument("-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. ") +parser.add_argument( + "-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. " +) args = parser.parse_args() # Create a sampling params object. @@ -30,7 +33,9 @@ def __init__(self, rank, world_size): self.group_name = "default" cc.init_collective_group(world_size, rank, backend="hccl", group_name=self.group_name) self.llm = LLM(model=args.model_path, max_model_len=args.max_length, tensor_parallel_size=args.world_size) - self.sampling_params = SamplingParams(temperature=args.temperature, top_p=args.top_p, max_tokens=args.max_length) + self.sampling_params = SamplingParams( + temperature=args.temperature, top_p=args.top_p, max_tokens=args.max_length + ) def run_ray_broadcast_object(self, obj, src, device): # Create an LLM. @@ -45,6 +50,7 @@ def run_ray_broadcast_tensor_dict(self, tensor_dict, src, device): def destroy_worker(self): cc.destroy_collective_group(self.group_name) + @parameterize( "test_config", [ @@ -75,13 +81,13 @@ def test_comm(test_config): torch.npu.synchronize() start_time = time.time() results = [worker.run_ray_broadcast_object.remote(test_obj, src, device) for worker in workers] - + # get result results = ray.get(results) end_time = time.time() total_time = end_time - start_time - + print(f"total_time {total_time}") for i, result in enumerate(results): @@ -92,5 +98,6 @@ def test_comm(test_config): worker.destroy_worker.remote() ray.shutdown() + if __name__ == "__main__": test_comm() diff --git a/applications/ColossalChat/tests/test_vllm.py b/applications/ColossalChat/tests/test_vllm.py index 325ddc0a9693..fc24cf1222de 100644 --- a/applications/ColossalChat/tests/test_vllm.py +++ b/applications/ColossalChat/tests/test_vllm.py @@ -1,22 +1,31 @@ -from vllm import LLM, SamplingParams -import torch import argparse -parser = argparse.ArgumentParser(description='VLLM args.') -parser.add_argument("-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. ") +from vllm import LLM, SamplingParams + +parser = argparse.ArgumentParser(description="VLLM args.") +parser.add_argument( + "-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. " +) parser.add_argument("-l", "--max_length", type=int, default=8192, help="Max sequence length") parser.add_argument("-tp", "--tp_size", type=int, default=8, help="Gpu nums") parser.add_argument("-pp", "--pp_size", type=int, default=2, help="Gpu nums") parser.add_argument("-t", "--temperature", type=float, default=0.8, help="Temperature") parser.add_argument("--top_p", type=float, default=0.95, help="Top p") -parser.add_argument("-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. ") +parser.add_argument( + "-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. " +) args = parser.parse_args() # Create a sampling params object. sampling_params = SamplingParams(temperature=args.temperature, top_p=args.top_p, max_tokens=args.max_length) # Create an LLM. -llm = LLM(model=args.model_path, max_model_len=args.max_length, tensor_parallel_size=args.tp_size, pipeline_parallel_size=args.pp_size) +llm = LLM( + model=args.model_path, + max_model_len=args.max_length, + tensor_parallel_size=args.tp_size, + pipeline_parallel_size=args.pp_size, +) # Generate texts from the prompts. The output is a list of RequestOutput objects # that contain the prompt, generated text, and other information. outputs = llm.generate(args.input_texts, sampling_params) @@ -24,4 +33,4 @@ for output in outputs: prompt = output.prompt generated_text = output.outputs[0].text - print(f"Prompt: {prompt!r}, Generated text: {generated_text}") \ No newline at end of file + print(f"Prompt: {prompt!r}, Generated text: {generated_text}") diff --git a/applications/ColossalChat/tests/test_vllm_multinode.py b/applications/ColossalChat/tests/test_vllm_multinode.py index 0434c48e1e92..41c241890968 100644 --- a/applications/ColossalChat/tests/test_vllm_multinode.py +++ b/applications/ColossalChat/tests/test_vllm_multinode.py @@ -11,11 +11,9 @@ import ray from packaging.version import Version from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy - from vllm import LLM, SamplingParams -assert Version(ray.__version__) >= Version( - "2.22.0"), "Ray version must be at least 2.22.0" +assert Version(ray.__version__) >= Version("2.22.0"), "Ray version must be at least 2.22.0" # Create a sampling params object. sampling_params = SamplingParams(temperature=0.8, top_p=0.95) @@ -32,8 +30,7 @@ class LLMPredictor: def __init__(self): # Create an LLM. - self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf", - tensor_parallel_size=tensor_parallel_size) + self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf", tensor_parallel_size=tensor_parallel_size) def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]: # Generate texts from the prompts. @@ -44,7 +41,7 @@ def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]: generated_text: List[str] = [] for output in outputs: prompt.append(output.prompt) - generated_text.append(' '.join([o.text for o in output.outputs])) + generated_text.append(" ".join([o.text for o in output.outputs])) return { "prompt": prompt, "generated_text": generated_text, @@ -61,14 +58,10 @@ def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]: def scheduling_strategy_fn(): # One bundle per tensor parallel worker pg = ray.util.placement_group( - [{ - "GPU": 1, - "CPU": 1 - }] * tensor_parallel_size, + [{"GPU": 1, "CPU": 1}] * tensor_parallel_size, strategy="STRICT_PACK", ) - return dict(scheduling_strategy=PlacementGroupSchedulingStrategy( - pg, placement_group_capture_child_tasks=True)) + return dict(scheduling_strategy=PlacementGroupSchedulingStrategy(pg, placement_group_capture_child_tasks=True)) resources_kwarg: Dict[str, Any] = {} @@ -105,4 +98,4 @@ def scheduling_strategy_fn(): # Multiple files would be written to the output destination, # and each task would write one or more files separately. # -# ds.write_parquet("s3://") \ No newline at end of file +# ds.write_parquet("s3://") diff --git a/colossalai/shardformer/layer/loss.py b/colossalai/shardformer/layer/loss.py index cf138aeb0bb2..1ef7f219a7e0 100644 --- a/colossalai/shardformer/layer/loss.py +++ b/colossalai/shardformer/layer/loss.py @@ -194,7 +194,7 @@ def forward( masked_target *= ~mask masked_target_1d = masked_target.view(-1).contiguous() handle.wait() - + ################## # Step3:Calculate global summation exp logits ################## diff --git a/colossalai/shardformer/modeling/qwen2.py b/colossalai/shardformer/modeling/qwen2.py index a8e561409f5d..78d1ba00f0e0 100644 --- a/colossalai/shardformer/modeling/qwen2.py +++ b/colossalai/shardformer/modeling/qwen2.py @@ -144,15 +144,15 @@ def qwen2_model_forward( # for the other stages, hidden_states is the output of the previous stage if shard_config.enable_flash_attention: # in this case, attention_mask is a dict rather than a tensor - mask_shape = (batch_size, 1, seq_length, seq_length_with_past) + (batch_size, 1, seq_length, seq_length_with_past) attention_mask = None - #attention_mask = ColoAttention.prepare_attn_kwargs( + # attention_mask = ColoAttention.prepare_attn_kwargs( # mask_shape, # hidden_states.dtype, # hidden_states.device, # q_padding_mask=attention_mask, # is_causal=True, - #) + # ) else: if self._attn_implementation == "flash_attention_2": # 2d mask is passed through the layers @@ -523,7 +523,7 @@ def forward( key_states = all_to_all_comm(key_states, sp_group, fp8_communication=shard_config.fp8_communication) value_states = all_to_all_comm(value_states, sp_group, fp8_communication=shard_config.fp8_communication) bsz, q_len, _ = query_states.size() - + query_states = query_states.view(bsz, q_len, self.num_heads, -1).transpose(1, 2) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, -1).transpose(1, 2) value_states = value_states.view(bsz, q_len, self.num_key_value_heads, -1).transpose(1, 2) @@ -580,7 +580,16 @@ def forward( diagonal=1, ).to(dtype=torch.bool, device="npu") scale = 1.0 / math.sqrt(query_states.shape[-1]) - attn_output = torch_npu.npu_fusion_attention(query_states, key_states, value_states, head_num=query_states.size(1), input_layout="BNSD", sparse_mode=1, atten_mask=atten_mask, scale = scale) + attn_output = torch_npu.npu_fusion_attention( + query_states, + key_states, + value_states, + head_num=query_states.size(1), + input_layout="BNSD", + sparse_mode=1, + atten_mask=atten_mask, + scale=scale, + ) attn_output = attn_output[0] else: attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) @@ -623,8 +632,6 @@ def forward( return forward - - def get_qwen2_flash_attention_forward(shard_config: ShardConfig, sp_mode=None, sp_size=None, sp_group=None): def forward( self: Qwen2Attention, diff --git a/colossalai/shardformer/policies/qwen2.py b/colossalai/shardformer/policies/qwen2.py index add00901d551..823527df61dc 100644 --- a/colossalai/shardformer/policies/qwen2.py +++ b/colossalai/shardformer/policies/qwen2.py @@ -19,7 +19,6 @@ from ..modeling.qwen2 import ( Qwen2PipelineForwards, get_lm_forward_with_dist_cross_entropy, - get_qwen2_flash_attention_forward, get_qwen2_flash_attention_npu_forward, get_qwen2_model_forward_for_flash_attn, ) From 260a25ac985af5ae7c5495bf4e127259512b59a8 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Fri, 23 May 2025 13:22:58 +0800 Subject: [PATCH 11/24] [feat] rm comments in qwen modeling --- colossalai/shardformer/modeling/qwen2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/colossalai/shardformer/modeling/qwen2.py b/colossalai/shardformer/modeling/qwen2.py index 78d1ba00f0e0..0923cc5956a4 100644 --- a/colossalai/shardformer/modeling/qwen2.py +++ b/colossalai/shardformer/modeling/qwen2.py @@ -218,7 +218,6 @@ def qwen2_model_forward( all_hidden_states += (hidden_states,) past_key_value = past_key_values[idx] if past_key_values is not None else None - print(f"######debug idx: {idx}") if idx - start_idx < num_ckpt_layers: layer_outputs = self._gradient_checkpointing_func( From 4c18679966b2bcbfcc3e4da28b929e2285022c55 Mon Sep 17 00:00:00 2001 From: YeAnbang <44796419+YeAnbang@users.noreply.github.com> Date: Fri, 23 May 2025 15:56:52 +0800 Subject: [PATCH 12/24] [Doc] Drafted README.md --- .../ColossalChat/coati/distributed/README.md | 180 +++++++++++++++++- 1 file changed, 177 insertions(+), 3 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index b7bac2b2db93..5e93a564d241 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -1,6 +1,180 @@ -# Requirements +Here's a clean and detailed `README.md` for your distributed RL framework: + +--- + +# Distributed RL Framework for Language Model Fine-Tuning + +This repository implements a distributed Reinforcement Learning (RL) training framework designed to fine-tune large language models using algorithms such as **GRPO** and **DAPO**. It supports multi-node and multi-GPU setups, scalable rollout generation, and policy optimization using libraries like HuggingFace Transformers or VLLM. + +--- + +## 🚀 Features + +* **Distributed Training with Ray**: Scalable to multiple machines and GPUs. +* **Support for GRPO and DAPO**: Choose your preferred policy optimization algorithm. +* **Flexible Model Backends**: Choose between `transformers` and `vllm` backends. +* **Rollout and Policy Decoupling**: Efficient generation and consumption of data through parallel inferencer-trainer architecture. +* **Evaluation Integration**: Easily plug in task-specific eval datasets. +* **Checkpoints and Logging**: Configurable intervals and directories. + +--- + +## 🛠 Installation + +Please fill this section + +## 🧠 Data Format + +Each data sample in the training or evaluation `.jsonl` file should follow this format: + +```json +{ + "messages": { + "role": "user", + "content": "Simplify $\\sqrt[3]{1+8} \\cdot \\sqrt[3]{1+\\sqrt[3]{8}}$. Let's think step by step and output the final answer within \\boxed{}." + }, + "gt_answer": "3" +} +``` + +--- + +## ⚙️ Hyperparameters & Arguments + +| Argument | Description | Example | +| ---------------- | --------------------------------------- | ----------------- | +| `--model` | Model path or identifier | `/path/to/model` | +| `--dataset` | Path to training `.jsonl` | `/path/to/train_data.jsonl` | +| `--eval-dataset` | JSON of task\:eval\_dataset\_path pairs | `{'eval_1':'/path/to/eval_1.jsonl'}` | +| `--project` | Project name | `Project1` | +| `--num-episodes` | Number of training episodes | `1` | + +### Distributed Training + +| Argument | Description | Example | +| ----------------------------- | ------------------------------------- | ------- | +| `--num-trainers` | Number of trainer processes | `4` | +| `--num-inferencer` | Number of inferencer processes | `4` | +| `--inference-batch-size` | Prompts per inference step | `8` | +| `--inference-microbatch-size` | Per-GPU batch size for inference | `8` | +| `--train-batch-size` | Prompts per trainer step per dp group | `8` | +| `--train-minibatch-size` | Mini-batch size before forward pass | `8` | +| `--train-microbatch-size` | Per-GPU batch size for training | `2` | + +### Sampling + +| Argument | Description | Example | +| --------------------- | --------------------- | -------------- | +| `--backend` | Generation backend, choose from `vllm` `transformers` | `vllm` | +| `--temperature` | Sampling temperature for generation | `1.0` | +| `--top-k` | Top-K sampling parameter for generation | `None` | +| `--top-p` | Top-P sampling parameter for generation | `1.0` | +| `--system-prompt` | System prompt, default to the system prompt for `think_answer_tags` format | `Please reason step by step, and put your final answer within \\boxed{}.` | +| `--max-new-tokens` | Max generation tokens | `3584` | +| `--max-prompt-tokens` | Max prompt tokens | `512` | + +### GRPO Specific + +| Argument | Description | Example | +| ----------------- | ---------------------------- | ------------------- | +| `--algo` | Algorithm (`GRPO` or `DAPO`), for more customization refer to [GRPO Settings](#️-grpo-settings) | `GRPO` | +| `--learning-rate` | Learning rate | `1e-6` | +| `--kl-coeff` | KL penalty coefficient | `0.01` | +| `--reward-type` | Reward signal type (choose from 'think_answer_tags', 'boxed') | `think_answer_tags` | +| `--eval-interval` | Evaluation interval in number of training steps (positive value to enable evaluation) | `100` | + +### Logging and Checkpointing + +| Argument | Description | Example | +| -------------------- | ------------------------- | ------------ | +| `--save-interval` | Training steps between checkpoints | `20` | +| `--save-dir` | Checkpoint directory | `./model` | +| `--eval-save-dir` | Evaluation save path | `./eval` | +| `--rollout-save-dir` | Rollout logs directory | `./rollouts` | + +### Miscellaneous + +| Argument | Description | Example | +| ------------------ | --------------------------------------- | ------- | +| `--ray_dir` | Custom Ray temp dir of a running Ray cluster (optional) | `None` | +| `--master_address` | Master address of a running Ray cluster | `None` | +| `--master_port` | Master port for torch DDP | `29506` | + +--- + +## ⚙️ GRPO Settings + +In addition to the two default training settings we provided--- original `GRPO` and `DAPO`, users can customize their training by changing the following hyperparameters in `grpo_config` in `rl_example.py`. + +| Argument Name | Description | Default | +| ----------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `filter_range` | Filters out rollout group if the success rate within that group is out of this range.| `[0.01, 0.99]` | +| `dynamic_batching` | Enables dynamic batching as described in the [DAPO paper](https://arxiv.org/abs/2503.14476). | `True` | +| `clip_eps_low` | epsilon_low in DAPO in equation in [DAPO paper](https://arxiv.org/abs/2503.14476) | `0.2` | +| `clip_eps_high` | epsilon_high in DAPO equation in [DAPO paper](https://arxiv.org/abs/2503.14476) | `0.28` | +| `skip_threshold` | If ratio is above this threshold, the sample is skipped to avoid instability. | `20.0` | +| `loss_variation` | Type of loss variation. Supports `"token_level"` for token-wise policy gradient loss and `sample_level` for original GRPO loss. | `"token_level"` | +| `soft_over_length_punishment` | Whether to use soft overlength penalty in [DAPO paper](https://arxiv.org/abs/2503.14476) or not. | `True` | +| `cache_length` | `L_cache` parameter for soft overlength penalty in e.q. 13 in [DAPO paper](https://arxiv.org/abs/2503.14476) | `min(1024, int(args.max_new_tokens / 4))` | +| `filter_truncated_response` | Mask out truncated responses in loss calculation. | `True` | + + + +## 🔄 Constraints and Notes + +* `num_inferencer + num_trainer == NUM_GPUs` +* `num_inferencer % num_trainer == 0` +* `(num_inferencer * inference_batch_size) % (num_trainer * train_batch_size) == 0` +* `train_batch_size >= train_minibatch_size >= train_microbatch_size` +* `inference_batch_size >= inference_microbatch_size` +* Set microbatch sizes based on **VRAM capacity** +* To use tensor parallelism on inferencer + * set backend to `vllm` + * change `tensor_parallel_size` in `inference_model_config` in rl_example.py + * set `num_inferencer = NUM_INFERENCE_GPUs / tensor_parallel_size` +* To set tensor parallelism / pipeline parallelism / zero stage + * change corresponding settings in `plugin_config` in rl_example.py +* Ensure rollout generation rate matches trainer consumption: + + ``` + num_inferencer * inference_batch_size % ( + num_trainer * train_batch_size / + train_pipeline_parallelism_size / + train_tensor_parallelism_size + ) == 0 + ``` +* Model weights sync every: + + ``` + (num_inferencer * inference_batch_size) / + (num_trainer * train_batch_size / + train_pipeline_parallelism_size / + train_tensor_parallelism_size) + ``` + +--- + +## 🧪 Example: single machine 8-GPU Zero2 Strategy ```bash -pip install cupy-cuda12x -python -m cupyx.tools.install_library --cuda 12.x --library nccl +python rl_example.py \ + --dataset /path/to/train_data.jsonl \ + --model /path/to/Qwen2.5-Math-7B/ \ + -t 4 -i 4 \ + -b vllm \ + -a DAPO \ + -ibs 8 -tbs 8 -e 2 \ + -rt boxed \ + -si 15 \ + -s "Please reason step by step, and put your final answer within \\boxed{}." \ + -tMbs 8 \ + -p GRPO-Reward-Debug \ + -ei 5 \ + -ed '{"Math_500_level_1": "path/to/math_500_level_1.jsonl", "Ma1h_500_level_3": "path/to/math_500_level_3.jsonl"}' ``` + +## 🧪 Example: multi-machine TP+PP Strategy + +Please add examples for starting ray cluster and training +--- + From 9a511aa9713fca033b7528f031b88873f11761ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 May 2025 07:57:47 +0000 Subject: [PATCH 13/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- applications/ColossalChat/coati/distributed/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index 5e93a564d241..1f67979b9723 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -177,4 +177,3 @@ python rl_example.py \ Please add examples for starting ray cluster and training --- - From 61aa1fc826141faa60973eca9bc8965639c3fa21 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Fri, 23 May 2025 16:21:28 +0800 Subject: [PATCH 14/24] [feat] fix ascend readme format --- .../ColossalChat/ColossalaiRL_On_Ascend.md | 95 ---------------- .../ColossalChat/coati/distributed/README.md | 107 +++++++++++++++++- 2 files changed, 101 insertions(+), 101 deletions(-) delete mode 100644 applications/ColossalChat/ColossalaiRL_On_Ascend.md diff --git a/applications/ColossalChat/ColossalaiRL_On_Ascend.md b/applications/ColossalChat/ColossalaiRL_On_Ascend.md deleted file mode 100644 index 28cf637fd5a9..000000000000 --- a/applications/ColossalChat/ColossalaiRL_On_Ascend.md +++ /dev/null @@ -1,95 +0,0 @@ -# ColossalaiRL On Ascend -The document is the instructions for using ColossalRL on Ascend. - -## 1.Prepare Develop Environment - -### Install Colossalai & ColossalChat -```bash -git clone https://github.com/hpcaitech/ColossalAI.git -git checkout grpo-latest -pip install -e . - -cd ./applications/ColossalChat -pip install -e . -``` - -### Install Fuyao Ray -Please update CANN before install fuyao ray -```bash -# Install CANN -source /usr/local/Ascend/ascend-toolkit/set_env.sh -./Ascend-cann-kernels-910b_8.1.RC1.alpha001_linux-aarch64.run --devel - -# Clone Fuyao Ray -git clone https://gitee.com/openfuyao/ray.git -cd ray -git pull origin pull/5/head - -# Install ray -pip install ray==2.43.0 --no-cache-dir - -# Create soft-link from fuyao-ray to ray site-package -cd .. -ln -s ./ray/python/ray/ /usr/local/python3.10/lib/python3.10/site-packages/ray - -# Install Fuyao Ray -cd ray -python python/ray/setup-dev.py -``` -### Prepare Model & dataset - -```bash -huggingface-cli download --local-dir-use-symlinks False Qwen/Qwen2.5-7B --local-dir /models/Qwen/Qwen2.5-7B -``` - - -## 2.Set Distributed Config -Now, we need to set distributed config for multi-node. - -### Set Host IP Config -First, we set host ip config. -For example. I need to configure a cluster of 4 nodes, then I do -```bash -vim /etc/hosts -``` -Then write IP node map to /etc/hosts -```bash -10.0.0.3 npu-3 -10.0.0.4 npu-4 -10.0.0.5 npu-5 -10.0.0.6 npu-6 -``` - -### Set Ascend Multi-Node Config - -```bash -export ATB_LLM_HCCL_ENABLE=1 -export ATB_LLM_COMM_BACKEND="hccl" -export HCCL_CONNECT_TIMEOUT=7200 -export WORLD_SIZE=32 -export HCCL_EXEC_TIMEOUT=7200 -export HCCL_SOCKET_IFNAME=eno0 -export RAY_COLLECTIVE_MEET_TIMEOUT_SECONDS=7200 -``` - -## 3.Run task on ColossalaiRL-Ascend - -### Start Ray Cluster -Now we use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: -```bash -ray start --head --node-ip-address=10.0.0.3 -``` -Then, for each slave node (10.0.0.4/10.0.0.5/10.0.0.6), we add to the ray cluser by following code: -```bash -ray start --address='10.0.0.3:6379' -``` - -### Run Scripts -Then, run start command at master node -```bash -# Hint1: replace /models/Qwen/Qwen2.5-7B to your model path -# replace /datasets/train-alignment.jsonl to your dataset path -python rl_example.py -m /models/Qwen/Qwen2.5-7B -d /datasets/train-alignment.jsonl --master_address '10.0.0.3' -t 16 -i 16 -p GRPO-Train-Align-Debug -g 2 -ibs 1 -tbs 2 -tMbs 1 -tmbs 2 -imbs 1 -b vllm -e 2 -rt boxed -s "Please reason step by step, and put your final answer within \\boxed{}." &>run_log.log & -``` - - diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index 1f67979b9723..5e4922deb703 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -1,7 +1,3 @@ -Here's a clean and detailed `README.md` for your distributed RL framework: - ---- - # Distributed RL Framework for Language Model Fine-Tuning This repository implements a distributed Reinforcement Learning (RL) training framework designed to fine-tune large language models using algorithms such as **GRPO** and **DAPO**. It supports multi-node and multi-GPU setups, scalable rollout generation, and policy optimization using libraries like HuggingFace Transformers or VLLM. @@ -21,7 +17,73 @@ This repository implements a distributed Reinforcement Learning (RL) training fr ## 🛠 Installation -Please fill this section +### Prepare Develop Environment + +Install Colossalai & ColossalChat +```bash +git clone https://github.com/hpcaitech/ColossalAI.git +git checkout grpo-latest +pip install -e . + +cd ./applications/ColossalChat +pip install -e . +``` +Install Fuyao Ray. +Please update CANN before install fuyao ray +```bash +# Install CANN +source /usr/local/Ascend/ascend-toolkit/set_env.sh +./Ascend-cann-kernels-910b_8.1.RC1.alpha001_linux-aarch64.run --devel + +# Clone Fuyao Ray +git clone https://gitee.com/openfuyao/ray.git +cd ray +git pull origin pull/5/head + +# Install ray +pip install ray==2.43.0 --no-cache-dir + +# Create soft-link from fuyao-ray to ray site-package +cd .. +ln -s ./ray/python/ray/ /usr/local/python3.10/lib/python3.10/site-packages/ray + +# Install Fuyao Ray +cd ray +python python/ray/setup-dev.py +``` +Prepare Model & dataset + +```bash +huggingface-cli download --local-dir-use-symlinks False Qwen/Qwen2.5-7B --local-dir /models/Qwen/Qwen2.5-7B +``` + +### Set Distributed Config +Now, we need to set distributed config for multi-node. + +First, we set host ip config. +For example. I need to configure a cluster of 4 nodes, then I do +```bash +vim /etc/hosts +``` +Then write IP node map to /etc/hosts +```bash +10.0.0.3 npu-3 +10.0.0.4 npu-4 +10.0.0.5 npu-5 +10.0.0.6 npu-6 +``` + +Set Ascend Multi-Node Config + +```bash +export ATB_LLM_HCCL_ENABLE=1 +export ATB_LLM_COMM_BACKEND="hccl" +export HCCL_CONNECT_TIMEOUT=7200 +export WORLD_SIZE=32 +export HCCL_EXEC_TIMEOUT=7200 +export HCCL_SOCKET_IFNAME=eno0 +export RAY_COLLECTIVE_MEET_TIMEOUT_SECONDS=7200 +``` ## 🧠 Data Format @@ -175,5 +237,38 @@ python rl_example.py \ ## 🧪 Example: multi-machine TP+PP Strategy -Please add examples for starting ray cluster and training +### Create ray cluster on multi-machine + +Now we use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: +```bash +ray start --head --node-ip-address=10.0.0.3 +``` + +Then, for each slave node (10.0.0.4/10.0.0.5/10.0.0.6), we add to the ray cluser by following code: +```bash +ray start --address='10.0.0.3:6379' +``` + +```bash +# Hint1: replace /models/Qwen/Qwen2.5-7B to your model path +# replace /datasets/train-alignment.jsonl to your dataset path +python rl_example.py +-m /path/to/Qwen2.5-Math-7B/ \ +-d /path/to/train_data.jsonl \ +--master_address '10.0.0.3' +-t 16 \ +-i 16 \ +-p GRPO-Train-Align-Debug \ +-g 2 \ +-ibs 1 \ +-tbs 2 \ +-tMbs 1 \ +-tmbs 2 \ +-imbs 1 \ +-b vllm \ +-e 2 \ +-rt boxed \ +-s "Please reason step by step, and put your final answer within \\boxed{}." +``` + --- From e4eedf9e628f09458eea08a6dc513b2fc19e696e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 May 2025 08:24:09 +0000 Subject: [PATCH 15/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../ColossalChat/coati/distributed/README.md | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index 5e4922deb703..68060028caf4 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -237,7 +237,7 @@ python rl_example.py \ ## 🧪 Example: multi-machine TP+PP Strategy -### Create ray cluster on multi-machine +### Create ray cluster on multi-machine Now we use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: ```bash @@ -252,23 +252,23 @@ ray start --address='10.0.0.3:6379' ```bash # Hint1: replace /models/Qwen/Qwen2.5-7B to your model path # replace /datasets/train-alignment.jsonl to your dataset path -python rl_example.py +python rl_example.py -m /path/to/Qwen2.5-Math-7B/ \ --d /path/to/train_data.jsonl \ ---master_address '10.0.0.3' --t 16 \ --i 16 \ --p GRPO-Train-Align-Debug \ --g 2 \ --ibs 1 \ --tbs 2 \ --tMbs 1 \ --tmbs 2 \ --imbs 1 \ --b vllm \ --e 2 \ --rt boxed \ --s "Please reason step by step, and put your final answer within \\boxed{}." +-d /path/to/train_data.jsonl \ +--master_address '10.0.0.3' +-t 16 \ +-i 16 \ +-p GRPO-Train-Align-Debug \ +-g 2 \ +-ibs 1 \ +-tbs 2 \ +-tMbs 1 \ +-tmbs 2 \ +-imbs 1 \ +-b vllm \ +-e 2 \ +-rt boxed \ +-s "Please reason step by step, and put your final answer within \\boxed{}." ``` --- From 2688954728f182f9877bb89317c80286243e7558 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Mon, 26 May 2025 18:12:37 +0800 Subject: [PATCH 16/24] [fix] fix readme --- .../ColossalChat/coati/distributed/README.md | 77 +++++++++++-------- applications/ColossalChat/requirements.txt | 10 ++- applications/ColossalChat/rl_example.py | 6 -- colossalai/shardformer/layer/loss.py | 3 +- colossalai/shardformer/modeling/qwen2.py | 8 +- 5 files changed, 58 insertions(+), 46 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index 68060028caf4..54f13a631f12 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -8,7 +8,7 @@ This repository implements a distributed Reinforcement Learning (RL) training fr * **Distributed Training with Ray**: Scalable to multiple machines and GPUs. * **Support for GRPO and DAPO**: Choose your preferred policy optimization algorithm. -* **Flexible Model Backends**: Choose between `transformers` and `vllm` backends. +* **Model Backends**: Support `vllm` as inference backends. * **Rollout and Policy Decoupling**: Efficient generation and consumption of data through parallel inferencer-trainer architecture. * **Evaluation Integration**: Easily plug in task-specific eval datasets. * **Checkpoints and Logging**: Configurable intervals and directories. @@ -22,7 +22,7 @@ This repository implements a distributed Reinforcement Learning (RL) training fr Install Colossalai & ColossalChat ```bash git clone https://github.com/hpcaitech/ColossalAI.git -git checkout grpo-latest +git checkout grpo-latest-ascend pip install -e . cd ./applications/ColossalChat @@ -35,7 +35,7 @@ Please update CANN before install fuyao ray source /usr/local/Ascend/ascend-toolkit/set_env.sh ./Ascend-cann-kernels-910b_8.1.RC1.alpha001_linux-aarch64.run --devel -# Clone Fuyao Ray +# Clone Fuyao Ray. Fuyao Ray is not an open source project, it will be inherited in the ColossalRL images. git clone https://gitee.com/openfuyao/ray.git cd ray git pull origin pull/5/head @@ -51,8 +51,8 @@ ln -s ./ray/python/ray/ /usr/local/python3.10/lib/python3.10/site-packages/ray cd ray python python/ray/setup-dev.py ``` -Prepare Model & dataset +Prepare Model & dataset ```bash huggingface-cli download --local-dir-use-symlinks False Qwen/Qwen2.5-7B --local-dir /models/Qwen/Qwen2.5-7B ``` @@ -221,25 +221,27 @@ In addition to the two default training settings we provided--- original `GRPO` ```bash python rl_example.py \ --dataset /path/to/train_data.jsonl \ - --model /path/to/Qwen2.5-Math-7B/ \ + --model /path/to/Qwen2.5-3B/ \ -t 4 -i 4 \ -b vllm \ - -a DAPO \ - -ibs 8 -tbs 8 -e 2 \ + -ibs 2 -tbs 4 -tMbs 1 -tmbs 4 -imbs 1 \ -rt boxed \ - -si 15 \ + -g 4 \ + -ibs 1 \ + -tbs 2 \ + -tMbs 1 \ + -tmbs 2 \ + -imbs 1 \ -s "Please reason step by step, and put your final answer within \\boxed{}." \ -tMbs 8 \ - -p GRPO-Reward-Debug \ - -ei 5 \ - -ed '{"Math_500_level_1": "path/to/math_500_level_1.jsonl", "Ma1h_500_level_3": "path/to/math_500_level_3.jsonl"}' + -p GRPO-Train-Align-Debug \ ``` ## 🧪 Example: multi-machine TP+PP Strategy -### Create ray cluster on multi-machine - -Now we use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: +### Create ray cluster on multi-machine +For example, now we have 4 nodes and their IPs are 10.0.0.3, 10.0.0.4, 10.0.0.5, 10.0.0.6. +We use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: ```bash ray start --head --node-ip-address=10.0.0.3 ``` @@ -249,26 +251,39 @@ Then, for each slave node (10.0.0.4/10.0.0.5/10.0.0.6), we add to the ray cluser ray start --address='10.0.0.3:6379' ``` +Modify plugin_config in ./applications/ColossalChat/rl_example.py +```python +plugin_config={ + "tp_size": 4, + "pp_size": 2, + "microbatch_size": max( + 1, args.train_microbatch_size // 2 + ), # microbatch size should be set to train_microbatch_size // pp_size + "zero_stage": 1, + "max_norm": 1.0, + }, # for pp, tp +``` + ```bash # Hint1: replace /models/Qwen/Qwen2.5-7B to your model path # replace /datasets/train-alignment.jsonl to your dataset path -python rl_example.py --m /path/to/Qwen2.5-Math-7B/ \ --d /path/to/train_data.jsonl \ ---master_address '10.0.0.3' --t 16 \ --i 16 \ --p GRPO-Train-Align-Debug \ --g 2 \ --ibs 1 \ --tbs 2 \ --tMbs 1 \ --tmbs 2 \ --imbs 1 \ --b vllm \ --e 2 \ --rt boxed \ --s "Please reason step by step, and put your final answer within \\boxed{}." +python rl_example.py + -m /path/to/Qwen2.5-Math-7B/ \ + -d /path/to/train_data.jsonl \ + --master_address '10.0.0.3' + -t 16 \ + -i 16 \ + -p GRPO-Train-Align-Debug \ + -g 2 \ + -ibs 1 \ + -tbs 2 \ + -tMbs 1 \ + -tmbs 2 \ + -imbs 1 \ + -b vllm \ + -e 2 \ + -rt boxed \ + -s "Please reason step by step, and put your final answer within \\boxed{}." ``` --- diff --git a/applications/ColossalChat/requirements.txt b/applications/ColossalChat/requirements.txt index 472080101b9b..849a6228877d 100755 --- a/applications/ColossalChat/requirements.txt +++ b/applications/ColossalChat/requirements.txt @@ -1,9 +1,9 @@ -transformers==4.39.3 +transformers==4.47.0 tqdm datasets==2.14.7 loralib colossalai>=0.4.7 -torch>=2.1.0 +torch==2.5.1 langchain tokenizers fastapi @@ -22,3 +22,9 @@ sentencepiece==0.1.99 flash-attn tiktoken jsonlines +math-verify==0.7.0 + +# The following packages be built into the image. +# torch_npu==2.5.1 +# fuyao-ray==2.43.0 +# vllm-ascend==0.7.3 \ No newline at end of file diff --git a/applications/ColossalChat/rl_example.py b/applications/ColossalChat/rl_example.py index 472c13e94ff0..6be66c1c6e03 100644 --- a/applications/ColossalChat/rl_example.py +++ b/applications/ColossalChat/rl_example.py @@ -248,9 +248,6 @@ num_generations=args.num_generations, train_model_config=train_model_config, grpo_config=grpo_config, - # plugin_config={ - # "zero_stage": 2, - # }, # for zero plugin_config={ "tp_size": 2, "pp_size": 2, @@ -259,9 +256,6 @@ ), # microbatch size should be set to train_microbatch_size // pp_size "zero_stage": 1, "max_norm": 1.0, - # "sp_size": 4, - # "enable_sequence_parallelism":True, - # "sequence_parallelism_mode":"split_gather" # ["split_gather", "ring", "all_to_all"] }, # for pp, tp inference_backend=args.backend, master_addr="localhost", diff --git a/colossalai/shardformer/layer/loss.py b/colossalai/shardformer/layer/loss.py index 1ef7f219a7e0..a9bb76fc7d6b 100644 --- a/colossalai/shardformer/layer/loss.py +++ b/colossalai/shardformer/layer/loss.py @@ -190,8 +190,7 @@ def forward( # mask mask = (target < down_threshold) | (target >= up_threshold) masked_target = target.clone() - down_threshold - # masked_target[mask] = 0 - masked_target *= ~mask + masked_target[mask] = 0 masked_target_1d = masked_target.view(-1).contiguous() handle.wait() diff --git a/colossalai/shardformer/modeling/qwen2.py b/colossalai/shardformer/modeling/qwen2.py index 0923cc5956a4..67d77757df95 100644 --- a/colossalai/shardformer/modeling/qwen2.py +++ b/colossalai/shardformer/modeling/qwen2.py @@ -574,10 +574,6 @@ def forward( value_states = repeat_kv(value_states, self.num_key_value_groups) if shard_config.enable_flash_attention: - atten_mask = torch.triu( - torch.ones(q_len, q_len), - diagonal=1, - ).to(dtype=torch.bool, device="npu") scale = 1.0 / math.sqrt(query_states.shape[-1]) attn_output = torch_npu.npu_fusion_attention( query_states, @@ -586,8 +582,10 @@ def forward( head_num=query_states.size(1), input_layout="BNSD", sparse_mode=1, - atten_mask=atten_mask, + atten_mask=None, scale=scale, + pre_tockens=65536, + next_tockens=65536, ) attn_output = attn_output[0] else: From 86715658b1b7fb0283894f1012eaddf11125b5dd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 May 2025 10:15:44 +0000 Subject: [PATCH 17/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../ColossalChat/coati/distributed/README.md | 48 +++++++++---------- applications/ColossalChat/requirements.txt | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index 54f13a631f12..d8740aaa32fa 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -226,12 +226,12 @@ python rl_example.py \ -b vllm \ -ibs 2 -tbs 4 -tMbs 1 -tmbs 4 -imbs 1 \ -rt boxed \ - -g 4 \ - -ibs 1 \ - -tbs 2 \ - -tMbs 1 \ - -tmbs 2 \ - -imbs 1 \ + -g 4 \ + -ibs 1 \ + -tbs 2 \ + -tMbs 1 \ + -tmbs 2 \ + -imbs 1 \ -s "Please reason step by step, and put your final answer within \\boxed{}." \ -tMbs 8 \ -p GRPO-Train-Align-Debug \ @@ -239,7 +239,7 @@ python rl_example.py \ ## 🧪 Example: multi-machine TP+PP Strategy -### Create ray cluster on multi-machine +### Create ray cluster on multi-machine For example, now we have 4 nodes and their IPs are 10.0.0.3, 10.0.0.4, 10.0.0.5, 10.0.0.6. We use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: ```bash @@ -251,7 +251,7 @@ Then, for each slave node (10.0.0.4/10.0.0.5/10.0.0.6), we add to the ray cluser ray start --address='10.0.0.3:6379' ``` -Modify plugin_config in ./applications/ColossalChat/rl_example.py +Modify plugin_config in ./applications/ColossalChat/rl_example.py ```python plugin_config={ "tp_size": 4, @@ -267,23 +267,23 @@ plugin_config={ ```bash # Hint1: replace /models/Qwen/Qwen2.5-7B to your model path # replace /datasets/train-alignment.jsonl to your dataset path -python rl_example.py +python rl_example.py -m /path/to/Qwen2.5-Math-7B/ \ - -d /path/to/train_data.jsonl \ - --master_address '10.0.0.3' - -t 16 \ - -i 16 \ - -p GRPO-Train-Align-Debug \ - -g 2 \ - -ibs 1 \ - -tbs 2 \ - -tMbs 1 \ - -tmbs 2 \ - -imbs 1 \ - -b vllm \ - -e 2 \ - -rt boxed \ - -s "Please reason step by step, and put your final answer within \\boxed{}." + -d /path/to/train_data.jsonl \ + --master_address '10.0.0.3' + -t 16 \ + -i 16 \ + -p GRPO-Train-Align-Debug \ + -g 2 \ + -ibs 1 \ + -tbs 2 \ + -tMbs 1 \ + -tmbs 2 \ + -imbs 1 \ + -b vllm \ + -e 2 \ + -rt boxed \ + -s "Please reason step by step, and put your final answer within \\boxed{}." ``` --- diff --git a/applications/ColossalChat/requirements.txt b/applications/ColossalChat/requirements.txt index 849a6228877d..e1b8291aba49 100755 --- a/applications/ColossalChat/requirements.txt +++ b/applications/ColossalChat/requirements.txt @@ -27,4 +27,4 @@ math-verify==0.7.0 # The following packages be built into the image. # torch_npu==2.5.1 # fuyao-ray==2.43.0 -# vllm-ascend==0.7.3 \ No newline at end of file +# vllm-ascend==0.7.3 From 3c14c8457a903cea1b0ea2df5ec9d3aff8b0c2ab Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Mon, 26 May 2025 18:27:18 +0800 Subject: [PATCH 18/24] [fix] fix readme --- .../ColossalChat/coati/distributed/README.md | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index d8740aaa32fa..d7b1350c248c 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -1,6 +1,6 @@ # Distributed RL Framework for Language Model Fine-Tuning -This repository implements a distributed Reinforcement Learning (RL) training framework designed to fine-tune large language models using algorithms such as **GRPO** and **DAPO**. It supports multi-node and multi-GPU setups, scalable rollout generation, and policy optimization using libraries like HuggingFace Transformers or VLLM. +This repository implements a distributed Reinforcement Learning (RL) training framework designed to fine-tune large language models using algorithms such as **GRPO** and **DAPO**. It supports multi-node and multi-GPU setups, scalable rollout generation, and policy optimization using libraries like VLLM. --- @@ -127,7 +127,7 @@ Each data sample in the training or evaluation `.jsonl` file should follow this | Argument | Description | Example | | --------------------- | --------------------- | -------------- | -| `--backend` | Generation backend, choose from `vllm` `transformers` | `vllm` | +| `--backend` | Generation backend, choose from `vllm` | `vllm` | | `--temperature` | Sampling temperature for generation | `1.0` | | `--top-k` | Top-K sampling parameter for generation | `None` | | `--top-p` | Top-P sampling parameter for generation | `1.0` | @@ -226,12 +226,12 @@ python rl_example.py \ -b vllm \ -ibs 2 -tbs 4 -tMbs 1 -tmbs 4 -imbs 1 \ -rt boxed \ - -g 4 \ - -ibs 1 \ - -tbs 2 \ - -tMbs 1 \ - -tmbs 2 \ - -imbs 1 \ + -g 4 \ + -ibs 1 \ + -tbs 2 \ + -tMbs 1 \ + -tmbs 2 \ + -imbs 1 \ -s "Please reason step by step, and put your final answer within \\boxed{}." \ -tMbs 8 \ -p GRPO-Train-Align-Debug \ @@ -239,7 +239,7 @@ python rl_example.py \ ## 🧪 Example: multi-machine TP+PP Strategy -### Create ray cluster on multi-machine +### Create ray cluster on multi-machine For example, now we have 4 nodes and their IPs are 10.0.0.3, 10.0.0.4, 10.0.0.5, 10.0.0.6. We use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: ```bash @@ -251,7 +251,7 @@ Then, for each slave node (10.0.0.4/10.0.0.5/10.0.0.6), we add to the ray cluser ray start --address='10.0.0.3:6379' ``` -Modify plugin_config in ./applications/ColossalChat/rl_example.py +Modify plugin_config in ./applications/ColossalChat/rl_example.py ```python plugin_config={ "tp_size": 4, @@ -267,23 +267,23 @@ plugin_config={ ```bash # Hint1: replace /models/Qwen/Qwen2.5-7B to your model path # replace /datasets/train-alignment.jsonl to your dataset path -python rl_example.py +python rl_example.py -m /path/to/Qwen2.5-Math-7B/ \ - -d /path/to/train_data.jsonl \ - --master_address '10.0.0.3' - -t 16 \ - -i 16 \ - -p GRPO-Train-Align-Debug \ - -g 2 \ - -ibs 1 \ - -tbs 2 \ - -tMbs 1 \ - -tmbs 2 \ - -imbs 1 \ - -b vllm \ - -e 2 \ - -rt boxed \ - -s "Please reason step by step, and put your final answer within \\boxed{}." + -d /path/to/train_data.jsonl \ + --master_address '10.0.0.3' + -t 16 \ + -i 16 \ + -p GRPO-Train-Align-Debug \ + -g 2 \ + -ibs 1 \ + -tbs 2 \ + -tMbs 1 \ + -tmbs 2 \ + -imbs 1 \ + -b vllm \ + -e 2 \ + -rt boxed \ + -s "Please reason step by step, and put your final answer within \\boxed{}." ``` --- From b264299738c42772828949af48ca93be58fc6426 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 May 2025 10:29:28 +0000 Subject: [PATCH 19/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../ColossalChat/coati/distributed/README.md | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index d7b1350c248c..73e9abdc7540 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -226,12 +226,12 @@ python rl_example.py \ -b vllm \ -ibs 2 -tbs 4 -tMbs 1 -tmbs 4 -imbs 1 \ -rt boxed \ - -g 4 \ - -ibs 1 \ - -tbs 2 \ - -tMbs 1 \ - -tmbs 2 \ - -imbs 1 \ + -g 4 \ + -ibs 1 \ + -tbs 2 \ + -tMbs 1 \ + -tmbs 2 \ + -imbs 1 \ -s "Please reason step by step, and put your final answer within \\boxed{}." \ -tMbs 8 \ -p GRPO-Train-Align-Debug \ @@ -239,7 +239,7 @@ python rl_example.py \ ## 🧪 Example: multi-machine TP+PP Strategy -### Create ray cluster on multi-machine +### Create ray cluster on multi-machine For example, now we have 4 nodes and their IPs are 10.0.0.3, 10.0.0.4, 10.0.0.5, 10.0.0.6. We use 10.0.0.3 as master node. First we start a ray cluster on 10.0.0.3: ```bash @@ -251,7 +251,7 @@ Then, for each slave node (10.0.0.4/10.0.0.5/10.0.0.6), we add to the ray cluser ray start --address='10.0.0.3:6379' ``` -Modify plugin_config in ./applications/ColossalChat/rl_example.py +Modify plugin_config in ./applications/ColossalChat/rl_example.py ```python plugin_config={ "tp_size": 4, @@ -267,23 +267,23 @@ plugin_config={ ```bash # Hint1: replace /models/Qwen/Qwen2.5-7B to your model path # replace /datasets/train-alignment.jsonl to your dataset path -python rl_example.py +python rl_example.py -m /path/to/Qwen2.5-Math-7B/ \ - -d /path/to/train_data.jsonl \ - --master_address '10.0.0.3' - -t 16 \ - -i 16 \ - -p GRPO-Train-Align-Debug \ - -g 2 \ - -ibs 1 \ - -tbs 2 \ - -tMbs 1 \ - -tmbs 2 \ - -imbs 1 \ - -b vllm \ - -e 2 \ - -rt boxed \ - -s "Please reason step by step, and put your final answer within \\boxed{}." + -d /path/to/train_data.jsonl \ + --master_address '10.0.0.3' + -t 16 \ + -i 16 \ + -p GRPO-Train-Align-Debug \ + -g 2 \ + -ibs 1 \ + -tbs 2 \ + -tMbs 1 \ + -tmbs 2 \ + -imbs 1 \ + -b vllm \ + -e 2 \ + -rt boxed \ + -s "Please reason step by step, and put your final answer within \\boxed{}." ``` --- From d0a6fedf96cde5bc4812143009c49c6302e65787 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Tue, 27 May 2025 13:58:02 +0800 Subject: [PATCH 20/24] [fix] fix Readme, rm irrelevant testcase --- .../ColossalChat/coati/distributed/README.md | 2 + .../ColossalChat/tests/test_hybrid.py | 189 ------------------ .../ColossalChat/tests/test_log_prob.py | 69 ------- applications/ColossalChat/tests/test_ray.py | 93 --------- .../ColossalChat/tests/test_ray_vllm.py | 103 ---------- applications/ColossalChat/tests/test_vllm.py | 36 ---- .../ColossalChat/tests/test_vllm_multinode.py | 101 ---------- 7 files changed, 2 insertions(+), 591 deletions(-) delete mode 100644 applications/ColossalChat/tests/test_hybrid.py delete mode 100644 applications/ColossalChat/tests/test_log_prob.py delete mode 100644 applications/ColossalChat/tests/test_ray.py delete mode 100644 applications/ColossalChat/tests/test_ray_vllm.py delete mode 100644 applications/ColossalChat/tests/test_vllm.py delete mode 100644 applications/ColossalChat/tests/test_vllm_multinode.py diff --git a/applications/ColossalChat/coati/distributed/README.md b/applications/ColossalChat/coati/distributed/README.md index 73e9abdc7540..e0773d838d1a 100644 --- a/applications/ColossalChat/coati/distributed/README.md +++ b/applications/ColossalChat/coati/distributed/README.md @@ -286,4 +286,6 @@ python rl_example.py -s "Please reason step by step, and put your final answer within \\boxed{}." ``` +## Acknowledgement + --- diff --git a/applications/ColossalChat/tests/test_hybrid.py b/applications/ColossalChat/tests/test_hybrid.py deleted file mode 100644 index ec1bf4c3e312..000000000000 --- a/applications/ColossalChat/tests/test_hybrid.py +++ /dev/null @@ -1,189 +0,0 @@ -import torch -import torch.distributed as dist -import torch_npu -from coati.dataset.loader import RawConversationDataset -from torch.utils.data import Dataset -from tqdm import tqdm -from transformers import AutoTokenizer, Qwen2ForCausalLM - -import colossalai -from colossalai.accelerator import get_accelerator -from colossalai.booster import Booster -from colossalai.booster.plugin import HybridParallelPlugin, Plugin -from colossalai.cluster import DistCoordinator -from colossalai.nn.optimizer import HybridAdam - -BATCH_SIZE = 2 -NUM_EPOCHS = 1 -LEARNING_RATE = 2e-5 -GRADIENT_ACCUMULATION_STEPS = 1 -DATA_PATH = "/home/duanjunwen/datasets/math_dataset.jsonl" -DATA_PATH = "/home/duanjunwen/datasets/train-alignment_10.jsonl" -MODEL_PATH = "/home/grpo/models/DeepSeek-R1-Distill-Qwen-7B" -Device = torch.device("npu" if torch.npu.is_available() else "cpu") - - -class RandomDataset(Dataset): - def __init__(self, num_samples, sequence_length, vocab_size=10000): - self.num_samples = num_samples - self.sequence_length = sequence_length - self.vocab_size = vocab_size - self.input_idx = torch.randint(0, vocab_size, (num_samples, sequence_length)) - self.attention_mask = torch.randint(0, 2, (num_samples, sequence_length), dtype=torch.long) - - def __len__(self): - return self.num_samples - - def __getitem__(self, idx): - return {"input_ids": self.input_idx[idx], "attention_mask": self.attention_mask[idx]} - - -def load_model_and_tokenizer(): - attn_impl = "eager" if get_accelerator().name == "npu" else "flash_attention_2" - tokenizer = AutoTokenizer.from_pretrained( - MODEL_PATH, - trust_remote_code=True, - attn_implementation=attn_impl, - ) - model = Qwen2ForCausalLM.from_pretrained(MODEL_PATH, trust_remote_code=True) - return tokenizer, model - - -def all_reduce_mean(loss: torch.Tensor, plugin: Plugin) -> torch.Tensor: - loss = loss.data - group = getattr(plugin, "dp_group", None) - dist.all_reduce(loss, group=group) - return loss / dist.get_world_size(group) - - -def test_hybrid_qwen(): - colossalai.launch_from_torch() - get_accelerator() - coordinator = DistCoordinator() - tokenizer, model = load_model_and_tokenizer() - # dataset = RandomDataset(num_samples=100, sequence_length=2304) - dataset = RawConversationDataset( - tokenizer, - DATA_PATH, - 16 * 1024, - system_prompt="Please reason step by step, and put your final answer within \\boxed{}.", - ) - # dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True) - - optimizer = HybridAdam(model.parameters(), lr=LEARNING_RATE) - # plugin = HybridParallelPlugin( - # tp_size=8, - # pp_size=1, - # precision="bf16", - # zero_stage=2, - # cpu_offload=True, - # ) - plugin = HybridParallelPlugin( - tp_size=4, - pp_size=2, - sp_size=2, - enable_sequence_parallelism=True, - sequence_parallelism_mode="split_gather", - precision="bf16", - zero_stage=1, - microbatch_size=1, - max_norm=1.0, - enable_flash_attention=True, - ) - - dataloader = plugin.prepare_dataloader( - dataset=dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - ) - - booster = Booster(plugin=plugin) - - model, optimizer, _, dataloader, _ = booster.boost(model, optimizer, None, dataloader) - - def is_master(): - if isinstance(plugin, HybridParallelPlugin) and plugin.pp_size > 1: - return coordinator.rank == coordinator.world_size - 1 - return coordinator.is_master() - - ##### - # train - ##### - model.train() - model.gradient_checkpointing = False - experimental_config = torch_npu.profiler._ExperimentalConfig( - aic_metrics=torch_npu.profiler.AiCMetrics.PipeUtilization, - profiler_level=torch_npu.profiler.ProfilerLevel.Level1, - l2_cache=False, - ) - prof = torch_npu.profiler.profile( - activities=[torch_npu.profiler.ProfilerActivity.CPU, torch_npu.profiler.ProfilerActivity.NPU], - record_shapes=True, - profile_memory=True, - with_stack=True, - experimental_config=experimental_config, - schedule=torch_npu.profiler.schedule(wait=0, warmup=2, active=1, repeat=1), - on_trace_ready=torch_npu.profiler.tensorboard_trace_handler("./train_profiling_data"), - ) - for epoch in range(NUM_EPOCHS): - if booster.plugin.pp_size > 1: - data_iter = iter(dataloader) - step_bar = tqdm( - range(len(dataloader)), - desc="Step", - disable=not is_master(), - ) - print(f"len step_bar {len(step_bar)}") - for step in step_bar: - print(f"Profile Start at step {step}") - prof.start() - outputs = booster.execute_pipeline( - data_iter, - model, - criterion=lambda outputs, inputs: outputs[0], - optimizer=optimizer, - return_loss=True, - ) - loss = outputs["loss"] - print(f"step {step} loss {loss}") - if booster.plugin.stage_manager.is_last_stage(): - global_loss = all_reduce_mean(loss, plugin) - - optimizer.step() - - if booster.plugin.stage_manager.is_last_stage(): - grad_norm = optimizer.get_grad_norm() - step_bar.set_postfix({"loss": global_loss.item(), "grad_norm": grad_norm}) - - optimizer.step() - optimizer.zero_grad() - - prof.step() - else: - total_loss = 0 - for step, batch in enumerate(dataloader): - prof.start() - input_ids = batch["input_ids"].to(device=model.module.device) - attention_mask = batch["attention_mask"].to(device=model.module.device) - outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids) - loss = outputs.loss - print(f"loss {loss}") - loss = loss / GRADIENT_ACCUMULATION_STEPS - booster.backward(loss, optimizer) - print(f"finish backward") - if (step + 1) % GRADIENT_ACCUMULATION_STEPS == 0: - optimizer.step() - optimizer.zero_grad() - print(f"finish optimizer step") - - total_loss += loss.item() - prof.step() - - print(f"Epoch {epoch + 1}, Loss: {total_loss / len(dataloader)}") - print(f"Profile Stop") - prof.stop() - - -if __name__ == "__main__": - test_hybrid_qwen() diff --git a/applications/ColossalChat/tests/test_log_prob.py b/applications/ColossalChat/tests/test_log_prob.py deleted file mode 100644 index 5e71f089cb4d..000000000000 --- a/applications/ColossalChat/tests/test_log_prob.py +++ /dev/null @@ -1,69 +0,0 @@ -import random -import time - -import torch - - -def code1(target, vocab_start_index, vocab_end_index): - """index Put""" - target_mask = (target < vocab_start_index) | (target >= vocab_end_index) - masked_target = target.clone() - vocab_start_index - masked_target[target_mask] = 0 - return masked_target - - -def code2(target, vocab_start_index, vocab_end_index): - """bool multiply""" - target_mask = (target < vocab_start_index) | (target >= vocab_end_index) - masked_target = target.clone() - vocab_start_index - masked_target *= ~target_mask - return masked_target - - -def test_performance(): - batch_size = 8 - sizes = [4096, 8192, 16384, 32768, 131072] - code1_times = [] - code2_times = [] - - for size in sizes: - target = torch.randint( - 0, - size, - ( - batch_size, - size, - ), - ).to("npu") - vocab_start_index = random.randint(0, size // 2) - vocab_end_index = random.randint(size // 2, size) - - # warmup - for _ in range(5): - code1(target, vocab_start_index, vocab_end_index) - code2(target, vocab_start_index, vocab_end_index) - - # Code 1: index input - start_time = time.time() - for _ in range(10): - code1(target, vocab_start_index, vocab_end_index) - code1_time = (time.time() - start_time) / 10 - code1_times.append(code1_time) - - # Code 2: bool multiply - start_time = time.time() - for _ in range(10): - code2(target, vocab_start_index, vocab_end_index) - code2_time = (time.time() - start_time) / 10 - code2_times.append(code2_time) - - print(f"DataSize: {size}") - print(f" Code 1:index input AvgRuntime: {code1_time:.6f} s") - print(f" Code 2:bool multiply AvgRuntime {code2_time:.6f} s") - # print(f" acceleration ratio: {(code1_time/code2_time-1)*100:.2f}%") - print(f" acceleration ratio: {(code1_time/code2_time - 1)*100:.2f}%") - - -if __name__ == "__main__": - print("\n===== Performance Benchmark =====") - test_performance() diff --git a/applications/ColossalChat/tests/test_ray.py b/applications/ColossalChat/tests/test_ray.py deleted file mode 100644 index 9868c6ed56b9..000000000000 --- a/applications/ColossalChat/tests/test_ray.py +++ /dev/null @@ -1,93 +0,0 @@ -import time - -import ray -import ray.util.collective as cc -import torch -from coati.distributed.comm import ray_broadcast_object, ray_broadcast_tensor_dict - -from colossalai.testing import parameterize - - -@ray.remote(num_cpus=1, num_gpus=0, resources={"NPU": 1}) -class Worker: - def __init__(self, rank, world_size): - self.rank = rank - self.world_size = world_size - self.group_name = "default" - cc.init_collective_group(world_size, rank, backend="hccl", group_name=self.group_name) - - def run_ray_broadcast_object(self, obj, src, device): - # ray_broadcast_object - received_obj = ray_broadcast_object(obj, src, device, group_name=self.group_name) - return received_obj - - def run_ray_broadcast_tensor_dict(self, tensor_dict, src, device): - # ray_broadcast_tensor_dict - received_dict = ray_broadcast_tensor_dict(tensor_dict, src, device, group_name=self.group_name) - return received_dict - - def destroy_worker(self): - cc.destroy_collective_group(self.group_name) - - -@parameterize( - "test_config", - [ - { - "precision": torch.bfloat16, - "device": "npu", - "num_devices": 1, - }, - ], -) -def test_comm(test_config): - # ray.init() - ray.init(address="local", namespace="ray-example") - # ray.init(_node_ip_address='10.0.0.5', namespace="ray-example") - - src = 0 - device = test_config["device"] - # create 4 - workers = [Worker.remote(i, test_config["num_devices"]) for i in range(test_config["num_devices"])] - - ############# - # 1. test ray_broadcast_object - ############# - # init broadcast_object data - test_obj = {"data": torch.tensor([1, 2, 3]), "message": "hello"} - - # run run_ray_broadcast_object - results = [worker.run_ray_broadcast_object.remote(test_obj, src, device) for worker in workers] - - time.sleep(60) - # get result - results = ray.get(results) - - for i, result in enumerate(results): - print(f"ray_broadcast_object Rank {i} received object: {result}") - - ############# - # 2. test ray_broadcast_tensor_dict - ############# - test_tensor_dict = { - "tensor1": torch.tensor([1, 2, 3], device=device), - "tensor2": torch.tensor([[4, 5], [6, 7]], device=device), - } - - # run ray_broadcast_tensor_dict - results = [worker.run_ray_broadcast_tensor_dict.remote(test_tensor_dict, src, device) for worker in workers] - - # get result - results = ray.get(results) - - for i, result in enumerate(results): - print(f"run_ray_broadcast_tensor_dict Rank {i} received object: {result}") - - # destory workers - for worker in workers: - worker.destroy_worker.remote() - ray.shutdown() - - -if __name__ == "__main__": - test_comm() diff --git a/applications/ColossalChat/tests/test_ray_vllm.py b/applications/ColossalChat/tests/test_ray_vllm.py deleted file mode 100644 index 37ea241dee4c..000000000000 --- a/applications/ColossalChat/tests/test_ray_vllm.py +++ /dev/null @@ -1,103 +0,0 @@ -import argparse -import time - -import ray -import ray.util.collective as cc -import torch -from coati.distributed.comm import ray_broadcast_tensor_dict -from vllm import LLM, SamplingParams - -from colossalai.testing import parameterize - -parser = argparse.ArgumentParser(description="VLLM args.") -parser.add_argument( - "-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. " -) -parser.add_argument("-l", "--max_length", type=int, default=8192, help="Max sequence length") -parser.add_argument("-w", "--world_size", type=int, default=8, help="Gpu nums") -parser.add_argument("-t", "--temperature", type=float, default=0.8, help="Temperature") -parser.add_argument("--top_p", type=float, default=0.95, help="Top p") -parser.add_argument( - "-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. " -) -args = parser.parse_args() - -# Create a sampling params object. - - -@ray.remote(num_cpus=args.world_size, num_gpus=0, resources={"NPU": args.world_size}) -class Worker: - def __init__(self, rank, world_size): - self.rank = rank - self.world_size = world_size - self.group_name = "default" - cc.init_collective_group(world_size, rank, backend="hccl", group_name=self.group_name) - self.llm = LLM(model=args.model_path, max_model_len=args.max_length, tensor_parallel_size=args.world_size) - self.sampling_params = SamplingParams( - temperature=args.temperature, top_p=args.top_p, max_tokens=args.max_length - ) - - def run_ray_broadcast_object(self, obj, src, device): - # Create an LLM. - outputs = self.llm.generate(args.input_texts, self.sampling_params) - return outputs - - def run_ray_broadcast_tensor_dict(self, tensor_dict, src, device): - # ray_broadcast_tensor_dict - received_dict = ray_broadcast_tensor_dict(tensor_dict, src, device, group_name=self.group_name) - return received_dict - - def destroy_worker(self): - cc.destroy_collective_group(self.group_name) - - -@parameterize( - "test_config", - [ - { - "precision": torch.bfloat16, - "device": "npu", - "num_devices": 1, - }, - ], -) -def test_comm(test_config): - ray.init(address="local", namespace="ray-example") - # ray.init(_node_ip_address="10.0.0.3", namespace="ray-vllm") - src = 0 - device = test_config["device"] - # create 4 - workers = [Worker.remote(i, test_config["num_devices"]) for i in range(test_config["num_devices"])] - - ############# - # 1. test ray_broadcast_object - ############# - # init broadcast_object data - test_obj = {"data": torch.tensor([1, 2, 3]), "message": "hello"} - - # run run_ray_broadcast_object - # for i in range(5): - # if i > 2: - torch.npu.synchronize() - start_time = time.time() - results = [worker.run_ray_broadcast_object.remote(test_obj, src, device) for worker in workers] - - # get result - results = ray.get(results) - - end_time = time.time() - total_time = end_time - start_time - - print(f"total_time {total_time}") - - for i, result in enumerate(results): - print(f"ray_broadcast_object Rank {i} received object: {result}") - - # destory workers - for worker in workers: - worker.destroy_worker.remote() - ray.shutdown() - - -if __name__ == "__main__": - test_comm() diff --git a/applications/ColossalChat/tests/test_vllm.py b/applications/ColossalChat/tests/test_vllm.py deleted file mode 100644 index fc24cf1222de..000000000000 --- a/applications/ColossalChat/tests/test_vllm.py +++ /dev/null @@ -1,36 +0,0 @@ -import argparse - -from vllm import LLM, SamplingParams - -parser = argparse.ArgumentParser(description="VLLM args.") -parser.add_argument( - "-m", "--model_path", type=str, default="/home/duanjunwen/models/Qwen/Qwen2.5-14B", help="The model path. " -) -parser.add_argument("-l", "--max_length", type=int, default=8192, help="Max sequence length") -parser.add_argument("-tp", "--tp_size", type=int, default=8, help="Gpu nums") -parser.add_argument("-pp", "--pp_size", type=int, default=2, help="Gpu nums") -parser.add_argument("-t", "--temperature", type=float, default=0.8, help="Temperature") -parser.add_argument("--top_p", type=float, default=0.95, help="Top p") -parser.add_argument( - "-i", "--input_texts", type=str, default="Find all prime numbers up to 100.", help="Prompts inputs. " -) -args = parser.parse_args() - -# Create a sampling params object. -sampling_params = SamplingParams(temperature=args.temperature, top_p=args.top_p, max_tokens=args.max_length) - -# Create an LLM. -llm = LLM( - model=args.model_path, - max_model_len=args.max_length, - tensor_parallel_size=args.tp_size, - pipeline_parallel_size=args.pp_size, -) -# Generate texts from the prompts. The output is a list of RequestOutput objects -# that contain the prompt, generated text, and other information. -outputs = llm.generate(args.input_texts, sampling_params) -# Print the outputs. -for output in outputs: - prompt = output.prompt - generated_text = output.outputs[0].text - print(f"Prompt: {prompt!r}, Generated text: {generated_text}") diff --git a/applications/ColossalChat/tests/test_vllm_multinode.py b/applications/ColossalChat/tests/test_vllm_multinode.py deleted file mode 100644 index 41c241890968..000000000000 --- a/applications/ColossalChat/tests/test_vllm_multinode.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -This example shows how to use Ray Data for running offline batch inference -distributively on a multi-nodes cluster. - -Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html -""" - -from typing import Any, Dict, List - -import numpy as np -import ray -from packaging.version import Version -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy -from vllm import LLM, SamplingParams - -assert Version(ray.__version__) >= Version("2.22.0"), "Ray version must be at least 2.22.0" - -# Create a sampling params object. -sampling_params = SamplingParams(temperature=0.8, top_p=0.95) - -# Set tensor parallelism per instance. -tensor_parallel_size = 1 - -# Set number of instances. Each instance will use tensor_parallel_size GPUs. -num_instances = 1 - - -# Create a class to do batch inference. -class LLMPredictor: - - def __init__(self): - # Create an LLM. - self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf", tensor_parallel_size=tensor_parallel_size) - - def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]: - # Generate texts from the prompts. - # The output is a list of RequestOutput objects that contain the prompt, - # generated text, and other information. - outputs = self.llm.generate(batch["text"], sampling_params) - prompt: List[str] = [] - generated_text: List[str] = [] - for output in outputs: - prompt.append(output.prompt) - generated_text.append(" ".join([o.text for o in output.outputs])) - return { - "prompt": prompt, - "generated_text": generated_text, - } - - -# Read one text file from S3. Ray Data supports reading multiple files -# from cloud storage (such as JSONL, Parquet, CSV, binary format). -ds = ray.data.read_text("s3://anonymous@air-example-data/prompts.txt") - - -# For tensor_parallel_size > 1, we need to create placement groups for vLLM -# to use. Every actor has to have its own placement group. -def scheduling_strategy_fn(): - # One bundle per tensor parallel worker - pg = ray.util.placement_group( - [{"GPU": 1, "CPU": 1}] * tensor_parallel_size, - strategy="STRICT_PACK", - ) - return dict(scheduling_strategy=PlacementGroupSchedulingStrategy(pg, placement_group_capture_child_tasks=True)) - - -resources_kwarg: Dict[str, Any] = {} -if tensor_parallel_size == 1: - # For tensor_parallel_size == 1, we simply set num_gpus=1. - resources_kwarg["num_gpus"] = 1 -else: - # Otherwise, we have to set num_gpus=0 and provide - # a function that will create a placement group for - # each instance. - resources_kwarg["num_gpus"] = 0 - resources_kwarg["ray_remote_args_fn"] = scheduling_strategy_fn - -# Apply batch inference for all input data. -ds = ds.map_batches( - LLMPredictor, - # Set the concurrency to the number of LLM instances. - concurrency=num_instances, - # Specify the batch size for inference. - batch_size=32, - **resources_kwarg, -) - -# Peek first 10 results. -# NOTE: This is for local testing and debugging. For production use case, -# one should write full result out as shown below. -outputs = ds.take(limit=10) -for output in outputs: - prompt = output["prompt"] - generated_text = output["generated_text"] - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") - -# Write inference output data out as Parquet files to S3. -# Multiple files would be written to the output destination, -# and each task would write one or more files separately. -# -# ds.write_parquet("s3://") From e966326efb2df466250d2b4149c49aa1a2c063ac Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Tue, 27 May 2025 14:14:47 +0800 Subject: [PATCH 21/24] [fix] fix some adapt modification --- colossalai/pipeline/schedule/one_f_one_b.py | 2 +- colossalai/shardformer/modeling/qwen2.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/colossalai/pipeline/schedule/one_f_one_b.py b/colossalai/pipeline/schedule/one_f_one_b.py index a21979d4ef7b..1f8582a5bfa8 100644 --- a/colossalai/pipeline/schedule/one_f_one_b.py +++ b/colossalai/pipeline/schedule/one_f_one_b.py @@ -92,7 +92,7 @@ def load_batch(self, data_iter: Iterable, device: Optional[torch.device] = None) assert ( self.num_microbatches >= self.stage_manager.num_stages - ), f"Number of microbatch should be larger than number of stages {self.num_microbatches} vs {self.stage_manager.num_stages}" + ), f"Number of microbatch should be larger than number of stages" if self.forward_only: self.num_microbatches = (self.batch_size - 1) // self.microbatch_size + 1 diff --git a/colossalai/shardformer/modeling/qwen2.py b/colossalai/shardformer/modeling/qwen2.py index 67d77757df95..620a7b09b4c7 100644 --- a/colossalai/shardformer/modeling/qwen2.py +++ b/colossalai/shardformer/modeling/qwen2.py @@ -199,7 +199,6 @@ def qwen2_model_forward( start_idx, end_idx = stage_index[0], stage_index[1] num_ckpt_layers = 0 - self.gradient_checkpointing = True if self.gradient_checkpointing and self.training: num_ckpt_layers = end_idx - start_idx # TODO: We can replace `gradient_checkpointing_enable` fn and initialize a gradient_checkpointing (List[bool]) for each layer @@ -852,9 +851,7 @@ def forward( hidden_states, 1, sp_group, 1 / sp_size, fp8_communication=shard_config.fp8_communication ) - layer_idx = 0 for decoder_layer in self.layers: - layer_idx += 1 if output_hidden_states: all_hidden_states += (hidden_states,) From 4812f7494e1cea127aa3bf451009cf2ca20fb3fa Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Wed, 28 May 2025 09:56:34 +0800 Subject: [PATCH 22/24] [fix] rm comments in modeling qwen --- applications/ColossalChat/rl_example.py | 2 +- colossalai/shardformer/modeling/qwen2.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/applications/ColossalChat/rl_example.py b/applications/ColossalChat/rl_example.py index 6be66c1c6e03..4efeb9f9c9eb 100644 --- a/applications/ColossalChat/rl_example.py +++ b/applications/ColossalChat/rl_example.py @@ -113,7 +113,7 @@ ) # Logging/Checkpointing parameters - parser.add_argument("-si", "--save-interval", type=int, default=20, help="Interval for saving checkpoints.") + parser.add_argument("-si", "--save-interval", type=int, default=100, help="Interval for saving checkpoints.") parser.add_argument("-sd", "--save-dir", type=str, default="./model", help="Directory for saving checkpoints.") parser.add_argument( "-esd", "--eval-save-dir", type=str, default="./eval", help="Directory for saving evaluation results." diff --git a/colossalai/shardformer/modeling/qwen2.py b/colossalai/shardformer/modeling/qwen2.py index 620a7b09b4c7..33256368404d 100644 --- a/colossalai/shardformer/modeling/qwen2.py +++ b/colossalai/shardformer/modeling/qwen2.py @@ -146,13 +146,6 @@ def qwen2_model_forward( # in this case, attention_mask is a dict rather than a tensor (batch_size, 1, seq_length, seq_length_with_past) attention_mask = None - # attention_mask = ColoAttention.prepare_attn_kwargs( - # mask_shape, - # hidden_states.dtype, - # hidden_states.device, - # q_padding_mask=attention_mask, - # is_causal=True, - # ) else: if self._attn_implementation == "flash_attention_2": # 2d mask is passed through the layers @@ -964,7 +957,6 @@ def forward( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - # force_sp_output_gather=False, ) hidden_states = outputs[0] From aff052c9243feb32b830a9dd2aed84d8fce00031 Mon Sep 17 00:00:00 2001 From: duanjunwen <935724073@qq.com> Date: Wed, 28 May 2025 10:02:29 +0800 Subject: [PATCH 23/24] [fix] rm comm, test and debug print --- .../coati/distributed/consumer.py | 2 - .../coati/distributed/grpo_consumer.py | 3 - .../ColossalChat/coati/distributed/launch.py | 64 ------------------- .../coati/distributed/producer.py | 5 +- 4 files changed, 1 insertion(+), 73 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/consumer.py b/applications/ColossalChat/coati/distributed/consumer.py index f92dc6c06968..b5e748d19264 100644 --- a/applications/ColossalChat/coati/distributed/consumer.py +++ b/applications/ColossalChat/coati/distributed/consumer.py @@ -55,9 +55,7 @@ def __init__( self.model_config = model_config self.plugin_config = plugin_config - # self.device = get_current_device() self.device = "npu" - # self.device = torch.device(f"npu:{torch.npu.current_device()}") self.lr_scheduler = None self.generate_config = generate_config diff --git a/applications/ColossalChat/coati/distributed/grpo_consumer.py b/applications/ColossalChat/coati/distributed/grpo_consumer.py index ea2ca0f7c53f..eaf3521b6381 100644 --- a/applications/ColossalChat/coati/distributed/grpo_consumer.py +++ b/applications/ColossalChat/coati/distributed/grpo_consumer.py @@ -341,7 +341,6 @@ def step(self, step_idx: int, pbar: Any, **kwargs) -> Optional[float]: num_action, self.plugin.shard_config, ) - del reference_model_logits else: # Dummy reference logprobs for data iterator. reference_action_log_probs = None @@ -421,7 +420,6 @@ def _criterion(outputs, inputs): num_action, self.plugin.shard_config, ) - del policy_model_logits if self.policy_loss_fn.beta > 0: with torch.no_grad(): @@ -435,7 +433,6 @@ def _criterion(outputs, inputs): num_action, self.plugin.shard_config, ) - del reference_model_logits per_token_kl = ( torch.exp(reference_action_log_probs - action_log_probs) - (reference_action_log_probs - action_log_probs) diff --git a/applications/ColossalChat/coati/distributed/launch.py b/applications/ColossalChat/coati/distributed/launch.py index 6bb10f9e7ac1..50169a49ff19 100644 --- a/applications/ColossalChat/coati/distributed/launch.py +++ b/applications/ColossalChat/coati/distributed/launch.py @@ -65,9 +65,6 @@ def launch_distributed( core_consumer = ALGO_MAP.get(core_algo, SimpleConsumer) train_dp_size = get_dp_size_fast(num_consumer_procs, plugin_config) - print( - f"inference_batch_size {inference_batch_size} num_producers {num_producers} train_batch_size {train_batch_size} train_dp_size {train_dp_size}" - ) assert (inference_batch_size * num_producers) % (train_batch_size * train_dp_size) == 0 dataset_path = train_dataset_config["path"] @@ -83,64 +80,6 @@ def launch_distributed( f"{project_name.replace(' ','_')}_run_{wandb_group_name}.jsonl", ) - # ########################################### - # # Old version, may lead colossalai init stuck in multinodes - # ############################################ - # procs = [] - # for i in range(num_producers): - # # producer = SimpleProducer.options(num_gpus=num_proc_per_producer).remote( - # producer = SimpleProducer.options(num_cpus=1, resources={"NPU":num_proc_per_producer}).remote( - # producer_idx=i, - # num_producers=num_producers, - # num_consumer_procs=num_consumer_procs, - # num_episodes=num_episodes, - # batch_size=inference_batch_size, - # dataset_config=dataset_config, - # dataloaders_config=dataloaders_config, - # model_config=inference_model_config, - # generate_config=generate_config, - # tokenizer_config=tokenizer_config, - # microbatch_size=inference_microbatch_size, - # backend=inference_backend, - # num_generations=num_generations, - # consumer_plugin_config=plugin_config, - # ) - # procs.append(producer) - # generate_config_consumer = copy.deepcopy(generate_config) - # generate_config_consumer.update( - # dict( - # backend=inference_backend, - # ) - # ) - # for i in range(num_consumer_procs): - # # consumer = core_consumer.options(num_gpus=1).remote( - # consumer = core_consumer.options(num_cpus=1, resources={"NPU":1}).remote( - # num_producers=num_producers, - # num_episodes=num_episodes, - # rank=i, - # world_size=num_consumer_procs, - # master_addr=master_addr, - # master_port=master_port, - # num_update_per_episode=num_update_per_episode, - # num_recv_per_update=num_recv_per_update, - # batch_size=train_batch_size, - # model_config=train_model_config, - # plugin_config=plugin_config, - # minibatch_size=train_minibatch_size, - # generate_config=generate_config_consumer, - # grpo_config=grpo_config, - # num_generations=num_generations, - # project_name=project_name, - # save_interval=save_interval, - # save_dir=save_dir, - # ) - # procs.append(consumer) - # ray.get([p.setup.remote() for p in procs]) - # ray.get([p.loop.remote() for p in procs]) - - ########################################### - # New version, assign master ip for colossalai & vllm respectively - ########################################### nodes = ray.nodes() node_info = { node["NodeID"]: { @@ -150,14 +89,12 @@ def launch_distributed( } # Default to 0 if no GPUs are available for node in nodes } - print(f"node_info {node_info}") gpu_to_node_id = [] gpu_to_ip_address = [] for node_id in node_info: for idx in range(int(node_info[node_id]["num_gpus"])): # use num_gpus instead of num_npus gpu_to_node_id.append(node_id) gpu_to_ip_address.append(node_info[node_id]["address"]) - print(f"node_info {node_info} \n gpu_to_node_id {gpu_to_node_id} \n gpu_to_ip_address {gpu_to_ip_address} \n") producer_procs = [] @@ -232,7 +169,6 @@ def launch_distributed( num_episodes=num_episodes, rank=i, world_size=num_consumer_procs, - # master_addr=master_addr, master_addr=consumer_master_ip_address, master_port=master_port, num_update_per_episode=num_update_per_episode, diff --git a/applications/ColossalChat/coati/distributed/producer.py b/applications/ColossalChat/coati/distributed/producer.py index 2911559929c2..436bbe32eb67 100644 --- a/applications/ColossalChat/coati/distributed/producer.py +++ b/applications/ColossalChat/coati/distributed/producer.py @@ -150,10 +150,8 @@ def __init__( raise ValueError(f"Unknown evaluation function type {evaluation_function_type}") else: print("No eval dataset provided, skip eval") - self.device = get_current_device() - # self.device = get_current_device() + self.device = "npu" - # self.device = torch.device(f"npu:{torch.npu.current_device()}") # init backend if backend in BACKEND_MAP: @@ -251,7 +249,6 @@ def loop(self) -> None: outputs["temperature"] = torch.tensor( [self.model.generate_config["temperature"]] * outputs["input_ids"].size(0) ).to(outputs["input_ids"].device) - # outputs = pre_send(outputs) ray_broadcast_tensor_dict( outputs, src=0, device=self.device, group_name=f"sync_data_{self.producer_idx}" ) From ed265e3ee0dae89f17ff930b1928772a945503bf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 May 2025 02:04:33 +0000 Subject: [PATCH 24/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- applications/ColossalChat/coati/distributed/producer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/producer.py b/applications/ColossalChat/coati/distributed/producer.py index 436bbe32eb67..66a3c5967894 100644 --- a/applications/ColossalChat/coati/distributed/producer.py +++ b/applications/ColossalChat/coati/distributed/producer.py @@ -15,8 +15,6 @@ from torch.utils.data import DataLoader, DistributedSampler from transformers import AutoTokenizer -from colossalai.utils import get_current_device - from .comm import ray_broadcast_tensor_dict from .inference_backend import BACKEND_MAP from .utils import safe_append_to_jsonl_file